Use new Singeli type destructuring (no change to generated C)

This commit is contained in:
Marshall Lochbaum 2024-05-25 17:47:28 -04:00
parent 05690c001a
commit 90863851b7
26 changed files with 368 additions and 386 deletions

@ -1 +1 @@
Subproject commit ce6ef5d06d35777f0016bbfe0c6c1cf6a9c1b48e
Subproject commit e3eb3d56b52604b43b07493f69dedfc43c83966d

View File

@ -1,37 +1,37 @@
# compact casting for the annoying intrinsic type system
def v2i{x:T if w256{T}} = if(isint{eltype{T}}) x else [32]u8 ~~ x
def v2i{x:T=[_]E if w256{T}} = if(isint{E}) x else [32]u8 ~~ x
def v2f{x:T if w256{T}} = [8]f32 ~~ x
def v2d{x:T if w256{T}} = [4]f64 ~~ x
def undefPromote{T, x:X if w128{X} and w256{T} and eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}}
def undefPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}}
# load & store
def loadLow{ptr:P, w if w256{eltype{P}} and w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}}
def loadLow{ptr:P, w if w256{eltype{P}} and w==256} = load{ptr}
def loadLow{ptr:*T, w if w256{T} and w<=128} = undefPromote{T, loadLow{*n_h{T} ~~ ptr, w}}
def loadLow{ptr:*T, w if w256{T} and w==256} = load{ptr}
def storeLow{ptr:P, w, x:T if w256{T} and w<=128} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:P, w, x:T if w256{T} and w==256} = store{*T~~ptr, 0, x}
def storeLow{ptr:*T, w, x:X if w256{X} and w<=128} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:*T, w, x:X if w256{X} and w==256} = store{*X~~ptr, 0, x}
# float comparison
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n}
def unord{a:T,b:T if T==[8]f32} = f32cmpAVX{a,b,3}
def unord{a:T,b:T if T==[4]f64} = f64cmpAVX{a,b,3}
def unord{a:T, b:T==[8]f32} = f32cmpAVX{a,b,3}
def unord{a:T, b:T==[4]f64} = f64cmpAVX{a,b,3}
# f32 arith
def rsqrtE{a:([8]f32)} = emit{[8]f32, '_mm256_rsqrt_ps', a}
def rcpE{a:([8]f32)} = emit{[8]f32, '_mm256_rcp_ps', a}
def rsqrtE{a:T==[8]f32} = emit{T, '_mm256_rsqrt_ps', a}
def rcpE{a:T==[8]f32} = emit{T, '_mm256_rcp_ps', a}
# conversion
def half{x:T, i if w256{T} and knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
def half{x:T, (0) if w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
def half{x:T, 0 if w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
def pair{a:T,b:T if w128{T}} = n_d{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b}
def widen{T==[4]f64, x:X if X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x}
def widen{T==[4]f64, x:X if X==[4]f32} = emit{T, '_mm256_cvtps_pd', x}
def widen{T==[4]f64, x:X if w128i{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T, x:X if w256{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}}
def widen{T==[4]f64, x:([4]i32)} = emit{T, '_mm256_cvtepi32_pd', x}
def widen{T==[4]f64, x:([4]f32)} = emit{T, '_mm256_cvtps_pd', x}
def widen{T==[4]f64, x:X=[_]U if w128i{X} and width{U}<32} = widen{T, widen{[4]i32, x}}
def widen{T=[k]_, x:X=[l]_ if w256{X} and l>k} = widen{T, half{x,0}}
# structural operations
@ -50,5 +50,5 @@ def homMask{x:T if w256{T}} = topMask{x}
def homAny{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0
def homAll{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff
def topAny{x:T if w256i{T} and elwidth{T}>=32} = topMask{x} != 0
def topAll{x:T if w256i{T} and elwidth{T}>=32} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T=[_]E if w256i{T} and width{E}>=32} = topMask{x} != 0
def topAll{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<<k)-1

View File

@ -1,23 +1,23 @@
# questionable pack
def unpackQ{a:T,b:T if T==[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}}
def unpackQ{a:T,b:T if T==[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}}
def unpackQ{a:T,b:T if T==[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi32', a, b}}}
def unpackQ{a:T,b:T if T==[ 4]i64} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi64', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi64', a, b}}}
def unpackQ{a:T,b:T==[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}}
def unpackQ{a:T,b:T==[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}}
def unpackQ{a:T,b:T==[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi32', a, b}}}
def unpackQ{a:T,b:T==[ 4]i64} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi64', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi64', a, b}}}
# inverse of questionable pack; these saturate the argument
def packQ{a:T,b:T if T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T if T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
def packQ{a:T,b:T if T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
def packQ{a:T,b:T if T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
def packQ{a:T,b:T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
def packQ{a:T,b:T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
# super questionable pack - assumes high halves are zero
def packQQ{a:T,b:T if T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
def packQQ{a:T,b:T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
def packQQ{{a, b}} = packQQ{a, b}
# arith
def mulh {a:T,b:T if [16]i16==T} = emit{T, '_mm256_mulhi_epi16', a, b}
def mulh {a:T,b:T if [16]u16==T} = emit{T, '_mm256_mulhi_epu16', a, b}
def mul32{a:T,b:T if [ 4]i64==T} = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments
def mul32{a:T,b:T if [ 4]u64==T} = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments
def mulh {a:T,b:T==[16]i16} = emit{T, '_mm256_mulhi_epi16', a, b}
def mulh {a:T,b:T==[16]u16} = emit{T, '_mm256_mulhi_epu16', a, b}
def mul32{a:T,b:T==[ 4]i64} = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments
def mul32{a:T,b:T==[ 4]u64} = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments
# structural operations
@ -40,13 +40,13 @@ def sel{L, x:T, i:I if w256{T} and lvec{L,8,32} and w256{I,32}} = T ~~ emit{[32]
def sel{L, x:T, i:I if w256{T} and lvec{L,16,8} and w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}
# masked store; F variants may not be a single instruction
def topMaskStore{p:P, m:M, v:T if w256i{M, 32} and w256{T,elwidth{M}} and eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
def topMaskStore{p:P, m:M, v:T if w256i{M, 64} and w256{T,elwidth{M}} and eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}
def homMaskStore{p:P, m:M, v:T if w256i{M} and w256{T,elwidth{M}} and eltype{P}==T} = topMaskStore{p, m, v}
def topMaskStore{p:*T, m:M, v:T if w256i{M, 32} and w256{T,elwidth{M}}} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
def topMaskStore{p:*T, m:M, v:T if w256i{M, 64} and w256{T,elwidth{M}}} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}
def homMaskStore{p:*T, m:M, v:T if w256i{M} and w256{T,elwidth{M}}} = topMaskStore{p, m, v}
def topMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def topMaskStoreF{p:*P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:*P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}
# mask stuff
def topMask{x:T if w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
@ -57,8 +57,8 @@ def topMask{x:T if w256{T, 16}} = {
def homAny{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
def homAll{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
def topAny{x:T if w256i{T}} = topMask{x} != 0
def topAll{x:T if w256i{T}} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T if w256i{T}} = topMask{x} != 0
def topAll{x:T=[k]_ if w256i{T}} = topMask{x} == (1<<k)-1
def homMask{a:T, b:T if w256i{T,16}} = homMask{shuf{[4]u64, packQ{ty_s{a},ty_s{b}}, 4b3120}}
def topAny{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
@ -70,12 +70,12 @@ def topAll{x:T if w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0}
# conversion
def widen{T==[16]u16, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi16', x}; def widen{T==[16]i16, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi16', x}
def widen{T==[ 8]u32, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi32', x}; def widen{T==[ 8]i32, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi32', x}
def widen{T==[ 8]u32, x:X if X==[8]u16} = emit{T, '_mm256_cvtepu16_epi32', x}; def widen{T==[ 8]i32, x:X if X==[8]i16} = emit{T, '_mm256_cvtepi16_epi32', x}
def widen{T==[ 4]u64, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi64', x}; def widen{T==[ 4]i64, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi64', x}
def widen{T==[ 4]u64, x:X if X==[8]u16} = emit{T, '_mm256_cvtepu16_epi64', x}; def widen{T==[ 4]i64, x:X if X==[8]i16} = emit{T, '_mm256_cvtepi16_epi64', x}
def widen{T==[ 4]u64, x:X if X==[4]u32} = emit{T, '_mm256_cvtepu32_epi64', x}; def widen{T==[ 4]i64, x:X if X==[4]i32} = emit{T, '_mm256_cvtepi32_epi64', x}
def widen{T==[16]u16, x:X==[16]u8} = emit{T, '_mm256_cvtepu8_epi16', x}; def widen{T==[16]i16, x:X==[16]i8} = emit{T, '_mm256_cvtepi8_epi16', x}
def widen{T==[ 8]u32, x:X==[16]u8} = emit{T, '_mm256_cvtepu8_epi32', x}; def widen{T==[ 8]i32, x:X==[16]i8} = emit{T, '_mm256_cvtepi8_epi32', x}
def widen{T==[ 8]u32, x:X==[8]u16} = emit{T, '_mm256_cvtepu16_epi32', x}; def widen{T==[ 8]i32, x:X==[8]i16} = emit{T, '_mm256_cvtepi16_epi32', x}
def widen{T==[ 4]u64, x:X==[16]u8} = emit{T, '_mm256_cvtepu8_epi64', x}; def widen{T==[ 4]i64, x:X==[16]i8} = emit{T, '_mm256_cvtepi8_epi64', x}
def widen{T==[ 4]u64, x:X==[8]u16} = emit{T, '_mm256_cvtepu16_epi64', x}; def widen{T==[ 4]i64, x:X==[8]i16} = emit{T, '_mm256_cvtepi16_epi64', x}
def widen{T==[ 4]u64, x:X==[4]u32} = emit{T, '_mm256_cvtepu32_epi64', x}; def widen{T==[ 4]i64, x:X==[4]i32} = emit{T, '_mm256_cvtepi32_epi64', x}
def narrow{T, x:X if w256i{X,32} and width{T}==8} = {
a:= packQ{x, x}
@ -93,5 +93,5 @@ def narrow{T, x:X if w256u{X,64} and T==u16} = re_el{T, sel{[16]i8, narrow{u32,x
def narrow{T, x:X if w256u{X,64} and T== u8} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, 4*iota{32}}}}
def cvt2{(i32), x:X if X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
def cvt2{(f64), x:X if X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}
def cvt2{(i32), x:X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
def cvt2{(f64), x:X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}

View File

@ -1,33 +1,30 @@
local {
def ismask{M} = if (isvec{M}) u1==eltype{M} else 0
def suf{T} = {
if (isfloat{T}) (if (width{T}==32) 'ps' else 'pd')
else merge{'epi', fmtnat{width{T}}}
}
def suf{V if isvec{V}} = suf{eltype{V}}
def suf{[_]T} = suf{T}
def pref{w} = merge{'_mm', if (w==128) '' else fmtnat{w}, '_'}
def pref{V if isvec{V}} = pref{width{V}}
def pref{V=[_]_} = pref{width{V}}
}
local def re_mask{M, sub} = {
def l = vcount{M}; def w = max{32,l}
local def re_mask{[l]_, sub} = {
def w = max{32,l}
sub{fmtnat{l}, fmtnat{w}, ty_u{w}}
}
def reinterpret{M, a:T if ismask{M} and width{T}==width{M}} = {
def reinterpret{M=[l==width{T}](u1), a:T} = {
re_mask{M, {l,w,W} => emit{M, merge{'_cvtu',w,'_mask',l}, promote{W, a}}}
}
def reinterpret{T, a:M if ismask{M} and width{T}==width{M}} = {
def reinterpret{T, a:M=[l==width{T}](u1)} = {
re_mask{M, {l,w,W} => cast_i{T, emit{W, merge{'_cvtmask',l,'_u',w}, a}}}
}
def maskStore{p:*V, m:M, v:V if ismask{M} and isvec{V} and vcount{M}==vcount{V}} = {
def maskStore{p:*V, m:[l](u1), v:V=[l]_} = {
emit{void, merge{pref{V}, 'mask_storeu_', suf{V}}, p, m, v}
}
def topMaskReg{x:V} = emit{[vcount{V}]u1, merge{pref{V},'mov',suf{V},'_mask'}, x}
def topMask{x:T if isvec{T} and 512==width{T}} = ty_u{vcount{T}}~~topMaskReg{x}
def homMask{x:T if isvec{T} and 512==width{T}} = topMask{x}
def topMaskReg{x:V=[k]_} = emit{[k]u1, merge{pref{V},'mov',suf{V},'_mask'}, x}
def topMask{x:V=[k]_ if 512==width{V}} = ty_u{k}~~topMaskReg{x}
def homMask{x:V=[_]_ if 512==width{V}} = topMask{x}
def maskToHom{T, x:M if ismask{M} and isvec{T} and vcount{M}==vcount{T}} = {
emit{T, merge{pref{T},'movm_',suf{T}}, x}
}
def maskToHom{V=[l]_, x:[l](u1)} = emit{V, merge{pref{V},'movm_',suf{V}}, x}

View File

@ -28,16 +28,16 @@ def andnot{a, b if anyNum{a} and anyNum{b}} = a & ~b
def load{p:P, n if isvec{eltype{P}}} = assert{0}
def store{p:P, n, v if isvec{eltype{P}}} = assert{0}
def load{p:P if isptr{P}} = load{p, 0}
# def store{p:P, v if isptr{P}} = store{p, 0, v}
def loadu{p:T if isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p}
def storeu{p:T, v:(eltype{T}) if isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v}
def loadu{p:T if issigned{eltype{T}}} = loadu {*ty_u{eltype{T}} ~~ p}
def storeu{p:T, v:(eltype{T}) if issigned{eltype{T}}} = storeu{*ty_u{eltype{T}} ~~ p, ty_u{v}}
def loadu{p:T if elwidth{T}==8} = load{p}
def storeu{p:T, v:(eltype{T}) if elwidth{T}==8} = store{p, v}
def load {p:*[_]_, n } = assert{0}
def store{p:*[_]_, n, v} = assert{0}
def load{p:*_} = load{p, 0}
# def store{p:*_, v} = store{p, 0, v}
def loadu {p:*T if isunsigned{T}} = emit{T, merge{'loadu_u', fmtnat{width{T}}}, p}
def storeu{p:*T, v:T if isunsigned{T}} = emit{void, merge{'storeu_u',fmtnat{width{T}}}, p, v}
def loadu {p:*T if issigned{T}} = loadu {*ty_u{T} ~~ p}
def storeu{p:*T, v:T if issigned{T}} = storeu{*ty_u{T} ~~ p, ty_u{v}}
def loadu {p:*T if width{T}==8} = load{p}
def storeu{p:*T, v:T if width{T}==8} = store{p, v}
def reinterpret{T, x:T} = x
@ -49,8 +49,8 @@ def exportT{name, fs} = { v:*oneType{fs} = fs; export{name, v} }
def rare{x if knum{x}} = x
def rare{x:(u1)} = emit{u1, '__builtin_expect', x, 0}
def assert{c, ...msg} = { if (not same{c,1}) { show{...msg}; 0{} } }
def assert{(0)} = assert{0, 'failed assertion'}
def assert{(1)} = 1
def assert{0} = assert{0, 'failed assertion'}
def assert{1} = 1
def unreachable{} = emit{void, 'si_unreachable'}
def assert{x:(u1)} = { if (not x) emit{void, 'si_unreachable'} }
@ -62,11 +62,8 @@ def oneVal{{h, ...t}} = {
def oneVal{{}} = {}
def oneType{x} = oneVal{each{type, x}}
def anyNum{x} = knum{x}
def anyNum{x:T} = isprim{T}
def anyInt{x} = knum{x} and (x>>0) == x
def anyInt{x:T} = isint{T}
def anyNum = match { {x:T}=>isprim{T}; {x} => knum{x} }
def anyInt = match { {x:T}=>isint{T} ; {x} => knum{x} and (x>>0) == x }
# vector width/type checks
def w64 {T} = isvec{T} and width{T}==64; def w64 {T,w} = w64{T} and elwidth{T}==w
@ -74,10 +71,9 @@ def w128{T} = isvec{T} and width{T}==128; def w128{T,w} = w128{T} and elwidth{T}
def w256{T} = isvec{T} and width{T}==256; def w256{T,w} = w256{T} and elwidth{T}==w
# width+type checks
def genchk{B, F} = {
def r{T } = B{T} and F{eltype{T}}
def r{T,w} = B{T} and F{eltype{T}} and elwidth{T}==w
r
def genchk{B, F} = match {
{V=[_]T } => B{V} and F{T}
{V=[_]T, w} => B{V} and F{T} and width{T}==w
}
def w256i = genchk{w256, isint}; def w128i = genchk{w128, isint}; def w64i = genchk{w64, isint}
def w256s = genchk{w256, issigned}; def w128s = genchk{w128, issigned}; def w64s = genchk{w64, issigned}
@ -104,7 +100,7 @@ def re_el{E, x:V} = re_el{E,V} ~~ x
local def qualChange{q} = {
def f{w if knum{w}} = primtype{q, w}
def f{T if isprim{T}} = primtype{q, width{T}}
def f{T if isvec{T}} = re_el{f{eltype{T}}, T}
def f{V=[_]T} = re_el{f{T}, V}
def f{x:T} = f{T}~~x
}
def ty_u = qualChange{'u'}
@ -115,14 +111,14 @@ def w_n{T, w if isprim{T}} = primtype{quality{T}, w}
def w_d{T if isprim{T}} = to_w{T, width{T}*2} # double/halve primitive type width
def w_h{T if isprim{T}} = to_w{T, width{T}/2}
def n_d{T if isvec{T}} = [vcount{T}*2](eltype{T}) # double/halve vector count
def n_h{T if isvec{T}} = [vcount{T}/2](eltype{T})
def n_d{[k]T} = [k*2]T # double/halve vector count
def n_h{[k]T} = [k/2]T
def el_d{T if isvec{T}} = [vcount{T}](w_d{eltype{T}}) # double/halve element width, preserving count
def el_h{T if isvec{T}} = [vcount{T}](w_h{eltype{T}})
def el_d{[k]T} = [k](w_d{T}) # double/halve element width, preserving count
def el_h{[k]T} = [k](w_h{T})
def el_m{T if isvec{T}} = re_el{w_d{eltype{T}}, T}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width
def el_s{T if isvec{T}} = re_el{w_h{eltype{T}}, T}; def el_s{x:T} = re_el{T}~~x
def el_m{V=[_]T} = re_el{w_d{T}, V}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width
def el_s{V=[_]T} = re_el{w_h{T}, V}; def el_s{x:T} = re_el{T}~~x
# type stats
def minvalue{T if isunsigned{T}} = 0
@ -153,7 +149,7 @@ def has_simd = hasarch{'X86_64'} or hasarch{'AARCH64'}
def fast_BMI2{} = hasarch{'BMI2'} and not SLOW_PDEP
# test if vector has a specific width & element type
def lvec{T, n, w} = isvec{T} and vcount{T}==n and elwidth{T}==w
def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
# base cases
def {
@ -169,11 +165,12 @@ def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeate
def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
def homMask{...vs if length{vs}>1} = {
def n = length{vs}
def T = oneType{vs}
def RT = ty_u{max{8,vcount{T}*n}}
def a = promote{RT, homMask{...slice{vs,0,n/2}}}
def b = promote{RT, homMask{...slice{vs,n/2}}}
(b<<(n/2 * vcount{T})) | a
def [k]_ = oneType{vs}
def RT = ty_u{max{8,k*n}}
def sl{...a} = promote{RT, homMask{...slice{vs,...a}}}
def h = n/2
def a = sl{0,h}
(sl{h} << (h * k)) | a
}
if_inline (hasarch{'X86_64'}) {
@ -195,15 +192,15 @@ def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b}
def packQ{{a, b}} = packQ{a, b}
def pair{{a, b}} = pair{a, b}
def widen{T, x:X if T==X} = x
def narrow{T, x:X if T==eltype{X}} = x
def undefPromote{T, x:X if T==X} = x
def cvt{T, x:X if T==eltype{X}} = x
def widen{T, x:T} = x
def narrow{T, x:[_]T} = x
def undefPromote{T, x:T} = x
def cvt{T, x:[_]T} = x
def broadcast{T, v if isvec{T}} = vec_broadcast{T, promote{eltype{T},v}}
def make{T, ...xs if isvec{T}} = vec_make{T, ...xs}
def iota{T if isvec{T}} = make{T, ...iota{vcount{T}}}
def absu{a:T if isvec{T}} = ty_u{abs{a}}
def broadcast{V=[_]T, v} = vec_broadcast{V, promote{T,v}}
def make{V=[_]_, ...xs} = vec_make{V, ...xs}
def iota{V=[k]_} = make{V, ...iota{k}}
def absu{a:[_]_} = ty_u{abs{a}}
def floor = __floor
def ceil = __ceil
@ -246,13 +243,13 @@ def tail{n,x} = x & ((1<<n) - 1) # get the n least significant bits
def bit {k,x} = x & (1<<k) # get the k-th bit
# range logic
def inRangeLen{x:TS, start, count if issigned{eltype{TS}}} = { # ∊ [start;start+count)
def inRangeLen{x:TS=[_]T, start, count if issigned{T}} = { # ∊ [start;start+count)
def TU = ty_u{TS}
(TU~~(x-TS**start)) < TU**count
}
def inRangeLen{x:TU, start, count if isunsigned{eltype{TU}}} = { # ∊ [start;start+count)
def inRangeLen{x:TU=[_]T, start, count if isunsigned{T}} = { # ∊ [start;start+count)
def TS = ty_s{TU}
def h = 1 << (elwidth{TU}-1)
def h = 1 << (width{T}-1)
(TS~~(x-TU**(start-h))) < TS**(count-h)
}
def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1} # ∊ [start;end]
@ -316,7 +313,7 @@ def tree_fold{F, x} = {
def tree_fold{F, {x}} = x
def eachx{F, ...args} = {
def l = tree_fold{max, each{{x} => if(ktup{x}) length{x} else 0, args}}
def l = tree_fold{max, each{{x} => if(istup{x}) length{x} else 0, args}}
each{F, ...each{{x} => if (istup{x}) x else l**x, args}}
}

View File

@ -41,21 +41,21 @@ fn max_scan{T, up}(x:*T, len:u64) : void = {
def getsel{...x} = assert{'shuffling not supported', show{...x}}
if_inline (hasarch{'AVX2'}) {
def getsel{h:H if lvec{H, 16, 8}} = {
def getsel{h:H=[16]T if width{T}==8} = {
sel{H, pair{h,h}, .}
}
def getsel{v:V if lvec{V, 32, 8}} = {
def getsel{v:V=[vl==32]T if width{T}==8} = {
def H = n_h{V}
vtop := V**(vcount{V}/2)
vtop := V**(vl/2)
hs := each{shuf{[4]u64, v, .}, tup{4b3232, 4b1010}}
{i} => homBlend{...each{sel{H,.,i}, hs}, V~~i<vtop}
}
def getsel{v:V if lvec{V, 8, 32}} = sel{V, v, .}
def getsel{v:V=[8]T if width{T}==32} = sel{V, v, .}
}
# Move evens to half 0 and odds to half 1
def uninterleave{x:V if hasarch{'AVX2'}} = {
def vl = vcount{V}; def bytes = width{eltype{V}}/8
def uninterleave{x:V=[vl]T if hasarch{'AVX2'}} = {
def bytes = width{T}/8
def i = 2*iota{vl/4}
def i2= join{table{+, bytes*merge{i,i+1}, iota{bytes}}}
t := V~~sel{[16]u8, re_el{u8,x}, make{[32]u8, merge{i2,i2}}}

View File

@ -4,13 +4,17 @@ def b_get{x:(*u64), n:(ux)} = {
((load{x,n>>6}>>(n&63)) & 1) != 0
}
def b_getBatchLo{sz, x:(*u64), n:(ux) if sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2})
def b_getBatchLo{sz, x:(*u64), n:(ux) if sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4})
def b_getBatchLo{sz, x:(*u64), n:(ux) if sz>=8} = load{*ty_u{sz}~~x, n}
def b_getBatchLo{sz, x:(*u64), n:(ux)} = match (sz) {
{2} => (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2})
{4} => (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4})
{sz if sz>=8} => load{*ty_u{sz}~~x, n}
}
def b_getBatch{sz, x:(*u64), n:(ux) if sz==2} = b_getBatchLo{sz, x, n} & 3
def b_getBatch{sz, x:(*u64), n:(ux) if sz==4} = b_getBatchLo{sz, x, n} & 15
def b_getBatch{sz, x:(*u64), n:(ux) if sz>=8} = load{*ty_u{sz}~~x, n}
def b_getBatch{sz, x:(*u64), n:(ux)} = match (sz) {
{2} => b_getBatchLo{sz, x, n} & 3
{4} => b_getBatchLo{sz, x, n} & 15
{sz if sz>=8} => load{*ty_u{sz}~~x, n}
}
def b_set{x:(*u64), n:(ux), v:(u1)} = {
@ -20,39 +24,40 @@ def b_set{x:(*u64), n:(ux), v:(u1)} = {
else store{x,n>>6,p & ~m}
}
def b_setBatch{sz, x:(*u64), n:(ux), v} = {
vc:u64 = promote{u64,v}
am:u64 = 64/sz
w:u64 = load{x,n/am}
sh:u64 = (n&(am-1)) * sz
w&= ~(ones{u64,sz}<<sh)
w|= (vc<<sh)
store{x, n/am, w}
}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz==4} = {
x8:= *u8 ~~ x
#w:u64 = cast_i{u64, load{x8,n/2}}
#sh:u64 = (n&1) * 4
#w&= ~(15<<sh)
#w|= (cast_i{u64,v}<<sh)
w:u8 = load{x8,n/2}
if ((n&1)==1) {
w&= ~(cast{u8,15}<<4)
w|= (cast_i{u8,v}<<4)
} else {
w&= ~(cast{u8,15})
w|= (cast_i{u8,v}<<0)
def b_setBatch{sz, x:(*u64), n:(ux), v} = match (sz) {
{4} => {
x8:= *u8 ~~ x
#w:u64 = cast_i{u64, load{x8,n/2}}
#sh:u64 = (n&1) * 4
#w&= ~(15<<sh)
#w|= (cast_i{u64,v}<<sh)
w:u8 = load{x8,n/2}
if ((n&1)==1) {
w&= ~(cast{u8,15}<<4)
w|= (cast_i{u8,v}<<4)
} else {
w&= ~(cast{u8,15})
w|= (cast_i{u8,v}<<0)
}
store{x8, n/2, cast_i{u8,w}}
}
{ 8} => store{*u8 ~~ x, n, cast_i{u8, v}}
{16} => store{*u16 ~~ x, n, cast_i{u16,v}}
{32} => store{*u32 ~~ x, n, cast_i{u32,v}}
{64} => store{ x, n, cast_i{u64,v}}
{sz} => {
vc:u64 = promote{u64,v}
am:u64 = 64/sz
w:u64 = load{x,n/am}
sh:u64 = (n&(am-1)) * sz
w&= ~(ones{u64,sz}<<sh)
w|= (vc<<sh)
store{x, n/am, w}
}
store{x8, n/2, cast_i{u8,w}}
}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz==64} = store{ x, n, cast_i{u64,v}}
def spreadBits{T==[32]u8, a:(u32)} = {
def idxs = iota{32}
@ -64,22 +69,23 @@ def spreadBits{T==[32]u8, a:(u32)} = {
}
def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = {
b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}}
andnz{b, make{[16]u8, 1<<(iota{16}&7)}}
b:= sel{T, T~~[8]u16**a, make{[16]i8, iota{16}>=8}}
andnz{b, make{T, 1<<(iota{16}&7)}}
}
def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = {
b:= [16]u8~~[8]u16**a
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100}
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
b:= T~~[8]u16**a
exp:= T~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100}
(exp & make{T, 1<<(iota{16}&7)}) != T**0
}
def spreadBits{T, a if vcount{T} <= elwidth{T} and quality{eltype{T}}=='u'} = {
b:= make{T, 1<<iota{vcount{T}}}
b == (b & T ~~ re_el{type{a}, T}**a) # not just T**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
def spreadBits{V=[k]T, a:A if k<=width{T} and quality{T}=='u'} = {
b:= make{V, 1<<iota{k}}
b == (b & V ~~ re_el{A, V}**a) # not just V**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
}
def loadBatchBit{T, x:(*u64), n:(ux)} = { # vector with type T with each element being either all 0s or 1s
spreadBits{T, b_getBatchLo{vcount{T}, x, n}}
# vector with type V with each element being either all 0s or 1s
def loadBatchBit{V=[k]T, x:(*u64), n:(ux)} = {
spreadBits{V, b_getBatchLo{k, x, n}}
}
# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it
@ -87,15 +93,15 @@ def loaduBitRaw{x:(*u64), i} = {
loadu{*u64~~((*u8~~x) + (i>>3))} >> (i&7)
}
def loaduBit{x:(*u64), i, n} = {
assert{(n<58) | (((n==58) | (n==60)) & (i%n == 0))}
assert{(n<58) | ((n==58 or n==60) & (i%n == 0))}
loaduBitRaw{x, i}
}
def loaduBitTrunc{x:(*u64), i, n if knum{n}} = truncBits{n, loaduBit{x, i, n}}
def loadBatchBit{T, x:(*u64), is if ktup{is}} = {
def loadBatchBit{T, x:(*u64), {...is}} = {
# def len = length{is}
# def count = vcount{T}
# def [count]_ = T
# assert{count*len <= 64}
# bits:= b_getBatchLo{count*len, x, select{is,0}}
# @collect(i to len) spreadBits{T, truncBits{count, bits>>(i*count)}}

View File

@ -1,4 +1,4 @@
def pdep{x:T, m:T if T==u64} = emit{T, '_pdep_u64', x, m}
def pdep{x:T, m:T if T==u32} = emit{T, '_pdep_u32', x, m}
def pext{x:T, m:T if T==u64} = emit{T, '_pext_u64', x, m}
def pext{x:T, m:T if T==u32} = emit{T, '_pext_u32', x, m}
def pdep{x:T, m:T==u64} = emit{T, '_pdep_u64', x, m}
def pdep{x:T, m:T==u32} = emit{T, '_pdep_u32', x, m}
def pext{x:T, m:T==u64} = emit{T, '_pext_u64', x, m}
def pext{x:T, m:T==u32} = emit{T, '_pext_u32', x, m}

View File

@ -21,18 +21,14 @@ def cbqn_nspTag{} = emit{u64, '', 'NSP_TAG'}
def cbqn_objTag{} = emit{u64, '', 'OBJ_TAG'}
def cbqn_arrTag{} = emit{u64, '', 'ARR_TAG'}
def cbqn_elType{(u1 )} = 0
def cbqn_elType{(i8 )} = 1
def cbqn_elType{(i16)} = 2
def cbqn_elType{(i32)} = 3
def cbqn_elType{(f64)} = 4
def cbqn_elType{(u8 )} = 5
def cbqn_elType{(u16)} = 6
def cbqn_elType{(u32)} = 7
def cbqn_elType = match {
{(u1)} => 0; {(i8)} => 1; {(i16)} => 2; {(i32)} => 3; {(f64)} => 4
{(u8)} => 5; {(u16)} => 6; {(u32)} => 7
}
def cbqn_tyArrOffset{} = emit{u64, 'offsetof', 'TyArr', 'a'}
def talloc{T, len} = emit{*T, 'TALLOCP', fmt_type{T}, len}
def tfree{ptr} = emit{void, 'TFREE', ptr}
def fmt_type{T} = merge{quality{T}, fmtnat{width{T}}}
def fmt_type{T if isptr{T}} = merge{'*',fmt_type{eltype{T}}}
def fmt_type{*T} = merge{'*',fmt_type{T}}

View File

@ -32,7 +32,7 @@ def pathAS{dst, len, T, op, x if issigned{T}} = {
if (isNaN{xf}) { call{cmpIX, dst, len, x, op{0,1}}; return{}; }
}
}
if (~hasarch{'X86_64'}) nanchk{}
if (not hasarch{'X86_64'}) nanchk{}
if (rare{promote{i64, xT}!=xi64}) {
if (hasarch{'X86_64'}) nanchk{}
fillbits{dst, len, op{0,xf}}
@ -40,9 +40,9 @@ def pathAS{dst, len, T, op, x if issigned{T}} = {
xT
}
def pathAS{dst, len, T, op, x if T==f64} = {
def pathAS{dst, len, T==f64, op, x} = {
if (rare{~q_f64{x}}) {
if (~eqne{op}) if (~q_chr{x}) cmp_err{x}
if (not eqne{op} and ~q_chr{x}) cmp_err{x}
fillbits{dst, len, op{0,1}, x}
}
from_B{T,x}
@ -50,7 +50,7 @@ def pathAS{dst, len, T, op, x if T==f64} = {
def pathAS{dst, len, T, op, x if isunsigned{T}} = {
if (rare{~q_chr{x}}) {
if (~eqne{op}) if (~q_f64{x}) cmp_err{x}
if (not eqne{op} and ~q_f64{x}) cmp_err{x}
fillbits{dst, len, op{1,0}, x}
}
xc32:u32 = from_B{u32,x}
@ -60,11 +60,10 @@ def pathAS{dst, len, T, op, x if isunsigned{T}} = {
def any2bit{VT, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = {
def bulk = vcount{VT}*unr
def any2bit{VT=[k]T, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = {
def bulk = k*unr
xi:ux = 0
def T = eltype{VT}
def op = match (op0) {
{_ if not hasarch{'X86_64'} or hasarch{'AVX512F'}} => op0
{(__le) if issigned{T}} => __gt
@ -81,15 +80,15 @@ def any2bit{VT, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = {
xi+= unr
}
}
fn aa2bit{VT, unr, op}(dst:*u64, wr:*void, xr:*void, len:ux) : void = {
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
xv:= *VT~~xr; xs:= *eltype{VT}~~xr
fn aa2bit{VT=[_]T, unr, op}(dst:*u64, wr:*void, xr:*void, len:ux) : void = {
wv:= *VT~~wr; ws:= *T~~wr
xv:= *VT~~xr; xs:= *T~~xr
any2bit{VT, unr, op, load{ws,.}, load{wv,.}, load{xs,.}, load{xv,.}, dst, len}
}
fn as2bit{VT, unr, op}(dst:*u64, wr:*void, x:u64, len:ux) : void = {
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
xv:= VT**pathAS{dst, len, eltype{VT}, op, x}
fn as2bit{VT=[_]T, unr, op}(dst:*u64, wr:*void, x:u64, len:ux) : void = {
wv:= *VT~~wr; ws:= *T~~wr
xv:= VT**pathAS{dst, len, T, op, x}
any2bit{VT, unr, op, load{ws,.}, load{wv,.}, {i}=>x, {i}=>xv, dst, len}
}
@ -120,7 +119,7 @@ def table{aa, F, w, G_bit, G_vec} = {
tup{
G_bit{},
...each{{E}=>{
def E2 = (if (aa and isunsigned{E} and ((same{F,__eq} | same{F,__ne}) | (E==u32))) {ty_s{E}} else E)
def E2 = (if (aa and isunsigned{E} and (eqne{F} or E==u32)) {ty_s{E}} else E)
def bulk = w/width{E2}
def unr = if (hasarch{'X86_64'} and width{E2}==16) 2 else max{8/bulk,1}
G_vec{[bulk]E2, max{tern{hasarch{'AARCH64'},2,1}, unr}}

View File

@ -3,13 +3,12 @@ include './mask'
include './cbqnDefs'
include './bitops'
def copyFromBits{T, loadFn, rp, l:(u64)} = {
def bulk = vcount{T}
def TU = ty_u{T}
def copyFromBits{V=[bulk]T, loadFn, rp, l:(u64)} = {
def U = ty_u{V}
@maskedLoop{bulk}(sr in tup{'g',*eltype{T}~~rp} over i to l) {
x:= loadFn{TU, i}
sr{T~~(x & TU ~~ T**1)}
@maskedLoop{bulk}(sr in tup{'g',*T~~rp} over i to l) {
x:= loadFn{U, i}
sr{V~~(x & U ~~ V**1)}
}
}
@ -74,7 +73,7 @@ fn copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
def RV = [bulk]R
rp:= *R~~r
copyFromBits{RV, {T, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*vcount{T}, vcount{T}}}, r, l}
copyFromBits{RV, {T=[k]_, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*k, k}}, r, l}
# }
}

View File

@ -4,7 +4,7 @@ include './vecfold'
if_inline (hasarch{'SSE2'}) {
fn sum_vec{T}(v:T) = vfold{+, fold{+, unpackQ{v, T**0}}}
def fold_addw{v:T if eltype{T}==i8} = sum_vec{T}(v)
def fold_addw{v:T=[_](i8)} = sum_vec{T}(v)
}
def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}}

View File

@ -7,7 +7,7 @@ include 'util/tup'
def rootty{T if isprim{T}} = T
def rootty{T if isvec{T}} = eltype{T}
def rootty{[_]T} = T
def is_s{X} = issigned{rootty{X}}
def is_u{X} = isunsigned{rootty{X}}
@ -20,10 +20,10 @@ def bqn_or{a, b} = (a+b)-(a*b)
# + & -
def arithChk1{F==__add, M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}}
def arithChk1{F==__sub, M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}}
def arithChk1{F==__add, M, w:T, x:T, r:T if isvec{T} and tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', adds{w,x}, r}
def arithChk1{F==__sub, M, w:T, x:T, r:T if isvec{T} and tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', subs{w,x}, r}
def arithChk1{(__add), M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}}
def arithChk1{(__sub), M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}}
def arithChk1{(__add), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', adds{w,x}, r}
def arithChk1{(__sub), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', subs{w,x}, r}
@ -33,7 +33,7 @@ def arithChk2{F, M, w:T, x:T if is_s{T} and (same{F,__add} or same{F,__sub})} =
}
# ×
def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i8==eltype{T} and hasarch{'X86_64'}} = {
def arithChk2{F=(__mul), M, w:T=[_](i8), x:T if hasarch{'X86_64'}} = {
def wp = unpackQ{w, T ~~ (T**0 > w)}
def xp = unpackQ{x, T ~~ (T**0 > x)}
def rp = each{__mul, wp, xp}
@ -46,12 +46,12 @@ def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i8==eltype{T} and
tup{packQ{rp}, tup{'~andAllZero', RU~~tree_fold{|, bad}, RU**0xff80}}
}
}
def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i16==eltype{T} and hasarch{'X86_64'}} = {
def arithChk2{F=(__mul), M, w:T=[_](i16), x:T if hasarch{'X86_64'}} = {
rl:= __mul{w,x}
rh:= mulh{w,x}
tup{rl, tup{'anyne', rh, rl>>15}}
}
def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i32==eltype{T} and hasarch{'X86_64'}} = {
def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = {
max:= re_el{f32, (ty_u{T})**0x4efffffe}
def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x}
f32mul:= cf32{w} * cf32{x}
@ -68,11 +68,11 @@ def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i32==eltype{T} an
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'homAny', tree_fold{|,bad}}} this doesn't use M
}
def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and hasarch{'AARCH64'}} = {
def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = {
def r12 = mulw{w, x}
rl:= packLo{r12}
rh:= packHi{r12}
tup{rl, tup{'homAny', M{rh != (rl >> (elwidth{T}-1))}}}
tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}}
}
@ -88,7 +88,7 @@ def runner{u, R, F} = {
def run{F, M, w, x if u} = tup{F{w, x}, tup{'none'}} # trivial base implementation
def toggleTop{x:X} = x ^ X**(1<<(elwidth{X}-1))
def toggleTop{x:X=[_]E} = x ^ X**(1<<(width{E}-1))
def run{F==__sub, M, w:VU, x:VU if c and is_u{VU}} = { # 'b'-'a'
def VS = ty_s{VU}
run{F, M, VS~~toggleTop{w}, VS~~toggleTop{x}}
@ -99,7 +99,7 @@ def runner{u, R, F} = {
}
def run{F==__add, M, w:VS, x:VU if c and is_s{VS} and is_u{VU}} = run{F, M, x, w} # 3+'a' → 'a'+3
def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} | same{F,__sub})} = { # 'a'+1, 'a'-1
def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} or same{F,__sub})} = { # 'a'+1, 'a'-1
r:= F{ty_u{w}, ty_u{x}}
tup{re_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}}
}

View File

@ -9,8 +9,8 @@ def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v}
def to_prim = map{names, prims, .}
# Defined in C
def memset{p:pT, v, l} = {
emit{void, merge{'memset',fmtnat{elwidth{pT}}}, p, v, l}
def memset{p:*T, v, l} = {
emit{void, merge{'memset',fmtnat{width{T}}}, p, v, l}
}
# These hashes are stored in tables and must be invertible!

View File

@ -42,29 +42,26 @@ def maskAfter{n} = {
def loadLowBatch{T, ptr:P, w, n if eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/elwidth{P})), w}
def loadLowBatch{V=[_]T, ptr:*T, w, n} = loadLow{*V ~~ (ptr + n*(w/width{T})), w}
# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M
def storeBatch{ptr:P, n, x:T, M} = {
def rpos = ptr + n*vcount{T}
def E0 = eltype{P}
def TF = re_el{E0, T}
# store k items into the n'th batch of ptr elements, compressing the items if needed; masked by M
def storeBatch{ptr:*E0, n, x:V=[k]T, M} = {
def rpos = ptr + n*k
def TF = re_el{E0, V}
xu:= narrow{E0, x}
if (M{0}) homMaskStoreF{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}}
else storeLow{rpos, vcount{T}*width{E0}, xu}
else storeLow{rpos, k*width{E0}, xu}
}
# (sign/zero)-extend n'th batch of vcount{T} elements of P into elements of T
def loadBatch{ptr:P, n, T} = {
def rpos = ptr + n*vcount{T}
def E0 = eltype{P}
widen{T, loadLow{*re_el{E0, T} ~~ rpos, vcount{T}*width{E0}}}
# (sign/zero)-extend n'th batch of k elements of ptr into elements of V
def loadBatch{ptr:*E0, n, V=[k]T} = {
def rpos = ptr + n*k
widen{V, loadLow{*re_el{E0, V} ~~ rpos, k*width{E0}}}
}
def loadBatch {ptr:P, ns, T if istup{ns}} = each{{n } => loadBatch {ptr, n, T }, ns}
def storeBatch{ptr:P, ns, xs, M if istup{ns}} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs}
def loadBatch {ptr:P, {...ns}, T } = each{{n } => loadBatch {ptr, n, T }, ns}
def storeBatch{ptr:P, {...ns}, xs, M} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs}
@ -74,14 +71,14 @@ def hCast{T,p:*T} = p
def hCast{T,p:(*void)} = *T~~p
def mlExec{i, iter, vars0, bulk, M} = {
def vproc{p:P if isptr{P}} = p
def vproc{('m')} = tptr{{_}=>M, '!'}
def vproc{p:*T} = p
def vproc{'m'} = tptr{{_}=>M, '!'}
def vproc{{T,p:P}} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}}
def vproc{{('b'), p:P}} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'}
def vproc{{('b'),T,p:P}} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'}
def vproc{{('g'), p:P}} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'}
def vproc{{('g'),T,p:P}} = tptr{{i} => {
def vproc{{'b', p:P}} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'}
def vproc{{'b',T,p:P}} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'}
def vproc{{'g', p:P}} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'}
def vproc{{'g',T,p:P}} = tptr{{i} => {
def dv{} = loadBatch{p, i, T}
def dv{x} = storeBatch{p, i, x, M}
}, '!'}

View File

@ -10,7 +10,7 @@ def nty{T} = {
def q = quality{T}
merge{if (q=='i') 's' else q, fmtnat{width{T}}}
}
def nty{T if isvec{T}} = nty{eltype{T}}
def nty{[_]T} = nty{T}
def ntyp{S, ...S2, T if w128{T}} = merge{S, 'q', ...S2, '_', nty{T}}
def ntyp{S, ...S2, T if w64{T}} = merge{S, ...S2, '_', nty{T}}
def ntyp0{S, T} = merge{S, '_', nty{T}}
@ -28,12 +28,12 @@ def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (
def bitBlend{f:T, t:T, m:M if nvec{T} and nvecu{M,elwidth{T}} and width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
def homBlend{f:T, t:T, m:M if nvec{M}} = bitBlend{f, t, m}
def addpw { x:T if nveci{T} and elwidth{T}<=32 } = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
def addpwa{a:D, x:T if nveci{T} and elwidth{T}<=32 and D==el_m{T}} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
def addpw { x:T if nveci{T} and elwidth{T}<=32} = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
def addpwa{a:D==el_m{T}, x:T if nveci{T} and elwidth{T}<=32} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
def mla{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y
def mls{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y
def rbit{x:T if nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x}
def rev{w, x:T if w==elwidth{T}} = x
def rev{(width{T}), x:[_]T} = x
def rev{w==16, x:T if elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window
def rev{w==32, x:T if elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x}
def rev{w==64, x:T if elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x}
@ -41,33 +41,32 @@ def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x}
def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x}
def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x}
def fold_add {a:T if nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a}
def fold_addw{a:T if nveci{T}} = emit{w_d{eltype{T}}, ntyp{'vaddlv', T}, a}
def fold_min {a:T if nvec{T} and ~nveci{T,64}} = emit{eltype{T}, ntyp{'vminv', T}, a}
def fold_max {a:T if nvec{T} and ~nveci{T,64}} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
def fold_add {a:T=[_]E if nvec{T}} = emit{E, ntyp{'vaddv', T}, a}
def fold_addw{a:T=[_]E if nveci{T}} = emit{w_d{E}, ntyp{'vaddlv', T}, a}
def fold_min {a:T=[_]E if nvec{T} and ~nveci{T,64}} = emit{E, ntyp{'vminv', T}, a}
def fold_max {a:T=[_]E if nvec{T} and ~nveci{T,64}} = emit{E, ntyp{'vmaxv', T}, a}
def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x}
def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x}
def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x}
def storeLow{ptr:P, w, x:T if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} }
def storeLow{ptr:P, w, x:T if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x}
def storeLow{ptr:*P, w, x:T if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} }
def storeLow{ptr:*P, w, x:T if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x}
def loadLow{ptr:P, w if w<=64} = { # a broadcast load
def T=eltype{P}
def loadLow{ptr:*T, w if w<=64} = { # a broadcast load
def L=re_el{ty_u{w}, T}
T ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr}
}
def loadLow{ptr:P, w if w==elwidth{P}} = load{ptr}
def loadLow{ptr:*T, (width{T})} = load{ptr}
def undefPromote{T, x:X if w64{X} and w128{T} and eltype{T}==eltype{X}} = emit{T, ntyp{'vcombine', X}, x, x} # arm_neon.h doesn't actually provide a way to do this in a 0-instruction way. ¯\_(ツ)_/¯
def undefPromote{T=[_]E, x:X=[_]E if w64{X} and w128{T}} = emit{T, ntyp{'vcombine', X}, x, x} # arm_neon.h doesn't actually provide a way to do this in a 0-instruction way. ¯\_(ツ)_/¯
def half{x:T, (0) if w128{T}} = emit{n_h{T}, ntyp0{'vget_low', T}, x}
def half{x:T, (1) if w128{T}} = emit{n_h{T}, ntyp0{'vget_high', T}, x}
def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b}
def copyLane{dst:D, di, src:S, si if w64{D} and nvec{S} and eltype{D}==eltype{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
def copyLane{dst:D, di, src:S, si if w128{D} and nvec{S} and eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
def copyLane{dst:D=[_]T, di, src:S=[_]T, si if w64{D} and nvec{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
def copyLane{dst:D=[_]T, di, src:S=[_]T, si if w128{D} and nvec{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
@ -89,22 +88,22 @@ def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T
local def eqqi{A, B} = isint{A} & (quality{A}==quality{B}) # equal quality integers
def cvt{T==f64, x:X if nveci{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_f64', X}, x}
def cvt{T==i64, x:X if nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_s64', X}, x}
def cvt{T==u64, x:X if nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_u64', X}, x}
def cvt{T==f64, x:X=[k]_ if nveci{X,64}} = emit{[k]T, ntyp{'vcvt', '_f64', X}, x}
def cvt{T==i64, x:X=[k]_ if nvecf{X,64}} = emit{[k]T, ntyp{'vcvt', '_s64', X}, x}
def cvt{T==u64, x:X=[k]_ if nvecf{X,64}} = emit{[k]T, ntyp{'vcvt', '_u64', X}, x}
def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x}
def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}> elwidth{X}*2} = widen{T, widen{el_s{T}, x}}
def widen{T, x:X if w64{X} and isfloat{eltype{T}}!=isfloat{eltype{X}} and elwidth{T}>elwidth{X}} = cvt{eltype{T}, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}}
def widen{T, x:X if w128{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}}
def widen{T=[k]E, x:X if w64{X} and isfloat{E}!=isfloat{eltype{X}} and elwidth{T}>elwidth{X}} = cvt{E, widen{[k](to_w{eltype{X},elwidth{T}}), x}}
def widen{T=[k]E, x:X if w128{X} and vcount{X}>k} = widen{T, half{x,0}}
def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2< elwidth{X}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{eltype{X}}, x}}}
def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2==elwidth{X}} = emit{el_h{X}, ntyp0{'vmovn', X}, x}
def narrow{T, x:X if w128{X} and isfloat{T}!=isfloat{eltype{X}} and width{T}<elwidth{X}} = narrow{T, cvt{to_w{T, elwidth{X}}, x}}
def narrowUpper{lowRes:L, x:X if w64i{L} and w128{X} and el_d{L}==X} = emit{[vcount{L}*2](eltype{L}), ntyp0{'vmovn_high', X}, lowRes, x}
def narrowPair{a:T, b:T} = narrowUpper{narrow{w_h{eltype{T}}, a}, b}
def narrowPair{a:T, b:T if isint{eltype{T}}} = packLo{a, b}
def narrowUpper{lowRes:L=[k]T, x:X if w64i{L} and w128{X} and el_d{L}==X} = emit{[k*2]T, ntyp0{'vmovn_high', X}, lowRes, x}
def narrowPair{a:T=[_]E, b:T} = narrowUpper{narrow{w_h{E}, a}, b}
def narrowPair{a:T=[_]E, b:T if isint{E}} = packLo{a, b}
def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x}
def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}}
@ -118,36 +117,35 @@ def homAny{x:T if nvec{T}} = bitAny{x}
def homAll{x:T if nvec{T}} = bitAll{x}
def homMask{x:T if nvecu{T} and elwidth{T}>=vcount{T}} = {
truncBits{vcount{T}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
def homMask{x:T=[k]E if nvecu{T} and width{E}>=k} = {
truncBits{k, fold_add{x & make{T, 1<<iota{k}}}}
}
def homMask{x:T if T==[16]u8} = {
def homMask{x:T==[16]u8} = {
t:= [8]u16~~sel{[16]u8, x, make{[16]u8, 0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15}}
fold_add{t & make{[8]u16, (1<<iota{8})*0x0101}}
}
def homMask{a:T,b:T if T==[16]u8} = {
def homMask{a:T,b:T==[16]u8} = {
m:= make{[16]u8, 1<<(iota{16}&7)}
fold_add{addpw{addpw{addp{a&m, b&m}}}<<make{[4]u32,iota{4}*8}}
}
def homMask{a:T,b:T,c:T,d:T if T==[16]u8} = {
def homMask{a:T,b:T,c:T,d:T==[16]u8} = {
m:= make{[16]u8, 1<<(iota{16}&7)}
t1:= addp{a&m, b&m}
t2:= addp{c&m, d&m}
t3:= addp{t1, t2}
extract{[2]u64~~addp{t3,t3},0}
}
def homMask{...as if length{as}>1 and elwidth{type{select{as,0}}}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}}
def homMask{a:T,b:T if vcount{T}*2<=elwidth{T}} = {
def n = vcount{T}
truncBits{n*2, fold_add{shrm{a,elwidth{T}-n,b} & make{T, (1<<iota{n}) | (1<<(iota{n}+n))}}}
def homMask{...as={a0:[_]E, _, ..._} if width{E}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}}
def homMask{a:T,b:T=[n]E if n*2<=width{E}} = {
truncBits{n*2, fold_add{shrm{a,width{E}-n,b} & make{T, (1<<iota{n}) | (1<<(iota{n}+n))}}}
}
def andAllZero{x:T, y:T if nveci{T}} = ~bitAny{x&y}
def homMaskX{a:T if eltype{T}!=u64} = {
def h = elwidth{T}/2
tup{h, truncBits{vcount{T}*h, extract{[1]u64~~shrn{el_m{T}~~a, h}, 0}}}
def homMaskX{a:T=[k]E if E!=u64} = {
def h = width{E}/2
tup{h, truncBits{k*h, extract{[1]u64~~shrn{el_m{T}~~a, h}, 0}}}
}
def homMaskStoreF{p:P, m:M, v:T if nveci{M} and nvec{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def homMaskStoreF{p:*T, m:M, v:T if nveci{M} and nvec{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}

View File

@ -6,25 +6,25 @@ def ind_types = tup{i8, i16, i32}
def dat_types = tup{...ind_types, u64}
# Indices and Replicate using plus- or max-scan
def scan_core{upd, set, scan, rp:pT, wp:W, s:(usz)} = {
def scan_core{upd, set, scan, rp:*T, wp:W, s:(usz)} = {
def getw{j} = if (isptr{W}) cast_i{usz,load{wp,j}} else wp
b:usz = 1<<10
k:usz = 0; j:usz = 0; ij:=getw{j}
while (1) {
e := tern{b<s-k, k+b, s}
@for (rp over i from k to e) rp = 0
if (set) store{rp, k, cast_i{eltype{pT},j}}
if (set) store{rp, k, cast_i{T,j}}
while (ij<e) { ++j; upd{rp, j, ij}; ij+=getw{j} }
scan{rp+k, e-k}
if (e==s) return{}
k = e
}
}
def indrep_by_sum{T, rp:(*T), wp, s:(usz), js, inc} = {
def indrep_by_sum{T, rp:*T, wp, s:(usz), js, inc} = {
def scan{ptr, len} = @for (ptr over len) js=ptr+=js
def scan{ptr, len if width{T}<=32} = {
def scan{ptr:*E, len if width{T}<=32} = {
def scanfn = merge{'si_scan_pluswrap_u',fmtnat{width{T}}}
p := *ty_u{eltype{type{ptr}}}~~ptr
p := *ty_u{E}~~ptr
emit{void, scanfn, p, p, len, js}; js=load{ptr,len-1}
}
def upd{rp, j, ij} = store{rp, ij, load{rp,ij}+inc{j}}
@ -33,7 +33,7 @@ def indrep_by_sum{T, rp:(*T), wp, s:(usz), js, inc} = {
fn ind_by_scan_i32{W}(xv:*void, rp:*i32, s:usz) : void = {
xp := *W~~xv
if (hasarch{'X86_64'} & ~hasarch{'SSE4.1'}) { # no min instruction
if (hasarch{'X86_64'} and not hasarch{'SSE4.1'}) { # no min instruction
js:i32 = 0
indrep_by_sum{i32, rp, xp, s, js, {j}=>1}
} else {
@ -59,7 +59,7 @@ exportT{'si_replicate_scan', flat_table{rep_by_scan, ind_types, dat_types}}
# Constant replicate
if_inline (not (hasarch{'AVX2'} | hasarch{'AARCH64'})) {
if_inline (not (hasarch{'AVX2'} or hasarch{'AARCH64'})) {
fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
rep_by_scan{T, cast_i{usz,wv}, x, r, cast_i{usz, wv*n}}
@ -101,8 +101,7 @@ def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}}
rcsh4_dat:*i8 = join{join{each{get_shuf_data{., 4}, rcsh4_dom}}}
rcsh4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}}
def read_shuf_vecs{l, ellw:(u64), shp:P} = { # tuple of byte selectors in 1<<ellw
def V = eltype{P}
def read_shuf_vecs{l, ellw:(u64), shp:*V} = { # tuple of byte selectors in 1<<ellw
def double{x:X if hasarch{'AVX2'}} = {
s:=shuf{[4]u64, x, 4b3120}; s+=s
r:=each{bind{~~,[32]i8},unpackQ{s, s + X**1}}
@ -126,8 +125,7 @@ def read_shuf_vecs{l, ellw:(u64), shp:P} = { # tuple of byte selectors in 1<<ell
sh
}
def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:(u64)} = { # onreps{inputVector, {nextOutputVector} => ...}
def step = vcount{V}
def rep_const_shuffle{wv, onreps, xv:*V=[step]T, rv:*V, n:(u64)} = { # onreps{inputVector, {nextOutputVector} => ...}
nv := n / step
j:u64 = 0
def write{v} = { store{rv, j, v}; ++j }
@ -168,7 +166,7 @@ if_inline (hasarch{'AVX2'}) {
{x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh}
}
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:(u64)} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
def rep_const_shuffle{wv, xv:*V, rv:*V, n:(u64)} = rep_const_shuffle{wv, get_rep_iter{V, wv}, xv, rv, n}
} else if_inline (hasarch{'AARCH64'}) {
@ -176,8 +174,8 @@ if_inline (hasarch{'AVX2'}) {
each{{s} => gen{sel{[16]u8, x, s}}, sh}
}
def rep_const_shuffle{V, wv==2, xv0:*V, rv0:*V, n:(u64)} = {
def E = ty_u{eltype{V}}
def rep_const_shuffle{wv==2, xv0:*V=[_]T, rv0:*V, n:(u64)} = {
def E = ty_u{T}
rv:= *E~~rv0
@for (x in *E~~xv0 over i to n) { # autovectorized well enough, probably
store{rv, i*2, x}
@ -190,7 +188,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
def h = 4
def V = sdtype
def sh = read_shuf_vecs{h, ellw, *V~~rcsh4_dat + h*load{rcsh4_lkup,wv}}
def step = vcount{V} # Bytes written
def [step]_ = V # Bytes written
def wvb = wv << ellw
def hs = (h*step) / wvb # Actual step size in argument elements
def shufbase{i if hasarch{'AVX2'}} = shuf{[4]u64, load{*V~~(x+i)}, 4b1010}
@ -226,7 +224,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
fn rcsh_sub{wv, V}(ellw:u64, x:*i8, r:*i8, n:u64, sh:*V) : void = {
def st = read_shuf_vecs{wv, ellw, sh}
rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n}
rep_const_shuffle{wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n}
}
fn rep_const_shuffle_any(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
@ -271,7 +269,7 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
def max_shuffle = 2*vn
if (wv <= max_shuffle) {
def specialize{k} = {
if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}}
if (wv==k) return{rep_const_shuffle{k, *V~~x, *V~~r, n}}
}
if (special_2) specialize{2}
rep_const_shuffle_any(wv, lb{wT/8}, *i8~~x, *i8~~r, n)

View File

@ -12,7 +12,7 @@ fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = {
@for (x, r over len) r = m = op{m, x}
}
def scan_loop{T, init, x:*T, r:*T, len:(u64), scan, scan_last} = {
def scan_loop{init, x:*T, r:*T, len:(u64), scan, scan_last} = {
def step = arch_defvw/width{T}
def V = [step]T
p:= V**init
@ -23,20 +23,20 @@ def scan_loop{T, init, x:*T, r:*T, len:(u64), scan, scan_last} = {
q:= len & (step-1)
if (q!=0) homMaskStoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
}
def scan_post{T, init, x:*T, r:*T, len:(u64), op, pre} = {
def scan_post{init, x:*T, r:*T, len:(u64), op, pre} = {
def last{v, p} = op{pre{v}, p}
def scan{v, p} = {
n:= last{v, p}
p = toLast{n}
n
}
scan_loop{T, init, x, r, len, scan, last}
scan_loop{init, x, r, len, scan, last}
}
# Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈
def scan_idem = scan_scal
fn scan_idem{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = {
scan_post{T, init, x, r, len, op, make_scan_idem{T, op}}
scan_post{init, x, r, len, op, make_scan_idem{T, op}}
}
export{'si_scan_min_init_i8', scan_idem{i8 , min}}; export{'si_scan_max_init_i8', scan_idem{i8 , max}}
@ -67,7 +67,7 @@ def scan_plus = scan_assoc{+}
def scan_assoc_0 = scan_scal
fn scan_assoc_0{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = {
# Prefix op on entire AVX register
scan_post{T, init, x, r, len, op, scan_plus}
scan_post{init, x, r, len, op, scan_plus}
}
export{'si_scan_pluswrap_u8', scan_assoc_0{u8 , +}}
export{'si_scan_pluswrap_u16', scan_assoc_0{u16, +}}
@ -202,11 +202,10 @@ def addChk{a:T, b:T} = {
def bad = emit{u1, '__builtin_add_overflow', a, b, mem}
tup{bad, load{mem}}
}
def addChk{a:T, b:T if T==f64} = tup{0, a+b}
def addChk{a:T, b:T==f64} = tup{0, a+b}
def widenFull{E, xs} = {
merge{...each{{x:X} => {
def n = vcount{X}
merge{...each{{x:X=[n]T} => {
def tb = width{E} * n
if (tb<=arch_defvw) tup{widen{[n]E, x}}
else if (1) {
@ -226,7 +225,7 @@ def maxsafeint{T==f64} = 1<<53
fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = {
i:u64 = 0
if (hasarch{'AVX2'}) simd_plus_scan_part{X,R}{x, c, r, len, i}
if (hasarch{'AVX2'}) simd_plus_scan_part{x, c, r, len, i}
@forUnroll{1,1} (js from i to len) {
def vs = eachx{load, x, js}
each{{j, v} => {
@ -239,7 +238,7 @@ fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = {
len
}
# Sum as many vector registers as possible; modifies c and i
def simd_plus_scan_part{X, R}{x:(*X), c:(R), r:(*R), len:(u64), i:(u64)} = {
def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
def b = max{width{R}/2, width{X}}
def bulk = arch_defvw/b

View File

@ -12,8 +12,8 @@ def sel8{v, t, up} = sel8{v, rev{up,t}}
def zip{up, x} = (if (up) zipHi else zipLo){x,x}
# Fill last 4 bytes with last element, in each lane
def spread{a:VT, ...up} = {
def w = elwidth{VT}
def spread{a:[_]T, ...up} = {
def w = width{T}
def b = w/8
if (w<=16) sel8{a,merge{iota{12},(16-b)+iota{4}%b}, ...up} else a
}
@ -39,7 +39,7 @@ def prefix_byshift{op, sh} = {
{v:T} => pre{v, if (isvec{T}) elwidth{T} else 1}
}
def get_id{op,T} = (match (op) { {_==min}=>maxvalue; {_==max}=>minvalue }){T}
def get_id{op,T} = (match (op) { {(min)}=>maxvalue; {(max)}=>minvalue }){T}
def make_scan_idem{T, op, up} = {
# Within each lane, scan using shifts by powers of 2. First k elements
@ -59,8 +59,8 @@ def make_scan_idem{T, op, up} = {
}
prefix_byshift{op, shb}
}
def make_scan_idem{T==f64, op, up} = {
def sc{a:T if vcount{T}==2} = op{a, zip{~up,a}}
def make_scan_idem{(f64), op, up} = {
def sc{a:[2]T} = op{a, zip{~up,a}}
def sc{a:T if hasarch{'AVX2'} and w256{T}} = {
def sh{s, a} = op{a, shuf{[4]u64, a, rev{up,s}}}
sh{tup{0,1,1,1},sh{tup{0,0,2,2},a}}

View File

@ -85,7 +85,7 @@ def bittab_init{tab, z if simd_bittab} = {
}
def bittab_selector{loadtab} = {
def nv = vcount{VI}
def [nv]_ = VI
{t0, t1}:= loadtab{}
low:= VI**7
hi4:= VI**(-(1<<4))
@ -101,7 +101,7 @@ def bittab_selector{loadtab} = {
}
def readbytes{vtab}{} = {
def k = vcount{VI}; def l = 128/vcount{VI}
def [k]_ = VI; def l = 128/k
def side{i} = {
def U = arch_vec{ty_u{k}}
def m = @collect (vtab over _ from i to i+l) homMask{vtab}
@ -126,7 +126,7 @@ def bittab_lookup{x0:(*void), n:(u64), r0:(*void), tab:(*void)} = {
}
def bittab_lookup{x0:(*void), n:(u64), r0:(*void), tab:(*void) if simd_bittab} = {
def {bitsel, _} = bittab_selector{readbytes{*VI~~tab}}
def k = vcount{VI}
def [k]_ = VI
@for (x in *VI~~x0, r in *ty_u{k}~~r0 over cdiv{n,k}) r = bitsel{x}
}
@ -175,7 +175,7 @@ def do_bittab{x0:(*void), n:(u64), tab:(*void), u:(u8), t, mode, r0} = {
# Do first few values with a scalar loop
# Avoids the cost of ever loading the table into vectors for n<=48
i:u64 = 32; if (n<=48) i=n
def k = vcount{VI}; def uk = ty_u{k}; def ik = ty_s{k}
def [k]_ = VI; def uk = ty_u{k}; def ik = ty_s{k}
{rw,rv} := undef{tup{u64,uk}} # Bit results, used if rbit
if (rbit) rw = 0
@for (x over j to i) {
@ -295,8 +295,8 @@ def acc{unr, init:T} = {
else a1 = F{a1}
}
}
def isI64{x:T if eltype{T}==f64 and hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}}
def isI64{x:T if eltype{T}==f64 and hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53))
def isI64{x:T=[_](f64) if hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}}
def isI64{x:T=[_](f64) if hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53))
def maskBlend{b:T, x:T, M} = x
def maskBlend{b:T, x:T, M if M{0}} = homBlend{b, x, M{T, 'to homogeneous bits'}}
@ -306,10 +306,10 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
x:= *E~~x0
min1:E = *x
max1:E = *x
if (has_simd and not (E==f64 and not (hasarch{'AARCH64'} or hasarch{'SSE4.1'}))) {
if (has_simd and (not E==f64 or hasarch{'AARCH64'} or hasarch{'SSE4.1'})) {
def bulk = arch_defvw/width{E}
def VT = [bulk]E
def unr = tern{(E==f64) & hasarch{'X86_64'}, 1, 2}
def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2}
def minA = acc{2, VT**min1}
def maxA = acc{2, VT**min1}
@muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
@ -599,8 +599,8 @@ fn hashtab{T, name}(rpi:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:it
}
def try_vec_memb{..._} = {}
def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done
if hasarch{'SSE4.2'} and T==u32} = {
def try_vec_memb{T==u32, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done
if hasarch{'SSE4.2'}} = {
# Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints)
# i-h>>sh = i+((1<<sh-1)-h)>>sh = (((i+1)<<sh-1)-h)>>sh
# We maintain io = (i+1)<<sh-1

View File

@ -40,9 +40,9 @@ def arg{c,T} = if (c) *T else if (T==i32) T else tup{} # type of x
# Modifies the input variable r
# Assumes iter{} will increment r, by at most write_len
def for_special_buffered{r, write_len}{vars,begin,sum,iter} = {
def for_special_buffered{r:*T, write_len}{vars,begin,sum,iter} = {
assert{isreg{r}}; assert{begin==0}
def T = eltype{type{r}}; def tw = width{T}
def tw = width{T}
def ov = write_len-1
def bufn = 2*(ov+1)
buf := undefined{T, bufn}
@ -86,14 +86,14 @@ fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
}
}
def getter{c, V, x} = {
def getter{c, V=[k]T, x} = {
if (c) {
i:u64 = 0
{} => { v:=load{*V~~x, i}; ++i; v }
} else {
i := iota{V}
if (isreg{x}) i += V**cast_i{eltype{V},x}
ii := V**vcount{V}
if (isreg{x}) i += V**cast_i{T,x}
ii := V**k
{} => { v:=i; i+=ii; v }
}
}

View File

@ -7,24 +7,24 @@ include './vecfold'
def preserve_negative_zero = 0
# SSE2 versions avoid any 64-bit integer comparsions
def anySNaN{M, x:T if eltype{T}==u64} = {
def anySNaN{M, x:[_](u64)} = {
homAny{inRangeLen{M{x}<<1, (0xFFE<<52)+2, (1<<52)-2}}
}
def anySNaN{M, x:T if T==[2]u64 and hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = {
def anySNaN{M, x:T==[2]u64 if hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = {
topAny{M{andnot{unord{[2]f64~~x, [2]f64~~x}, [2]u64~~([4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000))}}}
}
def anyNonChar{M, x:T if isvec{T} and eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
def anyNonChar{M, x:T if isvec{T} and hasarch{'X86_64'}} = {
def anyNonChar{M, x:[_](u64)} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
def anyNonChar{M, x:T=[_]_ if hasarch{'X86_64'}} = {
def H = re_el{u32, T}
def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16}
topAny{M{T~~ne}}
}
def cvtNarrow{T, x:X if width{T}==elwidth{X}} = cvt{T, x}
def cvtNarrow{T, x:X if width{T}< elwidth{X}} = narrow{T, x}
def cvtWiden{T, x:X if elwidth{T}==elwidth{X}} = cvt{eltype{T}, x}
def cvtWiden{T, x:X if elwidth{T}> elwidth{X}} = widen{T, x}
def cvtNarrow{T, x:[_]X if width{T}==width{X}} = cvt{T, x}
def cvtNarrow{T, x:[_]X if width{T}< width{X}} = narrow{T, x}
def cvtWiden{ [_]T, x:[_]X if width{T}==width{X}} = cvt{T, x}
def cvtWiden{V=[_]T, x:[_]X if width{T}> width{X}} = widen{V, x}
fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
assert{len>0}
@ -35,14 +35,11 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E}
# fold with either Max or Bitwise Or, truncating/zero-extending to TE
def foldTotal{TE, x:T} = cast_i{TE, vfold{|, x}}
def foldTotal{TE, x:T if hasarch{'AARCH64'}} = {
if (elwidth{T}==64) {
if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
else vfold{max, narrow{TE, x}}
} else {
vfold{max, x}
}
def foldTotal{TE, x:[_]T} = cast_i{TE, vfold{|, x}}
def foldTotal{TE, x:[_]T if hasarch{'AARCH64'}} = {
if (width{T}!=64) vfold{max, x}
else if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
else vfold{max, narrow{TE, x}}
}
# show{XV, EV, CHR, B}
@ -79,8 +76,8 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
return{0xffff_ffff} # float
}}
def getAcc{EV, x:T} = {
((EV ** ~(eltype{EV})~~1) & EV~~x) ^ EV~~(x >> (elwidth{T}-1))
def getAcc{EV=[_]E, x:[_]T} = {
((EV ** ~E~~1) & EV~~x) ^ EV~~(x >> (width{T}-1))
}
if (isint{X}) { # i16, i32
@ -89,7 +86,10 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
}
} else { # f64
r2:= EV2**0
@muLoop{bulk, hasarch{'AARCH64'}+1, {} => { r1 = half{r2,0}|half{r2,1} }}(v0 in tup{XV,xp}, M in 'm' over is to len) {
@muLoop{
bulk, hasarch{'AARCH64'}+1,
{} => { r1 = half{r2,0}|half{r2,1} }
}(v0 in tup{XV,xp}, M in 'm' over is to len) {
def int = {
def {int, wdn} = {
if (hasarch{'AARCH64'} and length{is}==2) {
@ -99,15 +99,12 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
} else {
def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0}
def wdn = each{{v} => cvtWiden{XV, v}, ints}
if (vcount{type{select{ints,0}}} == bulk) {
def intp = {
if (length{ints}==1) select{ints, 0}
else pair{ints}
}
tup{intp, wdn}
} else if (length{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX2'}) {
tup{select{ints, 0}, wdn}
} else assert{0}
def intp = match (...ints) {
{i:[(bulk)]_} => i
{i if hasarch{'X86_64'} and not hasarch{'AVX2'}} => i
{i:T=[(bulk)]_, j:T} => pair{ints}
}
tup{intp, wdn}
}
}
@ -124,7 +121,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
}
int
}
def acc = { if (length{is}==2) r2 else r1 }
def acc = if (length{is}==2) r2 else r1
acc|= M{getAcc{type{acc}, int}}
}

View File

@ -1,6 +1,6 @@
### SSSE3 ###
def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
def vshl{a:T, b:T, n if hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)}
def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = V~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(width{T}/8)}
### SSE4.1 ###
@ -8,16 +8,16 @@ def packs{a:T,b:T if hasarch{'SSE4.1'} and T==[4]u32} = emit{[ 8]u16, '_mm_packu
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
# conversion
def widen{T==[8]u16, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X if hasarch{'SSE4.1'} and X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X if hasarch{'SSE4.1'} and w128i{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T==[8]u16, x:X==[16]u8 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X==[16]u8 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X==[8]u16 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X==[16]u8 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X==[8]u16 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X==[4]u32 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X=[_]E if hasarch{'SSE4.1'} and w128i{X} and width{E}<32} = widen{T, widen{[4]i32, x}}
def narrow{T, x:X if hasarch{'SSE4.1'} and w128i{X,32} and T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X if hasarch{'SSE4.1'} and w128i{X,32} and T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
def narrow{(i8 ), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
# mask stuff
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}

View File

@ -1,25 +1,25 @@
# compact casting for the annoying intrinsic type system
def v2i{x:T if w128{T}} = if(isint{eltype{T}}) x else [16]u8 ~~ x
def v2i{x:V=[_]T if w128{V}} = if(isint{T}) x else [16]u8 ~~ x
def v2f{x:T if w128{T}} = [4]f32 ~~ x
def v2d{x:T if w128{T}} = [2]f64 ~~ x
# load & store
def loadLow{ptr:P, w if w128{eltype{P}} and w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def loadLow{ptr:P, w if w128{eltype{P}} and w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def loadLow{ptr:P, w if w128{eltype{P}} and w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
def loadLow{ptr:P, w if w128{eltype{P}} and w==128} = load{ptr}
def loadLow{ptr:*T, w if w128{T} and w== 16} = T ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def loadLow{ptr:*T, w if w128{T} and w== 32} = T ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def loadLow{ptr:*T, w if w128{T} and w== 64} = T ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
def loadLow{ptr:*T, w if w128{T} and w==128} = load{ptr}
def storeLow{ptr:P, w, x:T if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T if w128{T} and w==128} = store{*T~~ptr, 0, x}
def storeLow{ptr:*P, w, x:T if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
def storeLow{ptr:*P, w, x:T if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
def storeLow{ptr:*P, w, x:T if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
def storeLow{ptr:*P, w, x:T if w128{T} and w==128} = store{*T~~ptr, 0, x}
# float comparison
def unord{a:T,b:T if T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b}
def unord{a:T,b:T if T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
def unord{a:T,b:T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b}
def unord{a:T,b:T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
@ -51,21 +51,21 @@ def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
def topMask{x:T if w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def topMask{x:T if w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def homMask{x:T if w128{T}} = topMask{x}
def homMaskX{a:T if elwidth{T}==16} = tup{2, homMask{re_el{u8,a}}}
def homMaskX{a:[_]T if width{T}==16} = tup{2, homMask{re_el{u8,a}}}
def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}}
def homAny{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0
def homAll{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
def topAny{x:T if w128i{T}} = topMask{x} != 0
def topAll{x:T if w128i{T}} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T if w128i{T}} = topMask{x} != 0
def topAll{x:T=[k]_ if w128i{T}} = topMask{x} == (1<<k)-1
def topAny{x:T if w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
def topAll{x:T if w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}
# bits of other things SSE2 has
def packs{a:T,b:T if T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T if T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T if T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packs{a:T,b:T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packQ{a:T,b:T if w128i{T}} = packs{a,b}
@ -83,16 +83,16 @@ def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}
def homBlend{f:T, t:T, m:M if w128{T} and w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f &~ m))
def homMaskStoreF{p:P, m:M, v:T if w128i{M} and w128{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}
def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = {
def s{v} = s{mzipLo{v, v}}
def s{v:V if V==T} = v
def s{v:(T)} = v
s{x} >> (elwidth{T} - elwidth{X})
}
def widen{T==[2]f64, x:X if w128s{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T==[2]f64, x:X if X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
def widen{T==[2]f64, x:X if X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
def widen{T==[2]f64, x:X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
def widen{T==[2]f64, x:X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
def narrow{T==i16, x:([4]i32)} = packs{x,x}
def narrow{T==i8, x:([8]i16)} = packs{x,x}

View File

@ -27,9 +27,9 @@ def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x}
# Square kernel where width is a full vector
def transpose_square{VT, l, x if hasarch{'AVX2'}} = unpack_to{1, l/2, x}
def load2{a:T, b:T} = pair{load{a}, load{b}}
def store2{a:T, b:T, v:T2 if w128i{eltype{T}} and w256{T2}} = {
each{{p:P, i} => store{p, 0, eltype{P}~~half{v,i}}, tup{a,b}, iota{2}}
def load2{a:*T, b:*T} = pair{load{a}, load{b}}
def store2{a:*T, b:*T, v:T2 if w128i{T} and w256{T2}} = {
each{{p, i} => store{p, 0, T~~half{v,i}}, tup{a,b}, iota{2}}
}
def load_k {VT, src, l, w if w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}
def store_k{VT, dst, x, l, h if w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x}
@ -37,8 +37,7 @@ def load_k {VT, src, l, w if w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*V
def store_k{VT, dst, x, l, h if w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x}
# Transpose kernel of size kw,kh in size w,h array
def kernel{src:P, dst:P, kw, kh, w, h} = {
def T = eltype{P}
def kernel{src:*T, dst:*T, kw, kh, w, h} = {
def n = (kw*kh*width{T}) / 256 # Number of vectors
def xvs = load_k{[kw]T, src, n, w}
def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n
@ -105,7 +104,7 @@ def transpose_with_kernel{T, k, kh, call_base, rp:*T, xp:*T, w, h, ws, hs} = {
assert{k == kh}
def VT = [k]T
def line_vecs = line_bytes / (width{VT}/8)
def store_line{p:P, vs} = each{{i,v} => store{p, i, eltype{P}~~v}, iota{line_vecs}, vs}
def store_line{p:*T, vs} = each{{i,v} => store{p, i, T~~v}, iota{line_vecs}, vs}
def get_lines{loadx} = {
def vt{i} = transpose_square{VT, k, each{loadx, k*i + iota{k}}}
each{tup, ...each{vt, iota{line_vecs}}}

View File

@ -1,12 +1,12 @@
# Fold associative/commutative operation across a register
def vfold{F, x:T if w128{T} and hasarch{'X86_64'}} = {
def vfold{F, x:V=[_]T if w128{V} and hasarch{'X86_64'}} = {
c:= x
def EW = elwidth{T}
def EW = width{T}
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}}
if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{eltype{T}, v}, cast_i{eltype{T}, v>>8}} }
if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{T, v}, cast_i{T, v>>8}} }
else extract{c, 0}
}
def vfold{F, x:T if w256{T} and hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}