diff --git a/src/singeli/README.md b/src/singeli/README.md index b6658edd..e6b66cee 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -146,12 +146,12 @@ Some may also support one scalar argument or arguments with different widths. - `vshl{a:V, b:V, n} : V` - `vshl{[0,1,2,3], [4,5,6,7], 1} → [1,2,3,4]` - `sel{VI1, a:V, b:VI2} : V2` - shuffle `a` by indices `b` in `VI1`-long lanes; arch-specific behavior on out-of-bounds values -- `zip{a:T,b:T} : tup{T, T}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}` -- `mzip{a:T,b:T} : tup{el_dbl{T}, el_dbl{T}}` - reinterpreted `zip{a, b}` -- `zipLo{a:T,b:T} : T` - `select{zip{a, b}, 0}` -- `zipHi{a:T,b:T} : T` - `select{zip{a, b}, 1}` -- `mzipLo{a:T,b:T} : T` - `select{mzip{a, b}, 0}` -- `mzipHi{a:T,b:T} : T` - `select{mzip{a, b}, 1}` +- `zip{a:V,b:V} : tup{V, V}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}` +- `mzip{a:V,b:V} : tup{el_dbl{V}, el_dbl{V}}` - reinterpreted `zip{a, b}` +- `zipLo{a:V,b:V} : V` - `select{zip{a, b}, 0}` +- `zipHi{a:V,b:V} : V` - `select{zip{a, b}, 1}` +- `mzipLo{a:V,b:V} : V` - `select{mzip{a, b}, 0}` +- `mzipHi{a:V,b:V} : V` - `select{mzip{a, b}, 1}` ## Mask stuff @@ -176,11 +176,11 @@ Alignment requirements are target-specific, but at most one element. For unaligned scalar loads & stores, `loadu` & `storeu` should be used. - `loadu{p:*E} : E` - load scalar from unaligned memory -- `storeu{p:*E, a:V} : void` - store scalar to unaligned memory +- `storeu{p:*E, a:E} : void` - store scalar to unaligned memory - `load{p:*V} : V` - load full vector - `store{p:*V, a:V} : void` - store full vector - `loadLow{p:*V, w} : V` - load to low `w` bits -- `storeLow{p:*V, w, a:V}` - store low `w` bits +- `storeLow{p:*E, w, a:[n]E}` - store low `w` bits - `homMaskStore{p:*V, m:mt{V}, a:V}` - conditionally store elements based on mask; won't touch masked-off elements - `topMaskStore{p:*V, m:V, a:V}` - conditionally store elements based on top bit of `m`; won't touch masked-off elements - `homMaskStoreF` - `homMaskStore` but may touch masked-off elements and thus be supported on more types @@ -207,7 +207,7 @@ For float conversions, the used rounding mode is unspecified. - `mul32{a:VI, b:VI} : VI` - multiply, reading only low 32 bits - `blend{L, a:V, b:V, m}` - blend `L`-sized blocks via the immediate -- `shuf{L, x:T, n} : T` - shuffle by immediate in `L`-sized lanes +- `shuf{L, x:V, n} : V` - shuffle by immediate in `L`-sized lanes - `packQ` - pack 128-bit lanes (`packs`/`packus`) for 16-bit & 32-bit elements - `packQQ` - `packQ` but also defined for 64-bit elements, assuming the high halves are zeroes - `packs` - 128-bit `packs`/`packus` diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 18c136d4..e501884e 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -6,11 +6,11 @@ def v2d{x:T if w256{T}} = [4]f64 ~~ x def undefPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} # load & store -def loadLow{ptr:*T, w if w256{T} and w<=128} = undefPromote{T, loadLow{*n_h{T} ~~ ptr, w}} -def loadLow{ptr:*T, w if w256{T} and w==256} = load{ptr} +def loadLow{ptr:*V, w if w256{V} and w<=128} = undefPromote{V, loadLow{*n_h{V} ~~ ptr, w}} +def loadLow{ptr:*V, w if w256{V} and w==256} = load{ptr} -def storeLow{ptr:*T, w, x:X if w256{X} and w<=128} = storeLow{ptr, w, half{x, 0}} -def storeLow{ptr:*T, w, x:X if w256{X} and w==256} = store{*X~~ptr, 0, x} +def storeLow{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = storeLow{ptr, w, half{x, 0}} +def storeLow{ptr:*E, w, x:X=[_]E if w256{X} and w==256} = store{*X~~ptr, 0, x} # float comparison diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index d258031f..d983fefc 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -44,8 +44,8 @@ def topMaskStore{p:*T, m:M, v:T if w256i{M, 32} and w256{T,elwidth{M}}} = emit{v def topMaskStore{p:*T, m:M, v:T if w256i{M, 64} and w256{T,elwidth{M}}} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v} def homMaskStore{p:*T, m:M, v:T if w256i{M} and w256{T,elwidth{M}}} = topMaskStore{p, m, v} -def topMaskStoreF{p:*P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} -def homMaskStoreF{p:*P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} +def topMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} +def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}} # mask stuff diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 9c8c8e42..69727d8f 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -114,11 +114,11 @@ def w_h{T if isprim{T}} = to_w{T, width{T}/2} def n_d{[k]T} = [k*2]T # double/halve vector count def n_h{[k]T} = [k/2]T -def el_d{[k]T} = [k](w_d{T}) # double/halve element width, preserving count -def el_h{[k]T} = [k](w_h{T}) +def el_d{[k]T} = [k]w_d{T} # double/halve element width, preserving count +def el_h{[k]T} = [k]w_h{T} -def el_m{V=[_]T} = re_el{w_d{T}, V}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width -def el_s{V=[_]T} = re_el{w_h{T}, V}; def el_s{x:T} = re_el{T}~~x +def el_m{V=[_]T} = re_el{w_d{T}, V} # double/halve element width, preserving width +def el_s{V=[_]T} = re_el{w_h{T}, V} # type stats def minvalue{T if isunsigned{T}} = 0 diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 83b5794e..9b9b28b3 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -44,25 +44,25 @@ def maskAfter{n} = { def loadLowBatch{V=[_]T, ptr:*T, w, n} = loadLow{*V ~~ (ptr + n*(w/width{T})), w} -# store k items into the n'th batch of ptr elements, compressing the items if needed; masked by M -def storeBatch{ptr:*E0, n, x:V=[k]T, M} = { - def rpos = ptr + n*k - def TF = re_el{E0, V} +# store the i-th batch of k elements to ptr, narrowing elements if needed; masked by M +def storeBatch{ptr:*E0, i, x:[k]E1, M} = { + def rpos = ptr + i*k + def TF = re_el{E0, [k]E1} xu:= narrow{E0, x} if (M{0}) homMaskStoreF{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}} else storeLow{rpos, k*width{E0}, xu} } -# (sign/zero)-extend n'th batch of k elements of ptr into elements of V -def loadBatch{ptr:*E0, n, V=[k]T} = { - def rpos = ptr + n*k - widen{V, loadLow{*re_el{E0, V} ~~ rpos, k*width{E0}}} +# (sign/zero)-extend the i-th batch of k elements at ptr to [k]E1 +def loadBatch{ptr:*E0, i, [k]E1} = { + def rpos = ptr + i*k + def TF = re_el{E0, [k]E1} + widen{[k]E1, loadLow{*TF~~rpos, k*width{E0}}} } -def loadBatch {ptr:P, {...ns}, T } = each{{n } => loadBatch {ptr, n, T }, ns} -def storeBatch{ptr:P, {...ns}, xs, M} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs} - +def loadBatch {ptr:*E, {...ns}, T } = each{loadBatch {ptr, ., T }, ns} +def storeBatch{ptr:*E, {...ns}, xs, M} = each{storeBatch{ptr, ., ., M}, ns, xs} # "harmless" pointer cast that'll only cast void* @@ -71,14 +71,14 @@ def hCast{T,p:*T} = p def hCast{T,p:(*void)} = *T~~p def mlExec{i, iter, vars0, bulk, M} = { - def vproc{p:*T} = p + def vproc{p:*E} = p def vproc{'m'} = tptr{{_}=>M, '!'} - def vproc{{T,p:P}} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}} - def vproc{{'b', p:P}} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'} - def vproc{{'b',T,p:P}} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'} - def vproc{{'g', p:P}} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'} - def vproc{{'g',T,p:P}} = tptr{{i} => { + def vproc{{T,p:*E}} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}} + def vproc{{'b', p:*E}} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'} + def vproc{{'b',T,p:*E}} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'} + def vproc{{'g', p:*E}} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'} + def vproc{{'g',T,p:*E}} = tptr{{i} => { def dv{} = loadBatch{p, i, T} def dv{x} = storeBatch{p, i, x, M} }, '!'} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 3b5b515d..9f1bc2b6 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -34,9 +34,9 @@ def mla{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y def mls{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y def rbit{x:T if nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x} def rev{(width{T}), x:[_]T} = x -def rev{w==16, x:T if elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window -def rev{w==32, x:T if elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x} -def rev{w==64, x:T if elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x} +def rev{16, x:T if nveci{T} and elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window +def rev{32, x:T if nveci{T} and elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x} +def rev{64, x:T if nveci{T} and elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x} def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x} def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x} def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x} @@ -49,14 +49,14 @@ def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x} def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x} def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x} -def storeLow{ptr:*P, w, x:T if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} } -def storeLow{ptr:*P, w, x:T if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x} +def storeLow{ptr:*E, w, x:T=[_]E if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} } +def storeLow{ptr:*E, w, x:T=[_]E if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x} -def loadLow{ptr:*T, w if w<=64} = { # a broadcast load - def L=re_el{ty_u{w}, T} - T ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr} +def loadLow{ptr:*V, w if w<=64} = { # implemented via a broadcast load + def L=re_el{ty_u{w}, V} + V ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr} } -def loadLow{ptr:*T, (width{T})} = load{ptr} +def loadLow{ptr:*V, (width{V})} = load{ptr} @@ -65,8 +65,8 @@ def undefPromote{T=[_]E, x:X=[_]E if w64{X} and w128{T}} = emit{T, ntyp{'vcombin def half{x:T, (0) if w128{T}} = emit{n_h{T}, ntyp0{'vget_low', T}, x} def half{x:T, (1) if w128{T}} = emit{n_h{T}, ntyp0{'vget_high', T}, x} def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b} -def copyLane{dst:D=[_]T, di, src:S=[_]T, si if w64{D} and nvec{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si} -def copyLane{dst:D=[_]T, di, src:S=[_]T, si if w128{D} and nvec{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si} +def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w64{D} and nvec{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si} +def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w128{D} and nvec{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si} def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i} def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n} @@ -92,16 +92,16 @@ def cvt{T==f64, x:X=[k]_ if nveci{X,64}} = emit{[k]T, ntyp{'vcvt', '_f64', X}, x def cvt{T==i64, x:X=[k]_ if nvecf{X,64}} = emit{[k]T, ntyp{'vcvt', '_s64', X}, x} def cvt{T==u64, x:X=[k]_ if nvecf{X,64}} = emit{[k]T, ntyp{'vcvt', '_u64', X}, x} -def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x} -def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}> elwidth{X}*2} = widen{T, widen{el_s{T}, x}} -def widen{T=[k]E, x:X if w64{X} and isfloat{E}!=isfloat{eltype{X}} and elwidth{T}>elwidth{X}} = cvt{E, widen{[k](to_w{eltype{X},elwidth{T}}), x}} -def widen{T=[k]E, x:X if w128{X} and vcount{X}>k} = widen{T, half{x,0}} +def widen{R=[_]RE, x:X=[_]XE if w64{X} and eqqi{RE,XE} and width{RE}==width{XE}*2} = emit{R, ntyp{'vmovl', X}, x} +def widen{R=[_]RE, x:X=[_]XE if w64{X} and eqqi{RE,XE} and width{RE}> width{XE}*2} = widen{R, widen{el_s{R}, x}} +def widen{R=[rn]RE, x:X=[xn]XE if w64{X} and isfloat{RE}!=isfloat{XE} and width{RE}>width{XE}} = cvt{RE, widen{[rn]to_w{XE,width{RE}}, x}} +def widen{R=[rn]RE, x:X=[xn]XE if w128{X} and xn>rn} = widen{R, half{x,0}} -def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2< elwidth{X}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{eltype{X}}, x}}} -def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2==elwidth{X}} = emit{el_h{X}, ntyp0{'vmovn', X}, x} -def narrow{T, x:X if w128{X} and isfloat{T}!=isfloat{eltype{X}} and width{T}=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}} -def homMask{a:T,b:T=[n]E if n*2<=width{E}} = { - truncBits{n*2, fold_add{shrm{a,width{E}-n,b} & make{T, (1< { - def tb = width{E} * n - if (tb<=arch_defvw) tup{widen{[n]E, x}} + merge{...each{{x:X=[k]T} => { + def tb = width{E} * k + if (tb<=arch_defvw) tup{widen{[k]E, x}} else if (1) { assert{tb == 2*arch_defvw} tup{ - widen{[n/2]E, half{x,0}}, - widen{[n/2]E, half{x,1}} + widen{[k/2]E, half{x,0}}, + widen{[k/2]E, half{x,1}} } } }, xs}} diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 103df555..eaab2547 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -21,10 +21,10 @@ def anyNonChar{M, x:T=[_]_ if hasarch{'X86_64'}} = { } -def cvtNarrow{T, x:[_]X if width{T}==width{X}} = cvt{T, x} -def cvtNarrow{T, x:[_]X if width{T}< width{X}} = narrow{T, x} -def cvtWiden{ [_]T, x:[_]X if width{T}==width{X}} = cvt{T, x} -def cvtWiden{V=[_]T, x:[_]X if width{T}> width{X}} = widen{V, x} +def cvtNarrow{DE, x:[_]XE if width{DE}==width{XE}} = cvt{DE, x} +def cvtNarrow{DE, x:[_]XE if width{DE}< width{XE}} = narrow{DE, x} +def cvtWiden{ [_]DE, x:[_]XE if width{DE}==width{XE}} = cvt{DE, x} +def cvtWiden{D=[_]DE, x:[_]XE if width{DE}> width{XE}} = widen{D, x} fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { assert{len>0} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 14fc934f..a4f8c371 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -5,15 +5,15 @@ def v2d{x:T if w128{T}} = [2]f64 ~~ x # load & store -def loadLow{ptr:*T, w if w128{T} and w== 16} = T ~~ emit{[16]u8, '_mm_loadu_si16', ptr} -def loadLow{ptr:*T, w if w128{T} and w== 32} = T ~~ emit{[16]u8, '_mm_loadu_si32', ptr} -def loadLow{ptr:*T, w if w128{T} and w== 64} = T ~~ emit{[16]u8, '_mm_loadu_si64', ptr} -def loadLow{ptr:*T, w if w128{T} and w==128} = load{ptr} +def loadLow{ptr:*V, w if w128{V} and w== 16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr} +def loadLow{ptr:*V, w if w128{V} and w== 32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr} +def loadLow{ptr:*V, w if w128{V} and w== 64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr} +def loadLow{ptr:*V, w if w128{V} and w==128} = load{ptr} -def storeLow{ptr:*P, w, x:T if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} -def storeLow{ptr:*P, w, x:T if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} -def storeLow{ptr:*P, w, x:T if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} -def storeLow{ptr:*P, w, x:T if w128{T} and w==128} = store{*T~~ptr, 0, x} +def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} +def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} +def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} +def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w==128} = store{*T~~ptr, 0, x}