diff --git a/src/singeli/README.md b/src/singeli/README.md index fcb00a56..5766850c 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -146,13 +146,13 @@ Some may also support one scalar argument or arguments with different widths. - `vshl{a:V, b:V, n} : V` - `vshl{[0,1,2,3], [4,5,6,7], 1} → [1,2,3,4]` - `sel{VI1, a:V, b:VI2} : V2` - shuffle `a` by indices `b` in `VI1`-long lanes; arch-specific behavior on out-of-bounds values -- `zip{a:V,b:V} : tup{V, V}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}` -- `mzip{a:V,b:V} : tup{el_dbl{V}, el_dbl{V}}` - reinterpreted `zip{a, b}` -- `zipLo{a:V,b:V} : V` - `select{zip{a, b}, 0}` -- `zipHi{a:V,b:V} : V` - `select{zip{a, b}, 1}` -- `mzipLo{a:V,b:V} : V` - `select{mzip{a, b}, 0}` -- `mzipHi{a:V,b:V} : V` - `select{mzip{a, b}, 1}` -- `zip128{,Lo,Hi}`, `mzip128{,Lo,Hi}` - within-128-bit-lane versions of the above +- `zip{a:V, b:V} : tup{V, V}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}` +- `mzip{a:V, b:V} : tup{el_dbl{V}, el_dbl{V}}` - reinterpreted `zip{a, b}` +- `pack{a:V, b:V} : tup{}` +- `zip{a:V, b:V, k} : V` - `select{zip{a, b}, k}` +- `mzip{a:V, b:V, k} : V` - `select{mzip{a, b}, k}` +- `pack{a:V, b:V, k} : V` - `select{pack{a, b}, k}` +- `zip128`, `mzip128` - within-128-bit-lane versions of the above ## Mask stuff @@ -242,7 +242,6 @@ For float conversions, the used rounding mode is unspecified. - `mla` - multiply-add - `mls` - multiply-subtract - `ornot` - a|~b -- `packLo`, `packHi` - `popc` - popcount of each byte - `rbit` - reverse bits in bytes - `rev` - reverse elements in lanes diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 9484034a..839ed815 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -1,5 +1,5 @@ -def zip128Lo{a:T, b:T if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b} -def zip128Hi{a:T, b:T if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b} +def zip128{a:T, b:T, 0 if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b} +def zip128{a:T, b:T, 1 if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b} # questionable pack; these saturate the argument def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 5f4d0d75..d957d983 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -161,10 +161,9 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 } def { absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half, homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, - loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packHi,packLo,packQ,pair,pdep, + loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep, pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,shufInd,storeLow, topBlend,topMask,topMaskStore,topMaskStoreF,unord,vfold,widen,widenUpper, - zipHi,zipLo } def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times @@ -192,18 +191,15 @@ if_inline (hasarch{'X86_64'}) { def {__adds,__subs,__sqrt,vec_broadcast,vec_make} } -def zip128Lo{a:T, b:T if width{T}==128} = zipLo{a, b} -def zip128Hi{a:T, b:T if width{T}==128} = zipHi{a, b} -def zip {a:T, b:T} = tup{zipLo {a,b}, zipHi {a,b}} -def zip128{a:T, b:T} = tup{zip128Lo{a,b}, zip128Hi{a,b}} +def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k} +def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k} -local def make_mzip{l, h} = tup{ - {a:T, b:T} => el_m{T} ~~ l{a, b}, - {a:T, b:T} => el_m{T} ~~ h{a, b}, - {a:T, b:T} => each{{z} => el_m{T}~~z{a, b}, tup{l, h}} +def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k} +def mzip128{a:T, b:T, k} = el_m{T} ~~ zip128{a, b, k} +local def extend kpair{op} = { + def op{a:T, b:T} = tup{op{a,b,0}, op{a,b,1}} } -def {mzipLo, mzipHi, mzip } = make_mzip{zipLo, zipHi} -def {mzip128Lo, mzip128Hi, mzip128} = make_mzip{zip128Lo, zip128Hi} +extend ({...f}=>each{kpair,f}){pack, zip, zip128, mzip, mzip128} def packQ{{a, b}} = packQ{a, b} def pair{{a, b}} = pair{a, b} diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index 80071119..a8edba4d 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -74,7 +74,7 @@ def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = { } def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = { b:= T~~[8]u16**a - exp:= T~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100} + exp:= T~~shuf{[4]i32, shuf16Lo{mzip{b,b,0}, 4b1100}, 4b1100} (exp & make{T, 1<<(iota{16}&7)}) != T**0 } diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index bbd56fd7..e60e6529 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -70,8 +70,8 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = { def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = { def r12 = mulw{w, x} - rl:= packLo{r12} - rh:= packHi{r12} + rl:= pack{...r12, 0} + rh:= pack{...r12, 1} tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}} } diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 9df9b845..e3fe485e 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -70,20 +70,18 @@ def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w128{D} and nvec{S}} = emit{D, nt def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i} def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n} -def zipLo{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b} -def zipHi{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b} +def zip{a:T, b:T, 0 if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b} +def zip{a:T, b:T, 1 if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b} -def unzipLo{x:T, y:T if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y} -def unzipHi{x:T, y:T if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y} -def packLo{x:T, y:T if nvec{T}} = unzipLo{el_s{T}~~x, el_s{T}~~y} -def packHi{x:T, y:T if nvec{T}} = unzipHi{el_s{T}~~x, el_s{T}~~y} -def packLo{{x, y}} = packLo{x, y} -def packHi{{x, y}} = packHi{x, y} -def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 2*range{vcount{T}}}} = T~~unzipLo{x,y} -def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 1+2*range{vcount{T}}}} = T~~unzipHi{x,y} +def unzip{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y} +def unzip{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y} +def pack{x:T, y:T, 0 if nvec{T}} = unzip{el_s{T}~~x, el_s{T}~~y, 0} +def pack{x:T, y:T, 1 if nvec{T}} = unzip{el_s{T}~~x, el_s{T}~~y, 1} +def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 2*range{vcount{T}}}} = T~~unzip{x,y,0} +def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 1+2*range{vcount{T}}}} = T~~unzip{x,y,1} -def trn1{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y} -def trn2{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y} +def trn{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y} +def trn{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y} def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, re_el{eltype{I},x}, ty_u{i}}} def sel{{x0:T}, i:I} = sel{T, x0, i} @@ -112,7 +110,7 @@ def narrow {T, x:X=[_]E if w128{X} and isfloat{T}!=isfloat{E} and width{T} hasarch{'AVX2'}} = 2 def thresh_bool{if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32 def pext_popc{x0:V, m0:V if hasarch{'PCLMUL'} and V==[2]u64} = { - def clmul{a, b} = zipLo{...@collect (j to 2) clmul{a,b,j}} + def clmul{a, b} = zip{...@collect (j to 2) clmul{a,b,j}, 0} m := m0 x := x0 & m d := ~m << 1 # One bit of the position difference at x diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index bbd69ec4..d6bb4cfc 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -88,7 +88,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { if (B) { if (any_nonC32{M, ...xs}) return{3} } else { - def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) packHi{...xs} else packHi{orx,orx}} + def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}} else ~andAllZero{orx, ~XV**maxvalue{w_h{X}}} if (bad) return{lb{width{X}}-3} } @@ -140,7 +140,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { assert{hasarch{'AARCH64'}} def k = length{xs} def int = each{cvt{i64,.}, xs} - def int32 = if (k==2) packLo{...int} else packLo{...int, [bulk]i64**0} + def int32 = if (k==2) pack{...int, 0} else pack{...int, [bulk]i64**0, 0} def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widenUpper}, 0, k}} if (~homAll{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index f1b75cff..c7ae639f 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -36,7 +36,7 @@ def __mul{a:T,b:T if [4]i32==T} = { def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs} lo:= mu{a, b} hi:= mu{...sw{4b2301, a, b}} - [4]i32~~mzipLo{...sw{4b3120, lo, hi}} + [4]i32~~mzip{...sw{4b3120, lo, hi}, 0} } # float arith @@ -71,8 +71,8 @@ def packQ{a:T,b:T if w128i{T}} = packs{a,b} def x86_postfix{T if isint{T}} = merge{'_epi',fmtnat{width{T}}} def x86_postfix{(f32)} = '_ps' def x86_postfix{(f64)} = '_pd' -def zipLo{a:T, b:T if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b} -def zipHi{a:T, b:T if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b} +def zip{a:T, b:T, 0 if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b} +def zip{a:T, b:T, 1 if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b} def shuf{L, x:T, n if w128{T} and lvec{L,4,32} and knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n} def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n} @@ -84,7 +84,7 @@ def homBlend{f:T, t:T, m:M if w128{T} and w128i{M,elwidth{T}}} = T ~~ ((M~~t & m def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}} def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = { - def s{v} = s{mzipLo{v, v}} + def s{v} = s{mzip{v,v,0}} def s{v:(T)} = v s{x} >> (elwidth{T} - elwidth{X}) }