diff --git a/src/singeli/README.md b/src/singeli/README.md index a13069e2..3f723170 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -213,8 +213,8 @@ For float conversions, the used rounding mode is unspecified. - `mul32{a:VI, b:VI} : VI` - multiply, reading only low 32 bits - `blend{L, a:V, b:V, m}` - blend `L`-sized blocks via the immediate - `shuf{L, x:V, n} : V` - shuffle by immediate in `L`-sized lanes -- `packQ` - pack 128-bit lanes (`packs`/`packus`) for 16-bit & 32-bit elements -- `packQQ` - `packQ` but also defined for 64-bit elements, assuming the high halves are zeroes +- `packs128` - saturating pack 128-bit lanes (`packs`/`packus`) for 16-bit & 32-bit elements +- `packQQ` - `packs128` but also defined for 64-bit elements, assuming the high halves are zeroes - `packs` - 128-bit `packs`/`packus` - `shuf16Hi`, `shuf16Lo` - 16-bit shuffles with immediate - `shufHalves` diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index ab8ec284..d7d926fe 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -1,8 +1,3 @@ -# questionable pack; these work in 128-bit lanes and saturate elements -def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b} -def packQ{a:T,b:T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b} -def packQ{a:T,b:T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b} -def packQ{a:T,b:T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b} # super questionable pack - assumes high halves are zero def packQQ{a:T,b:T==[4]i64} = vec_shuffle{[4]i32, a, 0,2,1,1} | vec_shuffle{[4]i32, b, 1,1,0,2} def packQQ{{a, b}} = packQQ{a, b} @@ -31,7 +26,7 @@ def any_hom{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}} def all_hom{x:T if w256i{T}} = hom_to_int{[32]u8 ~~ x} == 0xffff_ffff def any_top{x:T if w256i{T}} = top_to_int{x} != 0 def all_top{x:T=[k]_ if w256i{T}} = top_to_int{x} == (1<> ew, re_el{ED,b} >> ew} + packs128{re_el{ED,a} >> ew, re_el{ED,b} >> ew} } def unzip128{a:T, b:T==[8](u32), k} = shuf_ind{a, b, tup{0,2,8,10, 4,6,12,14}+k} def unzip128{a:T, b:T==[4](u64), k} = shuf_ind{a, b, tup{0,4,2,6}+k} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 9433503d..8d4741e4 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -278,7 +278,8 @@ local def extend kpair{op} = { } extend (extend_each{kpair}){pack, zip, zip128, mzip, mzip128, unzip, unzip128} -def packQ{{a, b}} = packQ{a, b} +def packs{{a, b}} = packs{a, b} +def packs128{{a, b}} = packs128{a, b} def pair{{a, b}} = pair{a, b} def widen{T, x:T} = x diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 21801c26..c2e359e1 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -39,11 +39,11 @@ def arithChk2{F==__mul, M, w:T=[_](i8), x:T if hasarch{'X86_64'}} = { def rp = each{__mul, wp, xp} if (M{0}) { # masked check def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp} - tup{packQ{rp}, tup{'any_hom', M{packQ{bad}}}} + tup{packs128{rp}, tup{'any_hom', M{packs128{bad}}}} } else { # unmasked check; can do check in a simpler way def bad = each{{v} => ty_s{v ^ (v>>15)}, rp} def RU = re_el{u16,T} - tup{packQ{rp}, tup{'and_some', RU~~tree_fold{|, bad}, RU**0xff80}} + tup{packs128{rp}, tup{'and_some', RU~~tree_fold{|, bad}, RU**0xff80}} } } def arithChk2{F==__mul, M, w:T=[_](i16), x:T if hasarch{'X86_64'}} = { diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 0966f776..3b4bba5d 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -87,7 +87,7 @@ def extract_column_pow2{T, x0, r0, nv, k} = { m := make{V, - (iota{vcount{V}}%k == 0)} xs = each{&{m, .}, xs} # Mask off high bits def D = el_m{V} - {a, b} => packQ{D~~a, D~~b} + {a, b} => packs128{D~~a, D~~b} } else { # Two-vector shuffles # Could also be used for 1/2-byte with ending gap >= 4 bytes, @@ -108,7 +108,7 @@ def extract_column_pow2{T, x0, r0, nv, k} = { top := D**(1<<15); m := D**(1<<16 - 1) (ri & m) | (D~~(ri&top == top) &~ m) } - r = V~~packQ{...each{proc, split{k/2, xs}}} + r = V~~packs128{each{proc, split{k/2, xs}}} } if (width{V} > 128) { # Lane axis wasn't packed, need to shuffle to bottom def tr{E,a, r} = shuf{[1<1} +def packs{...} = packs_impl{{V}=>width{V}==128} + def widen{D=[k]DE, x:S=[k0]SE if isint{DE} and quality{DE}==quality{SE} and DE>SE and k<=k0 and hasarch{match (width{D}) { {128} => 'SSE4.1' {256} => 'AVX2'