proper packs128 instead of packQ

This commit is contained in:
dzaima 2025-04-27 19:06:48 +03:00
parent 7bf55793c5
commit e3a985c6e2
8 changed files with 21 additions and 26 deletions

View File

@ -213,8 +213,8 @@ For float conversions, the used rounding mode is unspecified.
- `mul32{a:VI, b:VI} : VI` - multiply, reading only low 32 bits - `mul32{a:VI, b:VI} : VI` - multiply, reading only low 32 bits
- `blend{L, a:V, b:V, m}` - blend `L`-sized blocks via the immediate - `blend{L, a:V, b:V, m}` - blend `L`-sized blocks via the immediate
- `shuf{L, x:V, n} : V` - shuffle by immediate in `L`-sized lanes - `shuf{L, x:V, n} : V` - shuffle by immediate in `L`-sized lanes
- `packQ` - pack 128-bit lanes (`packs`/`packus`) for 16-bit & 32-bit elements - `packs128` - saturating pack 128-bit lanes (`packs`/`packus`) for 16-bit & 32-bit elements
- `packQQ` - `packQ` but also defined for 64-bit elements, assuming the high halves are zeroes - `packQQ` - `packs128` but also defined for 64-bit elements, assuming the high halves are zeroes
- `packs` - 128-bit `packs`/`packus` - `packs` - 128-bit `packs`/`packus`
- `shuf16Hi`, `shuf16Lo` - 16-bit shuffles with immediate - `shuf16Hi`, `shuf16Lo` - 16-bit shuffles with immediate
- `shufHalves` - `shufHalves`

View File

@ -1,8 +1,3 @@
# questionable pack; these work in 128-bit lanes and saturate elements
def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
def packQ{a:T,b:T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
def packQ{a:T,b:T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
# super questionable pack - assumes high halves are zero # super questionable pack - assumes high halves are zero
def packQQ{a:T,b:T==[4]i64} = vec_shuffle{[4]i32, a, 0,2,1,1} | vec_shuffle{[4]i32, b, 1,1,0,2} def packQQ{a:T,b:T==[4]i64} = vec_shuffle{[4]i32, a, 0,2,1,1} | vec_shuffle{[4]i32, b, 1,1,0,2}
def packQQ{{a, b}} = packQQ{a, b} def packQQ{{a, b}} = packQQ{a, b}
@ -31,7 +26,7 @@ def any_hom{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
def all_hom{x:T if w256i{T}} = hom_to_int{[32]u8 ~~ x} == 0xffff_ffff def all_hom{x:T if w256i{T}} = hom_to_int{[32]u8 ~~ x} == 0xffff_ffff
def any_top{x:T if w256i{T}} = top_to_int{x} != 0 def any_top{x:T if w256i{T}} = top_to_int{x} != 0
def all_top{x:T=[k]_ if w256i{T}} = top_to_int{x} == (1<<k)-1 def all_top{x:T=[k]_ if w256i{T}} = top_to_int{x} == (1<<k)-1
def hom_to_int{a:T, b:T if w256i{T,16}} = hom_to_int{vec_shuffle{[4]u64, packQ{ty_s{a},ty_s{b}}, 0,2,1,3}} def hom_to_int{a:T, b:T if w256i{T,16}} = hom_to_int{vec_shuffle{[4]u64, packs128{ty_s{a},ty_s{b}}, 0,2,1,3}}
def any_top{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}} def any_top{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
def any_top{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}} def any_top{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
@ -43,12 +38,12 @@ def all_top{x:T if w256i{T,16}} = all_hom{[16]i16~~x < [16]i16**0}
# conversion # conversion
def narrow{T, x:X if w256i{X,32} and width{T}==8} = { def narrow{T, x:X if w256i{X,32} and width{T}==8} = {
a:= packQ{x, x} a:= packs128{x, x}
b:= packQ{a, a} b:= packs128{a, a}
re_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}} re_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}}
} }
def narrow{T, x:X if w256i{X,32} and width{T}==16} = re_el{T, vec_shuffle{[4]u64, packQ{x, x}, 0,2,1,3}} def narrow{T, x:X if w256i{X,32} and width{T}==16} = re_el{T, vec_shuffle{[4]u64, packs128{x, x}, 0,2,1,3}}
def narrow{T, x:X if w256i{X,16} and width{T}== 8} = re_el{T, vec_shuffle{[4]u64, packQ{x, x}, 0,2,1,3}} def narrow{T, x:X if w256i{X,16} and width{T}== 8} = re_el{T, vec_shuffle{[4]u64, packs128{x, x}, 0,2,1,3}}
def narrow{T, x:X if w256f{X,64} and T<i32} = narrow{T, narrow{i32, x}} def narrow{T, x:X if w256f{X,64} and T<i32} = narrow{T, narrow{i32, x}}
def narrow{T, x:X if w256f{X,64} and T==i32} = emit{[4]i32, '_mm256_cvtpd_epi32', x} def narrow{T, x:X if w256f{X,64} and T==i32} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
@ -65,12 +60,12 @@ def cvt2{(f64), x:X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}
def unzip128{a:[k]E, b:[k]E, 0 if hasarch{'X86_64'} and isunsigned{E}} = { def unzip128{a:[k]E, b:[k]E, 0 if hasarch{'X86_64'} and isunsigned{E}} = {
def ED = w_d{E} def ED = w_d{E}
def c = make{[k]E, maxvalue{E} * (1-(1 & range{k}))} def c = make{[k]E, maxvalue{E} * (1-(1 & range{k}))}
packQ{re_el{ED,a&c}, re_el{ED,b&c}} packs128{re_el{ED,a&c}, re_el{ED,b&c}}
} }
def unzip128{a:[k]E, b:[k]E, 1 if hasarch{'X86_64'} and isunsigned{E}} = { def unzip128{a:[k]E, b:[k]E, 1 if hasarch{'X86_64'} and isunsigned{E}} = {
def ED = w_d{E} def ED = w_d{E}
def ew = width{E} def ew = width{E}
packQ{re_el{ED,a} >> ew, re_el{ED,b} >> ew} packs128{re_el{ED,a} >> ew, re_el{ED,b} >> ew}
} }
def unzip128{a:T, b:T==[8](u32), k} = shuf_ind{a, b, tup{0,2,8,10, 4,6,12,14}+k} def unzip128{a:T, b:T==[8](u32), k} = shuf_ind{a, b, tup{0,2,8,10, 4,6,12,14}+k}
def unzip128{a:T, b:T==[4](u64), k} = shuf_ind{a, b, tup{0,4,2,6}+k} def unzip128{a:T, b:T==[4](u64), k} = shuf_ind{a, b, tup{0,4,2,6}+k}

View File

@ -278,7 +278,8 @@ local def extend kpair{op} = {
} }
extend (extend_each{kpair}){pack, zip, zip128, mzip, mzip128, unzip, unzip128} extend (extend_each{kpair}){pack, zip, zip128, mzip, mzip128, unzip, unzip128}
def packQ{{a, b}} = packQ{a, b} def packs{{a, b}} = packs{a, b}
def packs128{{a, b}} = packs128{a, b}
def pair{{a, b}} = pair{a, b} def pair{{a, b}} = pair{a, b}
def widen{T, x:T} = x def widen{T, x:T} = x

View File

@ -39,11 +39,11 @@ def arithChk2{F==__mul, M, w:T=[_](i8), x:T if hasarch{'X86_64'}} = {
def rp = each{__mul, wp, xp} def rp = each{__mul, wp, xp}
if (M{0}) { # masked check if (M{0}) { # masked check
def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp} def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp}
tup{packQ{rp}, tup{'any_hom', M{packQ{bad}}}} tup{packs128{rp}, tup{'any_hom', M{packs128{bad}}}}
} else { # unmasked check; can do check in a simpler way } else { # unmasked check; can do check in a simpler way
def bad = each{{v} => ty_s{v ^ (v>>15)}, rp} def bad = each{{v} => ty_s{v ^ (v>>15)}, rp}
def RU = re_el{u16,T} def RU = re_el{u16,T}
tup{packQ{rp}, tup{'and_some', RU~~tree_fold{|, bad}, RU**0xff80}} tup{packs128{rp}, tup{'and_some', RU~~tree_fold{|, bad}, RU**0xff80}}
} }
} }
def arithChk2{F==__mul, M, w:T=[_](i16), x:T if hasarch{'X86_64'}} = { def arithChk2{F==__mul, M, w:T=[_](i16), x:T if hasarch{'X86_64'}} = {

View File

@ -87,7 +87,7 @@ def extract_column_pow2{T, x0, r0, nv, k} = {
m := make{V, - (iota{vcount{V}}%k == 0)} m := make{V, - (iota{vcount{V}}%k == 0)}
xs = each{&{m, .}, xs} # Mask off high bits xs = each{&{m, .}, xs} # Mask off high bits
def D = el_m{V} def D = el_m{V}
{a, b} => packQ{D~~a, D~~b} {a, b} => packs128{D~~a, D~~b}
} else { } else {
# Two-vector shuffles # Two-vector shuffles
# Could also be used for 1/2-byte with ending gap >= 4 bytes, # Could also be used for 1/2-byte with ending gap >= 4 bytes,
@ -108,7 +108,7 @@ def extract_column_pow2{T, x0, r0, nv, k} = {
top := D**(1<<15); m := D**(1<<16 - 1) top := D**(1<<15); m := D**(1<<16 - 1)
(ri & m) | (D~~(ri&top == top) &~ m) (ri & m) | (D~~(ri&top == top) &~ m)
} }
r = V~~packQ{...each{proc, split{k/2, xs}}} r = V~~packs128{each{proc, split{k/2, xs}}}
} }
if (width{V} > 128) { # Lane axis wasn't packed, need to shuffle to bottom if (width{V} > 128) { # Lane axis wasn't packed, need to shuffle to bottom
def tr{E,a, r} = shuf{[1<<a]E, r, tr_iota{shiftright{a-1, iota{a}}}} def tr{E,a, r} = shuf{[1<<a]E, r, tr_iota{shiftright{a-1, iota{a}}}}

View File

@ -3,7 +3,6 @@ def sel{L=[_]E, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w1
### SSE4.1 ### ### SSE4.1 ###
def packs{a:T,b:T if hasarch{'SSE4.1'} and T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
def and_bit_none{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y} def and_bit_none{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
# conversion # conversion

View File

@ -41,12 +41,6 @@ def all_top{x:T=[k]_ if w128i{T}} = top_to_int{x} == (1<<k)-1
def any_top{x:T if w128i{T, 16}} = any_hom{[8]i16~~x < [8]i16**0} def any_top{x:T if w128i{T, 16}} = any_hom{[8]i16~~x < [8]i16**0}
def all_top{x:T if w128i{T, 16}} = all_hom{[8]i16~~x < [8]i16**0} def all_top{x:T if w128i{T, 16}} = all_hom{[8]i16~~x < [8]i16**0}
# bits of other things SSE2 has
def packs{a:T,b:T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packQ{a:T,b:T if w128i{T}} = packs{a,b}
def x86_shufps_range{is, hi} = in_range{is,0,2, 0,4} and in_range{is,2,4, hi,hi+4} def x86_shufps_range{is, hi} = in_range{is,0,2, 0,4} and in_range{is,2,4, hi,hi+4}
def shuf_ind{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3} def shuf_ind{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3}

View File

@ -114,6 +114,12 @@ def mul_sum_sat{2, a:V=[k](u8), b:[k](i8) if has_bw{V}} = {
emit{[k/2]i16, intrin{V, 'maddubs_epi16'}, a, b} emit{[k/2]i16, intrin{V, 'maddubs_epi16'}, a, b}
} }
local def packs_impl{check}{a:V=[k]E, b:V if veci{V} and check{V} and (width{E}==16 or width{E}==32) and hase{V, if (E==u32) 'SSE4.1' else 'SSE2', 'AVX2', 'AVX512BW'}} = {
emit{el_s{V}, intrin_i{V, 'pack', if (isunsigned{E}) 'us' else 's'}, a, b}
}
def packs128{...} = packs_impl{{_}=>1}
def packs{...} = packs_impl{{V}=>width{V}==128}
def widen{D=[k]DE, x:S=[k0]SE if isint{DE} and quality{DE}==quality{SE} and DE>SE and k<=k0 and hasarch{match (width{D}) { def widen{D=[k]DE, x:S=[k0]SE if isint{DE} and quality{DE}==quality{SE} and DE>SE and k<=k0 and hasarch{match (width{D}) {
{128} => 'SSE4.1' {128} => 'SSE4.1'
{256} => 'AVX2' {256} => 'AVX2'