proper packs128 instead of packQ
This commit is contained in:
parent
7bf55793c5
commit
e3a985c6e2
@ -213,8 +213,8 @@ For float conversions, the used rounding mode is unspecified.
|
||||
- `mul32{a:VI, b:VI} : VI` - multiply, reading only low 32 bits
|
||||
- `blend{L, a:V, b:V, m}` - blend `L`-sized blocks via the immediate
|
||||
- `shuf{L, x:V, n} : V` - shuffle by immediate in `L`-sized lanes
|
||||
- `packQ` - pack 128-bit lanes (`packs`/`packus`) for 16-bit & 32-bit elements
|
||||
- `packQQ` - `packQ` but also defined for 64-bit elements, assuming the high halves are zeroes
|
||||
- `packs128` - saturating pack 128-bit lanes (`packs`/`packus`) for 16-bit & 32-bit elements
|
||||
- `packQQ` - `packs128` but also defined for 64-bit elements, assuming the high halves are zeroes
|
||||
- `packs` - 128-bit `packs`/`packus`
|
||||
- `shuf16Hi`, `shuf16Lo` - 16-bit shuffles with immediate
|
||||
- `shufHalves`
|
||||
|
||||
@ -1,8 +1,3 @@
|
||||
# questionable pack; these work in 128-bit lanes and saturate elements
|
||||
def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
|
||||
def packQ{a:T,b:T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
|
||||
def packQ{a:T,b:T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
|
||||
# super questionable pack - assumes high halves are zero
|
||||
def packQQ{a:T,b:T==[4]i64} = vec_shuffle{[4]i32, a, 0,2,1,1} | vec_shuffle{[4]i32, b, 1,1,0,2}
|
||||
def packQQ{{a, b}} = packQQ{a, b}
|
||||
@ -31,7 +26,7 @@ def any_hom{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
|
||||
def all_hom{x:T if w256i{T}} = hom_to_int{[32]u8 ~~ x} == 0xffff_ffff
|
||||
def any_top{x:T if w256i{T}} = top_to_int{x} != 0
|
||||
def all_top{x:T=[k]_ if w256i{T}} = top_to_int{x} == (1<<k)-1
|
||||
def hom_to_int{a:T, b:T if w256i{T,16}} = hom_to_int{vec_shuffle{[4]u64, packQ{ty_s{a},ty_s{b}}, 0,2,1,3}}
|
||||
def hom_to_int{a:T, b:T if w256i{T,16}} = hom_to_int{vec_shuffle{[4]u64, packs128{ty_s{a},ty_s{b}}, 0,2,1,3}}
|
||||
|
||||
def any_top{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
|
||||
def any_top{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
|
||||
@ -43,12 +38,12 @@ def all_top{x:T if w256i{T,16}} = all_hom{[16]i16~~x < [16]i16**0}
|
||||
|
||||
# conversion
|
||||
def narrow{T, x:X if w256i{X,32} and width{T}==8} = {
|
||||
a:= packQ{x, x}
|
||||
b:= packQ{a, a}
|
||||
a:= packs128{x, x}
|
||||
b:= packs128{a, a}
|
||||
re_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}}
|
||||
}
|
||||
def narrow{T, x:X if w256i{X,32} and width{T}==16} = re_el{T, vec_shuffle{[4]u64, packQ{x, x}, 0,2,1,3}}
|
||||
def narrow{T, x:X if w256i{X,16} and width{T}== 8} = re_el{T, vec_shuffle{[4]u64, packQ{x, x}, 0,2,1,3}}
|
||||
def narrow{T, x:X if w256i{X,32} and width{T}==16} = re_el{T, vec_shuffle{[4]u64, packs128{x, x}, 0,2,1,3}}
|
||||
def narrow{T, x:X if w256i{X,16} and width{T}== 8} = re_el{T, vec_shuffle{[4]u64, packs128{x, x}, 0,2,1,3}}
|
||||
|
||||
def narrow{T, x:X if w256f{X,64} and T<i32} = narrow{T, narrow{i32, x}}
|
||||
def narrow{T, x:X if w256f{X,64} and T==i32} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
|
||||
@ -65,12 +60,12 @@ def cvt2{(f64), x:X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}
|
||||
def unzip128{a:[k]E, b:[k]E, 0 if hasarch{'X86_64'} and isunsigned{E}} = {
|
||||
def ED = w_d{E}
|
||||
def c = make{[k]E, maxvalue{E} * (1-(1 & range{k}))}
|
||||
packQ{re_el{ED,a&c}, re_el{ED,b&c}}
|
||||
packs128{re_el{ED,a&c}, re_el{ED,b&c}}
|
||||
}
|
||||
def unzip128{a:[k]E, b:[k]E, 1 if hasarch{'X86_64'} and isunsigned{E}} = {
|
||||
def ED = w_d{E}
|
||||
def ew = width{E}
|
||||
packQ{re_el{ED,a} >> ew, re_el{ED,b} >> ew}
|
||||
packs128{re_el{ED,a} >> ew, re_el{ED,b} >> ew}
|
||||
}
|
||||
def unzip128{a:T, b:T==[8](u32), k} = shuf_ind{a, b, tup{0,2,8,10, 4,6,12,14}+k}
|
||||
def unzip128{a:T, b:T==[4](u64), k} = shuf_ind{a, b, tup{0,4,2,6}+k}
|
||||
|
||||
@ -278,7 +278,8 @@ local def extend kpair{op} = {
|
||||
}
|
||||
extend (extend_each{kpair}){pack, zip, zip128, mzip, mzip128, unzip, unzip128}
|
||||
|
||||
def packQ{{a, b}} = packQ{a, b}
|
||||
def packs{{a, b}} = packs{a, b}
|
||||
def packs128{{a, b}} = packs128{a, b}
|
||||
def pair{{a, b}} = pair{a, b}
|
||||
|
||||
def widen{T, x:T} = x
|
||||
|
||||
@ -39,11 +39,11 @@ def arithChk2{F==__mul, M, w:T=[_](i8), x:T if hasarch{'X86_64'}} = {
|
||||
def rp = each{__mul, wp, xp}
|
||||
if (M{0}) { # masked check
|
||||
def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp}
|
||||
tup{packQ{rp}, tup{'any_hom', M{packQ{bad}}}}
|
||||
tup{packs128{rp}, tup{'any_hom', M{packs128{bad}}}}
|
||||
} else { # unmasked check; can do check in a simpler way
|
||||
def bad = each{{v} => ty_s{v ^ (v>>15)}, rp}
|
||||
def RU = re_el{u16,T}
|
||||
tup{packQ{rp}, tup{'and_some', RU~~tree_fold{|, bad}, RU**0xff80}}
|
||||
tup{packs128{rp}, tup{'and_some', RU~~tree_fold{|, bad}, RU**0xff80}}
|
||||
}
|
||||
}
|
||||
def arithChk2{F==__mul, M, w:T=[_](i16), x:T if hasarch{'X86_64'}} = {
|
||||
|
||||
@ -87,7 +87,7 @@ def extract_column_pow2{T, x0, r0, nv, k} = {
|
||||
m := make{V, - (iota{vcount{V}}%k == 0)}
|
||||
xs = each{&{m, .}, xs} # Mask off high bits
|
||||
def D = el_m{V}
|
||||
{a, b} => packQ{D~~a, D~~b}
|
||||
{a, b} => packs128{D~~a, D~~b}
|
||||
} else {
|
||||
# Two-vector shuffles
|
||||
# Could also be used for 1/2-byte with ending gap >= 4 bytes,
|
||||
@ -108,7 +108,7 @@ def extract_column_pow2{T, x0, r0, nv, k} = {
|
||||
top := D**(1<<15); m := D**(1<<16 - 1)
|
||||
(ri & m) | (D~~(ri&top == top) &~ m)
|
||||
}
|
||||
r = V~~packQ{...each{proc, split{k/2, xs}}}
|
||||
r = V~~packs128{each{proc, split{k/2, xs}}}
|
||||
}
|
||||
if (width{V} > 128) { # Lane axis wasn't packed, need to shuffle to bottom
|
||||
def tr{E,a, r} = shuf{[1<<a]E, r, tr_iota{shiftright{a-1, iota{a}}}}
|
||||
|
||||
@ -3,7 +3,6 @@ def sel{L=[_]E, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w1
|
||||
|
||||
|
||||
### SSE4.1 ###
|
||||
def packs{a:T,b:T if hasarch{'SSE4.1'} and T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
|
||||
def and_bit_none{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
|
||||
# conversion
|
||||
|
||||
@ -41,12 +41,6 @@ def all_top{x:T=[k]_ if w128i{T}} = top_to_int{x} == (1<<k)-1
|
||||
def any_top{x:T if w128i{T, 16}} = any_hom{[8]i16~~x < [8]i16**0}
|
||||
def all_top{x:T if w128i{T, 16}} = all_hom{[8]i16~~x < [8]i16**0}
|
||||
|
||||
# bits of other things SSE2 has
|
||||
def packs{a:T,b:T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
|
||||
def packs{a:T,b:T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
|
||||
def packs{a:T,b:T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T if w128i{T}} = packs{a,b}
|
||||
|
||||
def x86_shufps_range{is, hi} = in_range{is,0,2, 0,4} and in_range{is,2,4, hi,hi+4}
|
||||
def shuf_ind{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3}
|
||||
|
||||
|
||||
@ -114,6 +114,12 @@ def mul_sum_sat{2, a:V=[k](u8), b:[k](i8) if has_bw{V}} = {
|
||||
emit{[k/2]i16, intrin{V, 'maddubs_epi16'}, a, b}
|
||||
}
|
||||
|
||||
local def packs_impl{check}{a:V=[k]E, b:V if veci{V} and check{V} and (width{E}==16 or width{E}==32) and hase{V, if (E==u32) 'SSE4.1' else 'SSE2', 'AVX2', 'AVX512BW'}} = {
|
||||
emit{el_s{V}, intrin_i{V, 'pack', if (isunsigned{E}) 'us' else 's'}, a, b}
|
||||
}
|
||||
def packs128{...} = packs_impl{{_}=>1}
|
||||
def packs{...} = packs_impl{{V}=>width{V}==128}
|
||||
|
||||
def widen{D=[k]DE, x:S=[k0]SE if isint{DE} and quality{DE}==quality{SE} and DE>SE and k<=k0 and hasarch{match (width{D}) {
|
||||
{128} => 'SSE4.1'
|
||||
{256} => 'AVX2'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user