Merge pull request #122 from mlochbaum/vec-select
Use Singeli's select includes
This commit is contained in:
commit
4c8b8519fa
@ -1 +1 @@
|
||||
Subproject commit b43f3999b0c5a40b43ceee258fbe6bb8245d06af
|
||||
Subproject commit d432cb710911457169da5342d27ce8adffd5dd1a
|
||||
@ -36,14 +36,13 @@ def widen{T=[k]_, x:X=[l]_ if w256{X} and l>k} = widen{T, half{x,0}}
|
||||
|
||||
|
||||
# structural operations
|
||||
def topBlend{f:T, t:T, m:M if w256{T,32} and w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M if w256{T,64} and w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def homBlend{f:T, t:T, m:M if w256{T}} = topBlend{f, t, m}
|
||||
def topBlend{f:T, t:T, m:M if w256{T,32} and w256i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M if w256{T,64} and w256i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, v2d{m}}
|
||||
|
||||
# shuffles
|
||||
# TODO rename to shuf, move shuf to shuf_imm or something
|
||||
def shufInd{a:T, b:T=[8]E, {...is} if width{E}==32 and length{is}==8 and ({l,h} => same{l,h-4} and x86_shufps_range{l, 8}){slice{is,0,4}, slice{is,4,8}}} = {
|
||||
T~~emit{[8]f32, '_mm256_shuffle_ps', v2f{a}, v2f{b}, base{4, slice{is,0,4} & 3}}
|
||||
vec_shuffle{[4]f32, tup{a, b}, slice{is,0,4} & 3}
|
||||
}
|
||||
def shufInd{a:T, b:T=[4]E, {...is} if width{E}==64 and length{is}==4} = T~~shufInd{re_el{u32,a}, re_el{u32,b}, merge{...each{{i} => tup{i*2, i*2+1}, is}}}
|
||||
|
||||
|
||||
@ -1,13 +1,10 @@
|
||||
def zip128{a:T, b:T, 0 if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip128{a:T, b:T, 1 if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b}
|
||||
|
||||
# questionable pack; these work in 128-bit lanes and saturate elements
|
||||
def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
|
||||
def packQ{a:T,b:T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
|
||||
def packQ{a:T,b:T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
|
||||
# super questionable pack - assumes high halves are zero
|
||||
def packQQ{a:T,b:T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
|
||||
def packQQ{a:T,b:T==[4]i64} = vec_shuffle{[4]i32, a, 0,2,1,1} | vec_shuffle{[4]i32, b, 1,1,0,2}
|
||||
def packQQ{{a, b}} = packQQ{a, b}
|
||||
|
||||
|
||||
@ -19,23 +16,14 @@ def mul32{a:T,b:T==[ 4]u64} = emit{T, '_mm256_mul_epu32', a, b} # reads only l
|
||||
|
||||
# structural operations
|
||||
|
||||
def shl{([16]u8), x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
def shr{([16]u8), x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||
def blend{([8]u16), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[16]i16~~a, [16]i16~~b, m}
|
||||
def blend{([8]u32), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 8]i32~~a, [ 8]i32~~b, m}
|
||||
def blend{([4]u64), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 4]f64~~a, [ 4]f64~~b, m}
|
||||
|
||||
def blend{([8]u16), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}
|
||||
def blend{([8]u32), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
|
||||
def blend{([4]u64), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}
|
||||
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = blend_top{f, t, m}
|
||||
|
||||
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
def homBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M if w256{T, 16} and w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m}
|
||||
|
||||
def shuf{L, x:T, n if lvec{L,4,32} and w256{T} and knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
|
||||
def shuf{L, x:T, n if lvec{L,4,64} and w256{T} and knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}
|
||||
def shufHalves{x:T, y:T, n if w256{T} and knum{n}} = T ~~ emit{[4]i64, '_mm256_permute2x128_si256', v2i{x}, v2i{y}, n}
|
||||
|
||||
def sel{L, x:T, i:I if w256{T} and lvec{L,8,32} and w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}
|
||||
def sel{L, x:T, i:I if w256{T} and lvec{L,16,8} and w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}
|
||||
def sel{L, x:T, i:I if w256{T} and lvec{L,8,32} and w256{I,32}} = vec_shuffle{L, x, i}
|
||||
def sel{L, x:T, i:I if w256{T} and lvec{L,16,8} and w256{I, 8}} = vec_shuffle{L, x, i}
|
||||
|
||||
# masked store; F variants may not be a single instruction
|
||||
def topMaskStore{p:*T, m:M, v:T if w256i{M, 32} and w256{T,elwidth{M}}} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
|
||||
@ -44,7 +32,7 @@ def homMaskStore{p:*T, m:M, v:T if w256i{M} and w256{T,elwidth{M}}} = topMas
|
||||
|
||||
def topMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v}
|
||||
def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v}
|
||||
def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}}
|
||||
|
||||
# mask stuff
|
||||
def topMask{x:T if w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
|
||||
@ -57,7 +45,7 @@ def homAny{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
|
||||
def homAll{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
|
||||
def topAny{x:T if w256i{T}} = topMask{x} != 0
|
||||
def topAll{x:T=[k]_ if w256i{T}} = topMask{x} == (1<<k)-1
|
||||
def homMask{a:T, b:T if w256i{T,16}} = homMask{shuf{[4]u64, packQ{ty_s{a},ty_s{b}}, 4b3120}}
|
||||
def homMask{a:T, b:T if w256i{T,16}} = homMask{vec_shuffle{[4]u64, packQ{ty_s{a},ty_s{b}}, 0,2,1,3}}
|
||||
|
||||
def topAny{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
|
||||
def topAny{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
|
||||
@ -80,8 +68,8 @@ def narrow{T, x:X if w256i{X,32} and width{T}==8} = {
|
||||
b:= packQ{a, a}
|
||||
re_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}}
|
||||
}
|
||||
def narrow{T, x:X if w256i{X,32} and width{T}==16} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}}
|
||||
def narrow{T, x:X if w256i{X,16} and width{T}== 8} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}}
|
||||
def narrow{T, x:X if w256i{X,32} and width{T}==16} = re_el{T, vec_shuffle{[4]u64, packQ{x, x}, 0,2,1,3}}
|
||||
def narrow{T, x:X if w256i{X,16} and width{T}== 8} = re_el{T, vec_shuffle{[4]u64, packQ{x, x}, 0,2,1,3}}
|
||||
|
||||
def narrow{T, x:X if w256f{X,64} and T<i32} = narrow{T, narrow{i32, x}}
|
||||
def narrow{T, x:X if w256f{X,64} and T==i32} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
|
||||
@ -109,4 +97,4 @@ def unzip128{a:T, b:T==[8](u32), k} = shufInd{a, b, tup{0,2,8,10, 4,6,12,14}+k}
|
||||
def unzip128{a:T, b:T==[4](u64), k} = shufInd{a, b, tup{0,4,2,6}+k}
|
||||
|
||||
def unzip{a:T, b:T, k if w128u{T} and hasarch{'SSE2'}} = unzip128{a, b, k}
|
||||
def unzip{a:T, b:T, k if w256u{T} and hasarch{'AVX2'}} = shuf{[4]u64, unzip128{a, b, k}, 4b3120}
|
||||
def unzip{a:T, b:T, k if w256u{T} and hasarch{'AVX2'}} = vec_shuffle{[4]u64, unzip128{a, b, k}, 0,2,1,3}
|
||||
|
||||
@ -175,14 +175,17 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
|
||||
|
||||
# base cases
|
||||
def {
|
||||
absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half,
|
||||
homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
|
||||
homAll,homAny,bitAll,bitAny,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
|
||||
pext,popcRand,rbit,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,shufInd,storeLow,
|
||||
topBlend,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,widen,widenUpper,
|
||||
multishift,vshl,
|
||||
pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
|
||||
topBlend,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,
|
||||
multishift,
|
||||
}
|
||||
|
||||
def bitBlend{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m))
|
||||
def blend_hom{f:T, t:T, m:M} = bitBlend{f, t, m} # extended by arch/*/select
|
||||
|
||||
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
|
||||
def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
|
||||
def homMask{...vs if length{vs}>1} = {
|
||||
@ -199,6 +202,7 @@ def homMask{x if ktup{x}} = homMask{...x}
|
||||
|
||||
if_inline (hasarch{'X86_64'}) {
|
||||
include 'arch/iintrinsic/basic'
|
||||
include 'arch/iintrinsic/select'
|
||||
include './sse2'
|
||||
include './sse'
|
||||
include './avx'
|
||||
@ -206,14 +210,17 @@ if_inline (hasarch{'X86_64'}) {
|
||||
include './avx512'
|
||||
} else if_inline (hasarch{'AARCH64'}) {
|
||||
include 'arch/neon_intrin/basic'
|
||||
include 'arch/neon_intrin/select'
|
||||
include './neon'
|
||||
def {vec_shuffle16_lo}
|
||||
} else {
|
||||
def {__adds,__subs,__sqrt,vec_broadcast,vec_make}
|
||||
def {
|
||||
__adds,__subs,__sqrt,vec_broadcast,vec_make,
|
||||
vec_shift_left_128,vec_shift_right_128,vec_merge_shift_right,
|
||||
vec_shuffle16_lo
|
||||
}
|
||||
}
|
||||
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
|
||||
def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k}
|
||||
def mzip128{a:T, b:T, k} = el_m{T} ~~ zip128{a, b, k}
|
||||
def pack {a:T, b:T, k} = unzip{el_s{T}~~a, el_s{T}~~b, k}
|
||||
@ -236,6 +243,9 @@ def broadcast{V=[_]T, v} = vec_broadcast{V, promote{T,v}}
|
||||
def make{V=[_]_, ...xs} = vec_make{V, ...xs}
|
||||
def iota{V=[k]_} = make{V, ...iota{k}}
|
||||
def absu{a:[_]_} = ty_u{abs{a}}
|
||||
def shuf = vec_shuffle
|
||||
def vshl = vec_merge_shift_right
|
||||
def homBlend = blend_hom
|
||||
|
||||
def floor = __floor
|
||||
def ceil = __ceil
|
||||
|
||||
@ -19,6 +19,9 @@ def for_vec_overlap{vl}{vars,begin==0,n,iter} = {
|
||||
# Shift as u16, since x86 is missing 8-bit shifts
|
||||
def shr16{v:V, n} = V~~(re_el{u16, v} >> n)
|
||||
|
||||
# Copy half of a vector to the other half
|
||||
def dup_half{vec,half} = shuf{[4]u64, vec, 2*half+tup{0,1,0,1}}
|
||||
|
||||
# Forward or backwards in-place max-scan
|
||||
# Assumes a whole number of vectors and minimum 0
|
||||
include './scan_common'
|
||||
@ -41,24 +44,24 @@ fn max_scan{T, up}(x:*T, len:u64) : void = {
|
||||
def getsel{...x} = assert{'shuffling not supported', show{...x}}
|
||||
if_inline (hasarch{'AVX2'}) {
|
||||
def getsel{h:H=[16]T if width{T}==8} = {
|
||||
sel{H, pair{h,h}, .}
|
||||
shuf{H, pair{h,h}, .}
|
||||
}
|
||||
def getsel{v:V=[vl==32]T if width{T}==8} = {
|
||||
def H = n_h{V}
|
||||
vtop := V**(vl/2)
|
||||
hs := each{shuf{[4]u64, v, .}, tup{4b3232, 4b1010}}
|
||||
{i} => homBlend{...each{sel{H,.,i}, hs}, V~~i<vtop}
|
||||
hs := each{dup_half{v, .}, tup{1,0}}
|
||||
{i} => homBlend{...each{shuf{H,.,i}, hs}, V~~i<vtop}
|
||||
}
|
||||
def getsel{v:V=[8]T if width{T}==32} = sel{V, v, .}
|
||||
def getsel{v:V=[8]T if width{T}==32} = shuf{V, v, .}
|
||||
}
|
||||
|
||||
# Move evens to half 0 and odds to half 1
|
||||
def uninterleave{x:V=[vl]T if hasarch{'AVX2'}} = {
|
||||
def bytes = width{T}/8
|
||||
def i = 2*iota{vl/4}
|
||||
def i2= join{table{+, bytes*merge{i,i+1}, iota{bytes}}}
|
||||
t := V~~sel{[16]u8, re_el{u8,x}, make{[32]u8, merge{i2,i2}}}
|
||||
shuf{[4]u64, t, 4b3120}
|
||||
def i2= flat_table{+, bytes*merge{i,i+1}, iota{bytes}}
|
||||
t := shuf{[16]u8, x, i2}
|
||||
shuf{[4]u64, t, 0,2,1,3}
|
||||
}
|
||||
|
||||
def rtypes = tup{i8, i16, i32, f64}
|
||||
@ -153,9 +156,9 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
|
||||
@for (tui over promote{u64,nu}) { i = load{t, load{w, i}}; tui = i }
|
||||
def tv = load{*V~~tui, .}
|
||||
ui = tv{0}
|
||||
if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232}
|
||||
ui = shuf{[4]u64, ui, 4b1010}
|
||||
if (nu > vl) ui2 = each{shuf{[4]u64, tv{1}, .}, tup{4b1010, 4b3232}}
|
||||
if (nu > 16) ui1 = dup_half{ui, 1}
|
||||
ui = dup_half{ui, 0}
|
||||
if (nu > vl) ui2 = each{dup_half{tv{1}, .}, tup{0,1}}
|
||||
}
|
||||
# Popcount on 8-bit values
|
||||
def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} }
|
||||
@ -166,7 +169,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
|
||||
s{shr16{v,4}} + s{v}
|
||||
}
|
||||
# Bit table
|
||||
def swap{v} = shuf{[4]u64, v, 4b1032} # For signedness
|
||||
def swap{v} = shuf{[4]u64, v, 2,3,0,1} # For signedness
|
||||
def sel_b = getsel{swap{vb}}
|
||||
# Masks for filtering bit table
|
||||
def ms = if (up) 256-(1<<(1+iota{8})) else (1<<iota{8})-1
|
||||
@ -184,7 +187,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
|
||||
ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}}
|
||||
if (dup) {
|
||||
i0 := V~~ind # Can contain -1
|
||||
def isel{u} = sel{H, u, i0}
|
||||
def isel{u} = shuf{H, u, i0}
|
||||
ind = isel{ui}
|
||||
if (nu > 16) {
|
||||
b := V~~(re_el{u16, i0} << (7 - lb{vl/2}))
|
||||
@ -227,7 +230,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
|
||||
if (ex>=1 and wn >= svl) {
|
||||
--gap # Allows subtracting < instead of adding <=
|
||||
def un = uninterleave
|
||||
def tr_half{a, b} = each{shufHalves{a,b,.}, tup{16b20, 16b31}}
|
||||
def tr_half{a, b} = each{vec_select{128,tup{a,b},.}, tup{tup{0,2}, tup{1,3}}}
|
||||
def un{{a,b}} = tr_half{un{a},un{b}}
|
||||
if (not lanes) select{wv,1} = load{wg, 1}
|
||||
wv = un{wv}
|
||||
|
||||
@ -56,7 +56,7 @@ local def perm_split{x:X, lanes, chunks} = {
|
||||
def si0 = join{each{index_of, u, gr}*b + lanes%b}
|
||||
def si = tern{join{lanes>=0}, si0, 0xff}
|
||||
# def si = join{each{{u,g,l} => l>=0) index_of{u,g}*b + lanes%b else 0xff, u, gr, lanes}}
|
||||
sel{[16]u8,
|
||||
shuf{[16]u8,
|
||||
new_shuf{re_el{ty_u{b*8},X}, x, join{each{shiftright{.,chunks**0}, u}}},
|
||||
make{X, si}
|
||||
}
|
||||
@ -70,18 +70,18 @@ local def perm_shufb{x, is} = {
|
||||
}
|
||||
# TODO accept narrower inputs
|
||||
def shuf_impl{256, 256, data if hasarch{'AVX2'} and not same{perm_shufb{...data{u8}},'!'}} = perm_shufb{...data{u8}}
|
||||
def shuf_impl{256, 128, data if hasarch{'AVX2'} and data{'h',u8}} = { def {x,is} = data{u8}; sel{[16]u8, pair{x, x}, make{[32]i8, is}} }
|
||||
def shuf_impl{256, 128, data if hasarch{'AVX2'} and data{'h',u8}} = { def {x,is} = data{u8}; shuf{[16]u8, pair{x, x}, is} }
|
||||
|
||||
def shuf_impl{512, 512, data if hasarch{'AVX512F'} and data{'h','nz',u8}} = { def {x,is} = data{u8}; sel{[64]u8, x, make{[64]u8, is}} }
|
||||
|
||||
def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u32}} = { def {x,is} = data{'nz',u32}; sel{[8]u32, x, make{[8]u32, is}} }
|
||||
def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u64}} = { def {x,is} = data{'nz',u64}; shuf_imm{[4]u64, x, base{4,is}} }
|
||||
def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u32}} = { def {x,is} = data{'nz',u32}; shuf{[8]u32, x, is} }
|
||||
def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u64}} = { def {x,is} = data{'nz',u64}; shuf_imm{[4]u64, x, is} }
|
||||
|
||||
def in_chunks{c, is} = { def ls = split{c, is}; all{{l, s} => all{(l<0) | ((l>=s) & (l<s+c))}, ls, inds{ls}*c} }
|
||||
def shuf_impl{256, 256, data if hasarch{'AVX2'} and in_chunks{16,select{data{u8},1}}} = { def {x,is} = data{u8}; sel{[16]u8, x, make{[32]u8, is}} }
|
||||
def shuf_impl{128, 128, data if hasarch{'SSSE3'}} = { def {x,is} = data{u8}; sel{[16]u8, x, make{[16]i8, is}} }
|
||||
def shuf_impl{256, 256, data if hasarch{'AVX2'} and in_chunks{16,select{data{u8},1}}} = { def {x,is} = data{u8}; shuf{[16]u8, x, is} }
|
||||
def shuf_impl{128, 128, data if hasarch{'SSSE3'}} = { def {x,is} = data{u8}; shuf{[16]u8, x, is} }
|
||||
|
||||
def shuf_impl{128, 128, data if hasarch{'AARCH64'}} = { def {x,is} = data{u8}; sel{[16]u8, x, make{[16]i8, is}} }
|
||||
def shuf_impl{128, 128, data if hasarch{'AARCH64'}} = { def {x,is} = data{u8}; shuf{[16]u8, x, is} }
|
||||
|
||||
|
||||
|
||||
@ -130,9 +130,9 @@ def bitalign{s, 8 if s<7, a:V=[k](u8) if hasarch{'X86_64'}} = {
|
||||
def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = {
|
||||
def V16 = re_el{u16,V}
|
||||
|
||||
def b = new_shuf{a, range{k} - (((range{k}+2)/8)>>0)}
|
||||
def c = blend{[8]u16, b, b <<{u64} 4, 2b01100110}
|
||||
def d = (V16~~c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang
|
||||
def b = V16~~new_shuf{a, range{k} - (((range{k}+2)/8)>>0)}
|
||||
def c = blend_units{b, b <<{u64} 4, tup{0,1,1,0}}
|
||||
def d = (c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang
|
||||
homBlend{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f
|
||||
}
|
||||
|
||||
|
||||
@ -63,7 +63,7 @@ def spreadBits{T==[32]u8, a:(u32)} = {
|
||||
def idxs = iota{32}
|
||||
b:= [8]u32**a
|
||||
c:= [32]u8~~b
|
||||
d:= sel{[16]u8, c, make{[32]i8, idxs>>3 + bit{4, idxs}}}
|
||||
d:= shuf{[16]u8, c, make{[32]i8, idxs>>3 + bit{4, idxs}}}
|
||||
e:= make{[32]u8, 1<<tail{3, idxs}}
|
||||
e == (d&e)
|
||||
}
|
||||
@ -74,7 +74,7 @@ def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = {
|
||||
}
|
||||
def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = {
|
||||
b:= T~~[8]u16**a
|
||||
exp:= T~~shuf{[4]i32, shuf16Lo{mzip{b,b,0}, 4b1100}, 4b1100}
|
||||
exp:= T~~shuf{[4]i32, vec_shuffle16_lo{mzip{b,b,0}, tup{0,0,1,1}}, 0,0,1,1}
|
||||
(exp & make{T, 1<<(iota{16}&7)}) != T**0
|
||||
}
|
||||
|
||||
|
||||
@ -4,10 +4,9 @@ if_inline (hasarch{'BMI2'}) include './bmi2'
|
||||
include './spaced'
|
||||
include './scan_common'
|
||||
|
||||
def opsh64{op}{v:([4]f64), perm} = op{v, shuf{[4]u64, v, perm}}
|
||||
def opsh32{op}{v:([2]f64), perm} = op{v, shuf{[4]u32, v, perm}}
|
||||
def mix{op, v:([4]f64) if hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} }
|
||||
def mix{op, v:([2]f64) if hasarch{'X86_64'}} = opsh32{op}{v, 4b1032}
|
||||
def opsh64{op}{v:[_](f64), ...perm} = op{v, shuf{v, ...perm}}
|
||||
def mix{op, v:([4]f64) if hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 1,0}, 2,3,0,1} }
|
||||
def mix{op, v:([2]f64) if hasarch{'X86_64'}} = opsh64{op}{v, 1,0}
|
||||
|
||||
def reduce_pairwise{op, plog, x:*T, len, init:T} = {
|
||||
# Pairwise combination to shorten dependency chains
|
||||
@ -41,8 +40,8 @@ fn fold_idem{T==f64, op if has_simd}(x:*T, len:u64) : T = {
|
||||
if (bulk>2) {
|
||||
assert{(bulk==4) & hasarch{'AVX'}}
|
||||
if (len > 1) {
|
||||
if (len > 2) r = opsh64{op}{r, 4b2222}
|
||||
r = opsh64{op}{r, 4b1111}
|
||||
if (len > 2) r = opsh64{op}{r, 4**2}
|
||||
r = opsh64{op}{r, 4**1}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@ -40,7 +40,7 @@ def unzip_load{E, n, TG if width{E}*n <= arch_defvw} = each{tup, unzip{TG{[n]E,
|
||||
|
||||
def widen_tup{u32, is:([16]u8)} = tup{ # compiler will deduplicate all the repeated calls of this on the same is
|
||||
widen{[8]u32, is},
|
||||
widen{[8]u32, shuf{[4]u32, is, 4b3232}},
|
||||
widen{[8]u32, shuf{[2]u64, is, 1,1}},
|
||||
}
|
||||
|
||||
def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
|
||||
@ -86,9 +86,9 @@ def raw_widen_inds{[k]D, x:[k0]S if k0>=k} = { # : [k*sc]S
|
||||
def add = make{RV, range{k*sc} % sc}
|
||||
if (hasarch{'AARCH64'} and [k]D == [2]u64 and S==u8) {
|
||||
def x16 = undefPromote{[16]u8, x}
|
||||
(RV~~sel{[16]u8, x16, make{[16]u8, range{16}>>3}}<<3) + add
|
||||
(RV~~shuf{[16]u8, x16, range{16}>>3}<<3) + add
|
||||
} else if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) {
|
||||
(sel{[8]u32, undefPromote{[8]u32, x}, make{[8]u32, range{8}>>1}}<<sc) + add
|
||||
(shuf{[8]u32, undefPromote{[8]u32, x}, range{8}>>1}<<sc) + add
|
||||
} else {
|
||||
def wd = widen{[k]D, x}
|
||||
re_el{S, wd * [k]D**base{1<<width{S}, sc**sc}} + add
|
||||
@ -122,11 +122,11 @@ def zip_halves{mode, E, nt, ni} = match(lut_gen{mode, w_h{E}, nt, ni}) { # e.g.
|
||||
run_zip{mzip, is}
|
||||
}
|
||||
def me{is:([ni]u8) if hasarch{'X86_64'} and E==u64 and ni==16} = {
|
||||
def is2 = sel{[16]u8, is, make{[16]u8, 0,1,4,5,2,3,6,7,8,9,12,13,10,11,14,15}}
|
||||
def is2 = shuf{u8, is, tr_iota{0,2,1,3}}
|
||||
run_zip{mzip128, is2}
|
||||
}
|
||||
def me{is:([ni]u8) if hasarch{'X86_64'} and E==u16 and ni==32} = {
|
||||
def is2 = shuf{[4]u64, is, 4b3120}
|
||||
def is2 = shuf{u64, is, tr_iota{1,0}}
|
||||
run_zip{mzip128, is2}
|
||||
}
|
||||
}}}
|
||||
@ -149,13 +149,13 @@ def lut_gen{'c', E==u8, nt, ni if hasarch{'AVX2'} and nt<=64 and ni<=32} = { de
|
||||
def top = [32]u8**128
|
||||
def isp = each{{i} => is | (top &~ tree_fold{&, each{{m, o} => if (bit{o,i}!=0) m else ~m, bits, bi}}), range{vn}}
|
||||
|
||||
tup{tree_fold{|, each{sel{[16]u8,.,.}, luts, isp}}}
|
||||
tup{tree_fold{|, each{shuf{[16]u8,.,.}, luts, isp}}}
|
||||
}
|
||||
}}}}
|
||||
|
||||
def lut_gen{mode, E==u8, nt, ni if hasarch{'AVX2'} and nt<=16 and ni<=32} = tup{16, 32, loader{{TG} => {
|
||||
lut:[32]u8 = [32]u8**TG{[16]u8, 0}
|
||||
{is:([32]u8)} => tup{sel{[16]u8, lut, is}}
|
||||
{is:([32]u8)} => tup{shuf{[16]u8, lut, is}}
|
||||
}}}
|
||||
|
||||
def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=32 and ni<=16} = blend_halves{mode, E, 32, 16}
|
||||
@ -163,7 +163,7 @@ def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=16 and ni<=16} = ble
|
||||
|
||||
def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=8 and ni<=16} = tup{8, 16, loader{{TG} => {
|
||||
def lut = TG{[8]u32, 0}
|
||||
def me{'idxs', is:([16]u8)} = each{{wis} => tup{wis, sel{[8]u32, lut, wis}}, widen_tup{u32,is}} # TODO inline, or properly outline
|
||||
def me{'idxs', is:([16]u8)} = each{{wis} => tup{wis, shuf{[8]u32, lut, wis}}, widen_tup{u32,is}} # TODO inline, or properly outline
|
||||
def me{is:([16]u8)} = each{{{_,v}}=>v, me{'idxs', is}}
|
||||
}}}
|
||||
|
||||
@ -191,8 +191,8 @@ def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='c' and E>=u16} = 0
|
||||
def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='i' and E==u64 and nt>16} = 0
|
||||
|
||||
|
||||
def lut16{tab:([16]u8), idxs:([16]u8)} = sel{[16]u8, tab, idxs}
|
||||
def lut16{tab:([16]u8), idxs:([32]u8) if hasarch{'X86_64'}} = sel{[16]u8, pair{tab, tab}, idxs}
|
||||
def lut16{tab:([16]u8), idxs:([16]u8)} = shuf{[16]u8, tab, idxs}
|
||||
def lut16{tab:([16]u8), idxs:([32]u8) if hasarch{'X86_64'}} = shuf{[16]u8, pair{tab, tab}, idxs}
|
||||
|
||||
def shuf_u8bits{inds:*u8, ni} = 0
|
||||
def shuf_u8bits{inds:*u8, ni if has_sel} = {
|
||||
@ -215,4 +215,4 @@ def shuf_u8bits{inds:*u8, ni if has_sel} = {
|
||||
def hi = lut16{v1, x >> 4}
|
||||
lo | hi
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -31,8 +31,7 @@ def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, n
|
||||
def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
|
||||
def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
|
||||
|
||||
def bitBlend{f:T, t:T, m:M if nvec{T} and nvecu{M,elwidth{T}} and width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
|
||||
def homBlend{f:T, t:T, m:M if nvec{M}} = bitBlend{f, t, m}
|
||||
def bitBlend{f:T=[_]_, t:T, m:M=[_]_} = blend_bit{f, t, m}
|
||||
|
||||
def addpw { x:T if nveci{T} and elwidth{T}<=32} = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
|
||||
def addpwa{a:D==el_m{T}, x:T if nveci{T} and elwidth{T}<=32} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
|
||||
@ -40,9 +39,7 @@ def mla{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y
|
||||
def mls{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y
|
||||
def rbit{x:T if nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x}
|
||||
def rev{(width{T}), x:[_]T} = x
|
||||
def rev{16, x:T if nveci{T} and elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window
|
||||
def rev{32, x:T if nveci{T} and elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x}
|
||||
def rev{64, x:T if nveci{T} and elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x}
|
||||
def rev{w, x:[k]T if w > width{T}} = reverse_units{w/width{T}, x} # reverse the order of elements in each w-bit window
|
||||
def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x}
|
||||
def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x}
|
||||
def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x}
|
||||
@ -74,10 +71,6 @@ def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b}
|
||||
def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w64{D} and nvec{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
|
||||
def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w128{D} and nvec{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
|
||||
def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
|
||||
def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
|
||||
|
||||
def zip{a:T, b:T, 0 if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
||||
def zip{a:T, b:T, 1 if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
||||
|
||||
def unzip{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y}
|
||||
def unzip{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y}
|
||||
@ -87,12 +80,8 @@ def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 1+2*range{vcount{T}}}}
|
||||
def trn{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y}
|
||||
def trn{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y}
|
||||
|
||||
def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, re_el{eltype{I},x}, ty_u{i}}}
|
||||
def sel{{x0:T}, i:I} = sel{T, x0, i}
|
||||
def sel{{...xs}, i:I if length{xs}>=2 and length{xs}<=4 and allSame{each{type,xs}} and lvec{oneType{xs},16,8} and nvec{I, 8}} = {
|
||||
def T = oneType{xs}
|
||||
re_el{eltype{T}, emit{I, ntyp{merge{'unpacked_vqtbl',fmtnat{length{xs}}}, I}, ...each{reinterpret{[16]u8,.}, xs}, ty_u{i}}}
|
||||
}
|
||||
def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = vec_select{eltype{L}, x, i}
|
||||
def sel{{...xs}, i:I if length{xs}>=1 and length{xs}<=4 and allSame{each{type,xs}} and lvec{oneType{xs},16,8} and nvec{I, 8}} = vec_select{xs, i}
|
||||
|
||||
|
||||
|
||||
@ -132,7 +121,7 @@ def homMask{x:T=[k]E if nvecu{T} and width{E}>=k} = {
|
||||
truncBits{k, fold_add{x & make{T, 1<<iota{k}}}}
|
||||
}
|
||||
def homMask{x:T=[16]E if width{E}==8} = {
|
||||
t:= [8]u16~~sel{[16]u8, x, make{[16]u8, 0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15}}
|
||||
t:= [8]u16~~sel{[16]u8, x, make{[16]u8, tr_iota{3,0,1,2}}}
|
||||
fold_add{t & make{[8]u16, (1<<iota{8})*0x0101}}
|
||||
}
|
||||
def homMask{a:T,b:T=[16]E if width{E}==8} = {
|
||||
|
||||
@ -103,7 +103,7 @@ rcsh4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}}
|
||||
|
||||
def read_shuf_vecs{l, ellw:(u64), shp:*V} = { # tuple of byte selectors in 1<<ellw
|
||||
def double{x:X if hasarch{'AVX2'}} = {
|
||||
s:=shuf{[4]u64, x, 4b3120}; s+=s
|
||||
s:=shuf{u64, x, 0,2,1,3}; s+=s
|
||||
r:=each{bind{~~,[32]i8},mzip128{s, s + X**1}}
|
||||
r
|
||||
}
|
||||
@ -150,19 +150,19 @@ if_inline (hasarch{'AVX2'}) {
|
||||
def rep_iter_from_sh{sh}{x, gen} = {
|
||||
def l = length{sh}
|
||||
def h = l>>1
|
||||
def fs{v, s} = gen{sel{[16]i8, v, s}}
|
||||
a := shuf{[4]u64, x, 4b1010}; each{fs{a,.}, slice{sh,0,h}}
|
||||
def fs{v, s} = gen{shuf{[16]i8, v, s}}
|
||||
a := shuf{u64, x, 0,1,0,1}; each{fs{a,.}, slice{sh,0,h}}
|
||||
if (l%2) fs{x, select{sh, h}}
|
||||
b := shuf{[4]u64, x, 4b3232}; each{fs{b,.}, slice{sh,-h}}
|
||||
b := shuf{u64, x, 2,3,2,3}; each{fs{b,.}, slice{sh,-h}}
|
||||
}
|
||||
|
||||
def get_rep_iter{V, wv==2}{x, gen} = {
|
||||
def s = shuf{[4]u64, x, 4b3120}
|
||||
def s = shuf{u64, x, 0,2,1,3}
|
||||
each{{q}=>gen{V~~q}, mzip128{s, s}}
|
||||
}
|
||||
def get_rep_iter{V==[4]u64, wv} = {
|
||||
def step = 4
|
||||
def sh = each{base{4,.}, get_shufs{step, wv, wv}}
|
||||
def sh = get_shufs{step, wv, wv}
|
||||
{x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh}
|
||||
}
|
||||
|
||||
@ -171,7 +171,7 @@ if_inline (hasarch{'AVX2'}) {
|
||||
} else { # has_bytesel_128
|
||||
|
||||
def rep_iter_from_sh{sh}{x, gen} = {
|
||||
each{{s} => gen{sel{[16]u8, x, s}}, sh}
|
||||
each{{s} => gen{shuf{[16]u8, x, s}}, sh}
|
||||
}
|
||||
|
||||
def rep_const_shuffle{wv==2, xv0:*V=[_]T, rv0:*V, n:(u64)} = {
|
||||
@ -191,9 +191,9 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
def [step]_ = V # Bytes written
|
||||
def wvb = wv << ellw
|
||||
def hs = (h*step) / wvb # Actual step size in argument elements
|
||||
def shufbase{i if hasarch{'AVX2'}} = shuf{[4]u64, load{*V~~(x+i)}, 4b1010}
|
||||
def shufbase{i if hasarch{'AVX2'}} = shuf{u64, load{*V~~(x+i)}, 0,1,0,1}
|
||||
def shufbase{i if has_bytesel_128} = load{*V~~(x+i)}
|
||||
def shufrun{a, s} = sel{[16]i8, a, s} # happens to be the same across AVX2 & NEON
|
||||
def shufrun{a, s} = shuf{[16]i8, a, s} # happens to be the same across AVX2 & NEON
|
||||
|
||||
i:u64 = 0
|
||||
re := r + n*wvb - h*step
|
||||
@ -442,18 +442,18 @@ def rep_const_bool_div8{wv, x, r, nw if has_simd} = {
|
||||
run8{{x} => z{z{z{x}}}}
|
||||
}
|
||||
} else { # any_sel
|
||||
def selH = sel{[16]u8, ., .}
|
||||
def selH = shuf{[16]u8, ., .}
|
||||
def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .}
|
||||
def id{xv} = xv
|
||||
if (wv == 2) {
|
||||
def init = if (avx2) shuf{[4]u64, ., 4b3120} else id
|
||||
def init = if (avx2) shuf{u64, ., 0,2,1,3} else id
|
||||
# Expander for half byte
|
||||
def tabr = makeTab{tr_iota{2*iota{4}} * 2b11}
|
||||
m4 := V**0xf
|
||||
run24{*V~~x, init, {x} => tabr{x & m4}, 4}
|
||||
} else if (wv == 4) {
|
||||
# Unzip 32-bit elements (result lanes) across AVX2 lanes
|
||||
def pre = if (avx2) sel{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id
|
||||
def pre = if (avx2) shuf{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id
|
||||
def init{xv} = { u:=pre{xv}; zip128{u,u,0} }
|
||||
# Expander for two bits in either bottom or next-to-bottom position
|
||||
def tabr = makeTab{tr_iota{0,4,0,4} * 2b1111}
|
||||
@ -467,17 +467,6 @@ def rep_const_bool_div8{wv, x, r, nw if has_simd} = {
|
||||
flush{}
|
||||
}
|
||||
|
||||
def sel_imm{V=[4]T, x, {...inds} if hasarch{'SSE2'} and (T==u32 or T==u64)} = {
|
||||
assert{length{inds} == 4}
|
||||
shuf{V, x, base{4, inds}}
|
||||
}
|
||||
def sel_imm{V=([16]u8), x:X, {...inds}} = {
|
||||
def I = re_el{u8,X}; def [n]_ = I
|
||||
def l = length{inds}
|
||||
assert{l==16 or l==n}
|
||||
sel{V, x, make{I, cycle{n, inds}}}
|
||||
}
|
||||
|
||||
# Data for the permutation that sends bit i to k*i % width{T}
|
||||
def modperm_dat{T, k} = {
|
||||
def w = width{T}
|
||||
@ -501,17 +490,15 @@ def modperm_shuf_step{x:V=[_]T, l, m if l%8==0} = {
|
||||
(x &~ m) | (swap_elts{x, l/8} & m)
|
||||
}
|
||||
# Reverse each pair of elements
|
||||
def swap_elts{x:V=[_]_, el_bytes} = {
|
||||
def selx{n, wd} = {
|
||||
def swap_elts{x:V=[k]_, el_bytes} = {
|
||||
if (hasarch{'AARCH64'}) {
|
||||
if (el_bytes == 8) vshl{x, x, k/2}
|
||||
else V~~rev{el_bytes*16, re_el{ty_u{8*el_bytes}, V} ~~ x}
|
||||
} else if (any_sel or (hasarch{'SSE2'} and el_bytes >= 2)) {
|
||||
def wd = min{el_bytes, 8}
|
||||
def l = el_bytes/wd
|
||||
def i = iota{n}
|
||||
def rev = i + (l - 2*(i&l))
|
||||
sel_imm{[n]ty_u{wd*8}, x, rev}
|
||||
}
|
||||
if (hasarch{'SSE2'} and el_bytes >= 4) {
|
||||
selx{4, max{4, el_bytes/2}}
|
||||
} else if (any_sel) {
|
||||
selx{16, 1}
|
||||
def i = iota{l}
|
||||
shuf{ty_u{wd*8}, x, merge{i+l, i}}
|
||||
} else {
|
||||
def sh = el_bytes*8
|
||||
xe := re_el{ty_u{2*sh}, V} ~~ x
|
||||
@ -526,7 +513,7 @@ def extract_modperm_mask{full:W, lane, l} = {
|
||||
def n = (if (f) 256 else 128) / w # and number of elements
|
||||
def o = l / w # Extract [o,2*o)
|
||||
def i = o + iota{n}%o
|
||||
sel_imm{[n]ty_u{w}, if (f) full else lane, i}
|
||||
shuf{ty_u{w}, if (f) full else lane, i}
|
||||
}
|
||||
def proc_mod_dat{swap_data:W} = {
|
||||
def ww = width{W}; def avx2 = ww==256
|
||||
@ -534,7 +521,7 @@ def proc_mod_dat{swap_data:W} = {
|
||||
def lens = reverse{1 << slice{iota{lb{hi}}, lb{lo}}}
|
||||
fold{{x,s}=>s{x}, ., each{get_swap, lens}}
|
||||
}
|
||||
swap_lane := if (avx2) shuf{W, swap_data, 4b1010} else swap_data
|
||||
swap_lane := if (avx2) shuf{swap_data, 0,1,0,1} else swap_data
|
||||
# Transform width-w units with shifts only
|
||||
def get_shiftperm{data, w} = {
|
||||
dat := data
|
||||
@ -551,20 +538,20 @@ def proc_mod_dat{swap_data:W} = {
|
||||
# Within-byte transformation
|
||||
def get_byteperm{} = {
|
||||
def V = re_el{u8, W}
|
||||
def sw_bytes = sel{[16]u8, swap_lane, V**0}
|
||||
def sw_bytes = shuf{u8, swap_lane, 16**0}
|
||||
m4 := W~~V**0xf
|
||||
t0 := fold{{v,a}=>modperm_shift_step{v,...a}, W~~make{V,iota{vcount{V}}%16}, tup{
|
||||
tup{4, sw_bytes &~ m4},
|
||||
tup{2, ({v} => v | v<<4){sw_bytes & W~~V**0xc}}
|
||||
}}
|
||||
t4 := (t0&m4)<<4 | (t0&~m4)>>4
|
||||
def selI{v,i} = sel{[16]u8, v, V~~i}
|
||||
def selI{v,i} = shuf{[16]u8, v, V~~i}
|
||||
{xv} => selI{t0, xv & m4} | selI{t4, (xv>>4) & m4}
|
||||
}
|
||||
def {partwidth, partperm} = {
|
||||
def sh{w, d} = tup{w, get_shiftperm{d, w}}
|
||||
if (ww==64) sh{64, swap_data}
|
||||
else if (not any_sel) sh{32, shuf{[4]u32, swap_data, 4b0000}}
|
||||
else if (not any_sel) sh{32, shuf{u32, swap_data, 0,0,0,0}}
|
||||
else tup{8, get_byteperm{}}
|
||||
}
|
||||
# Fill in higher steps
|
||||
@ -647,7 +634,7 @@ def rep_const_bool_odd_mask4{
|
||||
def sub_carry{a, c} = match (M) {
|
||||
{[l](u64)} => {
|
||||
ca := if (hasarch{'SSE4.2'} or hasarch{'AARCH64'}) { def S = [l]i64; S~~c > S**0 }
|
||||
else { def S = [2*l]i32; cm := S~~c; cm != shuf{S, cm, 4b2301} }
|
||||
else { cm := [2*l]i32~~c; cm != shuf{cm, 1,0} }
|
||||
a + M~~ca
|
||||
}
|
||||
{_} => a - promote{u64, c != 0}
|
||||
@ -656,9 +643,9 @@ def rep_const_bool_odd_mask4{
|
||||
while (1) {
|
||||
x:M = get_modperm_x{}
|
||||
def vrot1 = ifvec{{x} => {
|
||||
if (w256{M}) shuf{M, x, 4b2103}
|
||||
if (w256{M}) shuf{x, 3,0,1,2}
|
||||
else if (any_sel) vshl{x, x, vcount{type{x}}-1}
|
||||
else shuf{[4]u32, x, 4b1032}
|
||||
else shuf{x, 1,0}
|
||||
}}
|
||||
xo := x<<k | vrot1{x>>(64-k)}
|
||||
# Write result word given starting bits
|
||||
@ -712,23 +699,23 @@ def rep_const_bool_small_odd{W=[wl](u64), max_wv, wv, get_perm_x, output} = {
|
||||
def ix = 64*slice{iota{k},1} // k # bits that overhang within a word
|
||||
def ib = ix // ew # byte index
|
||||
def io = ew*ib + k*ix%ew # where they are in xv
|
||||
def wi = split{wl, dup{tup{255, ...ib, 255, ...se+ib}}}
|
||||
def wi = split{wl, dup{tup{-1, ...ib, -1, ...se+ib}}}
|
||||
xo := ov_bytes{(xv & W**fold{|, 1<<io}) >> (ew-k)}
|
||||
# Permute and mask bytes
|
||||
def step{jj, oi, ind, mask} = {
|
||||
def hk = (k-1) / 2
|
||||
def getv = if (not lanes or jj==hk) ({x}=>x)
|
||||
else sel_imm{[4]u64, ., 2*(jj>hk) + iota{4}%2}
|
||||
def selx{x, i} = sel_imm{[128/ew]ty_u{ew}, getv{x}, i}
|
||||
else shuf{[4]u64, ., 2*(jj>hk) + iota{4}%2}
|
||||
def selx{x, i} = shuf{[128/ew]ty_u{ew}, getv{x}, i}
|
||||
b := selx{xv, ind} & make{W, mask}
|
||||
r := (b<<k) - b
|
||||
def selx_nz{x, i} = { def nz = i!=255; selx{x, i * nz} & W~~make{[4]i32, -nz} }
|
||||
o := (if (ew==8) selx else selx_nz){xo, flat_table{max, oi, 255*(0<iota{se})}}
|
||||
def selx_nz{x, i} = { def nz = i != -1; selx{x, i * nz} & W~~make{[4]i32, -nz} }
|
||||
o := (if (ew==8) selx else selx_nz){xo, flat_table{min, oi, 255-256*(0<iota{se})}}
|
||||
output{r|o}
|
||||
}
|
||||
each{step,
|
||||
iota{k}, wi,
|
||||
split{ne, replicate{k, iota{ne}}},
|
||||
split{ne, replicate{k, dup{iota{ne/(1+lanes)}}}},
|
||||
split{wl, each{base{2,.}, split{64, cycle{k*ww, 0==iota{k}}}}}
|
||||
}
|
||||
}
|
||||
|
||||
@ -75,7 +75,7 @@ export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', sca
|
||||
|
||||
# Assumes identity is 0
|
||||
def scan_assoc{op} = {
|
||||
def shl0{v, k} = shl{[16]u8, v, k/8} # Lanewise
|
||||
def shl0{v:[_]T, k} = vec_shift_right_128{v, k/width{T}} # Lanewise
|
||||
def shl0{v:V, k==128 if hasarch{'AVX2'}} = {
|
||||
# Broadcast end of lane 0 to entire lane 1
|
||||
l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v}
|
||||
@ -107,7 +107,7 @@ fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64
|
||||
m := V**mark
|
||||
def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total
|
||||
p := clmul{a, m, i}
|
||||
t := shr{[16]u8, p, 8}
|
||||
t := vec_shift_left_128{p, 1}
|
||||
s := p ^ carry
|
||||
carry = s ^ t
|
||||
s
|
||||
@ -141,7 +141,7 @@ fn scan_neq{if hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}}(init:u64, x:*u64, r:*u
|
||||
xh := exor64{hb} # Exclusive xor of high bits (xh ^ hb for inclusive)
|
||||
xc := xh ^ carry
|
||||
v := x8 ^ V~~maskToHom{[64]u8, [64]u1~~extract{xc,0}}
|
||||
carry = (xc ^ hb) ^ shuf{[4]u32, xh, 4b3232}
|
||||
carry = (xc ^ hb) ^ shuf{u64, xh, 1,1}
|
||||
rem:= nw - 8*i
|
||||
if (rem < 8) {
|
||||
maskStore{*V~~r+i, [8]u1~~(~(u8~~0xff<<rem)), v}
|
||||
@ -169,19 +169,19 @@ fn bcs{T if hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = {
|
||||
|
||||
def ii32 = iota{32}; def bit{k}=bit{k,ii32}; def tail{k}=tail{k,ii32}
|
||||
def sums{n} = (if (n==0) tup{0} else { def s=sums{n-1}; merge{s,s+1} })
|
||||
def widen{v:T} = mzip128{shuf{[4]u64, v, 4b3120}, T**0}
|
||||
def widen{v:T} = mzip128{shuf{[4]u64, v, 0,2,1,3}, T**0}
|
||||
|
||||
def sumlanes{x:(u32)} = {
|
||||
b:= [8]u32**x >> make{[8]u32, 4*tail{1, iota{8}}}
|
||||
s:= sel8{[32]u8~~b, ii32>>3 + bit{2}}
|
||||
s:= shuf{16, [32]u8~~b, ii32>>3 + bit{2}}
|
||||
p:= s & make{[32]u8, (1<<(1+tail{2})) - 1} # Prefixes
|
||||
d:= sel{[16]u8, make{[32]u8, merge{sums{4},sums{4}}}, [32]i8~~p}
|
||||
d+= sel8{d, bit{2}*(1+bit{3}>>2)-1}
|
||||
d + sel8{d, bit{3}-1}
|
||||
d:= shuf{[16]u8, make{[32]u8, merge{sums{4},sums{4}}}, [32]i8~~p}
|
||||
d+= shuf{16, d, bit{2}*(1+bit{3}>>2)-1}
|
||||
d + shuf{16, d, bit{3}-1}
|
||||
}
|
||||
def step{x:(u32), i, store1} = {
|
||||
d:= sumlanes{x}
|
||||
if (w==8) d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 4b1100}
|
||||
if (w==8) d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 0,0,1,1}
|
||||
j:= (w/8)*i
|
||||
def out{v, k} = each{out, widen{v}, 2*k+iota{2}}
|
||||
def out{v0:([vl]T), k} = {
|
||||
@ -341,8 +341,8 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
|
||||
@maskedLoop{4} (x in tup{V, xp},
|
||||
r in tup{V, rp} over promote{u64,nw}) {
|
||||
s := scan_words{x, m}
|
||||
pc:= c; c = shuf{V, -(s>>63), 4b2103}
|
||||
r = apply_carry{s, blend{V, c, pc, 2b0001}, (m-V**1)&~m}
|
||||
pc:= c; c = shuf{-(s>>63), 3,0,1,2}
|
||||
r = apply_carry{s, blend{c, pc, 1,0,0,0}, (m-V**1)&~m}
|
||||
m = m>>d4 | m<<(l-d4)
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,8 +3,6 @@
|
||||
def sel8{v:V, t} = sel{[16]u8, v, make{re_el{i8,V}, t}}
|
||||
def sel8{v:V, t if w256{V} and istup{t} and length{t}==16} = sel8{v, merge{t,t}}
|
||||
|
||||
def shuf{T, v, n if istup{n}} = shuf{T, v, base{4,n}}
|
||||
|
||||
local def rev{t} = { def l=length{t}; def j=l-1; select{j-t, j-range{l}} }
|
||||
local def rev{up,t} = if (up) t else rev{t}
|
||||
def sel8{v, t, up} = sel8{v, rev{up,t}}
|
||||
@ -23,7 +21,7 @@ def toLast{n:VT, up if hasarch{'X86_64'} and w128{VT}} = {
|
||||
def l{v, w} = l{zip{up,v}, 2*w}
|
||||
def l{v, w if hasarch{'SSSE3'}} = sel8{v, up*(16-w/8)+iota{16}%(w/8)}
|
||||
def l{v, w==32} = shuf{[4]i32, v, 4**(up*3)}
|
||||
def l{v, w==64} = shuf{[4]i32, v, (2*up) + tup{0,1,0,1}}
|
||||
def l{v, w==64} = shuf{[2]i64, v, 2** up }
|
||||
l{n, elwidth{VT}}
|
||||
}
|
||||
def toLast{n:VT, up if hasarch{'AVX2'} and w256{VT}} = {
|
||||
@ -49,13 +47,13 @@ def make_scan_idem{T, op, up} = {
|
||||
def w=width{T}; def c = k/w
|
||||
def merger{a,b} = if (up) merge{a,b} else merge{b,a}
|
||||
def id = make{V, merger{c**get_id{op,T}, (width{V}/w-c)**0}}
|
||||
(if (up) shl else shr){[16]u8, v, k/8} | id
|
||||
(if (up) vec_shift_right_128 else vec_shift_left_128){v, c} | id
|
||||
}
|
||||
def shb{v, k if hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}}
|
||||
def shb{v, k if k>=32} = shuf{[4]u32, v, shift{k/32,4}}
|
||||
def shb{v, k if k==128 and hasarch{'AVX2'}} = {
|
||||
# After lanewise scan, broadcast end of lane 0 to entire lane 1
|
||||
sel{[8]i32, spread{v,up}, make{[8]i32, rev{up,3*(3<iota{8})}}}
|
||||
shuf{[8]i32, spread{v,up}, rev{up,3*(3<iota{8})}}
|
||||
}
|
||||
prefix_byshift{op, shb}
|
||||
}
|
||||
|
||||
@ -122,7 +122,7 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
|
||||
tab:= *V~~(repeat_tab + 32 * (start-1))
|
||||
def l0 = load{tab, 0}
|
||||
def l1 = load{tab, 1}
|
||||
v = sel{V, v, l0}
|
||||
v = shuf{V, v, l0}
|
||||
v+= V~~(VU16~~l1 * broadcast{VU16, csz})
|
||||
shufr = extract{l1, 15}+1
|
||||
}
|
||||
@ -278,24 +278,24 @@ exportT{'si_select_tab', join{table{select_fn,
|
||||
def xrev = rbit{load{*VU ~~ x0}}
|
||||
@maskedLoop{16}(cw0 in w, r in *u16~~r0, M in 'm' over i to wl) {
|
||||
def cw = ty_u{wrapChk{cw0, xlf, M}}
|
||||
def byte = sel{[16]u8, xrev, cw>>3}
|
||||
def byte = shuf{[16]u8, xrev, cw>>3}
|
||||
r = homMask{ty_s{byte << (cw & VU**7)} < VI**0}
|
||||
}
|
||||
} else {
|
||||
if (wl>32 and xl<=16) {
|
||||
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
|
||||
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
|
||||
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
|
||||
cw:= wrapChk{cw0, xlf, M}
|
||||
sr = homMask{sel{[16]i8, xb, cw}}
|
||||
sr = homMask{shuf{[16]i8, xb, cw}}
|
||||
}
|
||||
} else {
|
||||
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
|
||||
x:= shuf{[4]u64, load{*VI ~~ x0}, 0,1,0,1}
|
||||
low:= VI**7
|
||||
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
|
||||
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
|
||||
cw:= wrapChk{cw0, xlf, M}
|
||||
byte:= sel{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
|
||||
mask:= sel{[16]i8, b, cw & low}
|
||||
byte:= shuf{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
|
||||
mask:= shuf{[16]i8, b, cw & low}
|
||||
sr = homMask{(mask & byte) == mask}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
### SSSE3 ###
|
||||
def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
|
||||
def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = V~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(width{T}/8)}
|
||||
def sel{L=[_]E, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = vec_select{E, x, i}
|
||||
|
||||
|
||||
### SSE4.1 ###
|
||||
@ -21,12 +20,9 @@ def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16
|
||||
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
# assumes all bits are the same in each mask item
|
||||
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M} and elwidth{M}!=16} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M,16}} = topBlend{f, t, [16]i8~~m}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = blend_top{ f , t , m}
|
||||
|
||||
|
||||
|
||||
|
||||
@ -23,20 +23,16 @@ def unord{a:T,b:T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
|
||||
|
||||
|
||||
|
||||
# shift
|
||||
def shl{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
|
||||
def shr{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
|
||||
|
||||
# integer arith
|
||||
def mulh{a:T,b:T if [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
|
||||
def mulh{a:T,b:T if [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
|
||||
def mul32{a:T,b:T if [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments
|
||||
def __mul{a:T,b:T if [4]i32==T} = {
|
||||
def mu{x, y} = [4]i32 ~~ mul32{[2]u64~~x, [2]u64~~y}
|
||||
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
|
||||
def sw{n, ...vs} = each{{c} => vec_shuffle{i32, c, n}, vs}
|
||||
lo:= mu{a, b}
|
||||
hi:= mu{...sw{4b2301, a, b}}
|
||||
[4]i32~~mzip{...sw{4b3120, lo, hi}, 0}
|
||||
hi:= mu{...sw{tup{1,0}, a, b}}
|
||||
[4]i32~~mzip{...sw{tup{0,2,1,3}, lo, hi}, 0}
|
||||
}
|
||||
|
||||
# float arith
|
||||
@ -68,19 +64,9 @@ def packs{a:T,b:T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
|
||||
def packs{a:T,b:T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T if w128i{T}} = packs{a,b}
|
||||
|
||||
def x86_postfix{T if isint{T}} = merge{'_epi',fmtnat{width{T}}}
|
||||
def x86_postfix{(f32)} = '_ps'
|
||||
def x86_postfix{(f64)} = '_pd'
|
||||
def zip{a:T, b:T, 0 if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip{a:T, b:T, 1 if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b}
|
||||
|
||||
def shuf{L, x:T, n if w128{T} and lvec{L,4,32} and knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
|
||||
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
|
||||
def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}
|
||||
def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4}
|
||||
def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = T~~emit{[4]f32, '_mm_shuffle_ps', v2f{a}, v2f{b}, base{4, is&3}}
|
||||
def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3}
|
||||
|
||||
def homBlend{f:T, t:T, m:M if w128{T} and w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f &~ m))
|
||||
def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
|
||||
def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = {
|
||||
@ -92,15 +78,17 @@ def widen{T==[2]f64, x:X if w128s{X} and elwidth{X}<32} = widen{T, widen{[4]i32,
|
||||
def widen{T==[2]f64, x:X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
|
||||
def widen{T==[2]f64, x:X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
|
||||
|
||||
local def nsh = tup{0,2,3,3}
|
||||
local def nar = vec_shuffle{..., nsh}
|
||||
def narrow{T==i16, x:([4]i32)} = packs{x,x}
|
||||
def narrow{T==i8, x:([8]i16)} = packs{x,x}
|
||||
def narrow{T==u8, x:([8]u16)} = packs{x,x}
|
||||
def narrow{T==u16, x:([4]u32)} = [8]u16~~shuf{[4]i32, shuf16Hi{shuf16Lo{x, 4b3320}, 4b3320}, 4b3320}
|
||||
def narrow{T==u16, x:([4]u32)} = [8]u16~~nar{i32, nar{i16, x}}
|
||||
def narrow{T==i8, x:([4]i32)} = narrow{T, narrow{i16, x}}
|
||||
def narrow{T==u8, x:([4]u32)} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{x}}}
|
||||
def narrow{T==u8, x:([2]u64)} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{f{x}}}}
|
||||
def narrow{T==u16, x:([2]u64)} = shuf16Lo{[8]u16~~shuf{[4]i32, x, 4b3320}, 4b3320}
|
||||
def narrow{T==u32, x:([2]u64)} = [4]u32~~shuf{[4]i32, x, 4b3320}
|
||||
def narrow{T==u16, x:([2]u64)} = vec_shuffle16_lo{[8]u16~~nar{i32, x}, nsh}
|
||||
def narrow{T==u32, x:([2]u64)} = [4]u32~~nar{i32, x}
|
||||
|
||||
def narrow{T, x:X if w128f{X,64} and T<i32} = narrow{T, narrow{i32, x}}
|
||||
def narrow{T==i32, x:([2]f64)} = emit{[4]i32, '_mm_cvtpd_epi32', x}
|
||||
|
||||
@ -18,15 +18,15 @@ def unpack_pass{o, x} = merge{...each{unpack_typed, ...pairs{o, x}}}
|
||||
|
||||
def permute_pass{o, x} = {
|
||||
def p = pairs{o, x}
|
||||
def h{s} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,s}, ...p}
|
||||
merge{h{16b20}, h{16b31}}
|
||||
def h{...s} = each{vec_select{128, ., s}, flip{p}}
|
||||
merge{h{0,2}, h{1,3}}
|
||||
}
|
||||
def unpack_to{f, l, x} = {
|
||||
def pass = if (f) permute_pass else unpack_pass
|
||||
pass{l, if (l==1) x else unpack_to{0, l/2, x}}
|
||||
}
|
||||
# Last pass for square kernel packed in halves
|
||||
def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x}
|
||||
def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 0,2,1,3}, x}
|
||||
|
||||
# Square kernel where width is a full vector
|
||||
def transpose_square{VT, l, x if hasarch{'AVX2'}} = unpack_to{1, l/2, x}
|
||||
|
||||
@ -3,9 +3,9 @@
|
||||
def vfold{F, x:V=[_]T if w128{V} and hasarch{'X86_64'}} = {
|
||||
c:= x
|
||||
def EW = width{T}
|
||||
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
|
||||
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
|
||||
if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}}
|
||||
if (EW<=64) c = F{c, shuf{u64, c, 1,0}}
|
||||
if (EW<=32) c = F{c, shuf{u32, c, 1,0}}
|
||||
if (EW<=16) c = F{c, vec_shuffle16_lo{c, tup{1,0,3,2}}}
|
||||
if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{T, v}, cast_i{T, v>>8}} }
|
||||
else extract{c, 0}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user