Alias vec_shuffle to shuf

This commit is contained in:
Marshall Lochbaum 2024-10-12 20:48:33 -04:00
parent 27af1c0558
commit fa43ffd882
12 changed files with 76 additions and 75 deletions

View File

@ -241,6 +241,7 @@ def broadcast{V=[_]T, v} = vec_broadcast{V, promote{T,v}}
def make{V=[_]_, ...xs} = vec_make{V, ...xs}
def iota{V=[k]_} = make{V, ...iota{k}}
def absu{a:[_]_} = ty_u{abs{a}}
def shuf = vec_shuffle
def floor = __floor
def ceil = __ceil

View File

@ -20,7 +20,7 @@ def for_vec_overlap{vl}{vars,begin==0,n,iter} = {
def shr16{v:V, n} = V~~(re_el{u16, v} >> n)
# Copy half of a vector to the other half
def dup_half{vec,half} = vec_shuffle{[4]u64, vec, 2*half+tup{0,1,0,1}}
def dup_half{vec,half} = shuf{[4]u64, vec, 2*half+tup{0,1,0,1}}
# Forward or backwards in-place max-scan
# Assumes a whole number of vectors and minimum 0
@ -44,15 +44,15 @@ fn max_scan{T, up}(x:*T, len:u64) : void = {
def getsel{...x} = assert{'shuffling not supported', show{...x}}
if_inline (hasarch{'AVX2'}) {
def getsel{h:H=[16]T if width{T}==8} = {
vec_shuffle{H, pair{h,h}, .}
shuf{H, pair{h,h}, .}
}
def getsel{v:V=[vl==32]T if width{T}==8} = {
def H = n_h{V}
vtop := V**(vl/2)
hs := each{dup_half{v, .}, tup{1,0}}
{i} => homBlend{...each{vec_shuffle{H,.,i}, hs}, V~~i<vtop}
{i} => homBlend{...each{shuf{H,.,i}, hs}, V~~i<vtop}
}
def getsel{v:V=[8]T if width{T}==32} = vec_shuffle{V, v, .}
def getsel{v:V=[8]T if width{T}==32} = shuf{V, v, .}
}
# Move evens to half 0 and odds to half 1
@ -60,8 +60,8 @@ def uninterleave{x:V=[vl]T if hasarch{'AVX2'}} = {
def bytes = width{T}/8
def i = 2*iota{vl/4}
def i2= flat_table{+, bytes*merge{i,i+1}, iota{bytes}}
t := vec_shuffle{[16]u8, x, i2}
vec_shuffle{[4]u64, t, 0,2,1,3}
t := shuf{[16]u8, x, i2}
shuf{[4]u64, t, 0,2,1,3}
}
def rtypes = tup{i8, i16, i32, f64}
@ -169,7 +169,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
s{shr16{v,4}} + s{v}
}
# Bit table
def swap{v} = vec_shuffle{[4]u64, v, 2,3,0,1} # For signedness
def swap{v} = shuf{[4]u64, v, 2,3,0,1} # For signedness
def sel_b = getsel{swap{vb}}
# Masks for filtering bit table
def ms = if (up) 256-(1<<(1+iota{8})) else (1<<iota{8})-1
@ -187,7 +187,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}}
if (dup) {
i0 := V~~ind # Can contain -1
def isel{u} = vec_shuffle{H, u, i0}
def isel{u} = shuf{H, u, i0}
ind = isel{ui}
if (nu > 16) {
b := V~~(re_el{u16, i0} << (7 - lb{vl/2}))

View File

@ -2,7 +2,7 @@ def xrange{s, e} = s + range{e-s}
def __shl{(u64)}{a:T, b} = T~~(re_el{u64,a}<<b)
def __shr{(u64)}{a:T, b} = T~~(re_el{u64,a}>>b)
def shuf_imm = vec_shuffle
def shuf_imm = shuf
def shuf_impl{rw, ...iw, data} = assert{0, 'shuffling failed', ...data{'info'}}
def type0{vs} = type{select{vs,0}}
@ -56,7 +56,7 @@ local def perm_split{x:X, lanes, chunks} = {
def si0 = join{each{index_of, u, gr}*b + lanes%b}
def si = tern{join{lanes>=0}, si0, 0xff}
# def si = join{each{{u,g,l} => l>=0) index_of{u,g}*b + lanes%b else 0xff, u, gr, lanes}}
vec_shuffle{[16]u8,
shuf{[16]u8,
new_shuf{re_el{ty_u{b*8},X}, x, join{each{shiftright{.,chunks**0}, u}}},
make{X, si}
}
@ -70,18 +70,18 @@ local def perm_shufb{x, is} = {
}
# TODO accept narrower inputs
def shuf_impl{256, 256, data if hasarch{'AVX2'} and not same{perm_shufb{...data{u8}},'!'}} = perm_shufb{...data{u8}}
def shuf_impl{256, 128, data if hasarch{'AVX2'} and data{'h',u8}} = { def {x,is} = data{u8}; vec_shuffle{[16]u8, pair{x, x}, is} }
def shuf_impl{256, 128, data if hasarch{'AVX2'} and data{'h',u8}} = { def {x,is} = data{u8}; shuf{[16]u8, pair{x, x}, is} }
def shuf_impl{512, 512, data if hasarch{'AVX512F'} and data{'h','nz',u8}} = { def {x,is} = data{u8}; sel{[64]u8, x, make{[64]u8, is}} }
def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u32}} = { def {x,is} = data{'nz',u32}; vec_shuffle{[8]u32, x, is} }
def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u32}} = { def {x,is} = data{'nz',u32}; shuf{[8]u32, x, is} }
def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u64}} = { def {x,is} = data{'nz',u64}; shuf_imm{[4]u64, x, is} }
def in_chunks{c, is} = { def ls = split{c, is}; all{{l, s} => all{(l<0) | ((l>=s) & (l<s+c))}, ls, inds{ls}*c} }
def shuf_impl{256, 256, data if hasarch{'AVX2'} and in_chunks{16,select{data{u8},1}}} = { def {x,is} = data{u8}; vec_shuffle{[16]u8, x, is} }
def shuf_impl{128, 128, data if hasarch{'SSSE3'}} = { def {x,is} = data{u8}; vec_shuffle{[16]u8, x, is} }
def shuf_impl{256, 256, data if hasarch{'AVX2'} and in_chunks{16,select{data{u8},1}}} = { def {x,is} = data{u8}; shuf{[16]u8, x, is} }
def shuf_impl{128, 128, data if hasarch{'SSSE3'}} = { def {x,is} = data{u8}; shuf{[16]u8, x, is} }
def shuf_impl{128, 128, data if hasarch{'AARCH64'}} = { def {x,is} = data{u8}; vec_shuffle{[16]u8, x, is} }
def shuf_impl{128, 128, data if hasarch{'AARCH64'}} = { def {x,is} = data{u8}; shuf{[16]u8, x, is} }

View File

@ -63,7 +63,7 @@ def spreadBits{T==[32]u8, a:(u32)} = {
def idxs = iota{32}
b:= [8]u32**a
c:= [32]u8~~b
d:= vec_shuffle{[16]u8, c, make{[32]i8, idxs>>3 + bit{4, idxs}}}
d:= shuf{[16]u8, c, make{[32]i8, idxs>>3 + bit{4, idxs}}}
e:= make{[32]u8, 1<<tail{3, idxs}}
e == (d&e)
}
@ -74,7 +74,7 @@ def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = {
}
def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = {
b:= T~~[8]u16**a
exp:= T~~vec_shuffle{[4]i32, vec_shuffle16_lo{mzip{b,b,0}, tup{0,0,1,1}}, 0,0,1,1}
exp:= T~~shuf{[4]i32, vec_shuffle16_lo{mzip{b,b,0}, tup{0,0,1,1}}, 0,0,1,1}
(exp & make{T, 1<<(iota{16}&7)}) != T**0
}

View File

@ -4,7 +4,7 @@ if_inline (hasarch{'BMI2'}) include './bmi2'
include './spaced'
include './scan_common'
def opsh64{op}{v:[_](f64), ...perm} = op{v, vec_shuffle{v, ...perm}}
def opsh64{op}{v:[_](f64), ...perm} = op{v, shuf{v, ...perm}}
def mix{op, v:([4]f64) if hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 1,0}, 2,3,0,1} }
def mix{op, v:([2]f64) if hasarch{'X86_64'}} = opsh64{op}{v, 1,0}

View File

@ -40,7 +40,7 @@ def unzip_load{E, n, TG if width{E}*n <= arch_defvw} = each{tup, unzip{TG{[n]E,
def widen_tup{u32, is:([16]u8)} = tup{ # compiler will deduplicate all the repeated calls of this on the same is
widen{[8]u32, is},
widen{[8]u32, vec_shuffle{[2]u64, is, 1,1}},
widen{[8]u32, shuf{[2]u64, is, 1,1}},
}
def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
@ -86,9 +86,9 @@ def raw_widen_inds{[k]D, x:[k0]S if k0>=k} = { # : [k*sc]S
def add = make{RV, range{k*sc} % sc}
if (hasarch{'AARCH64'} and [k]D == [2]u64 and S==u8) {
def x16 = undefPromote{[16]u8, x}
(RV~~vec_shuffle{[16]u8, x16, range{16}>>3}<<3) + add
(RV~~shuf{[16]u8, x16, range{16}>>3}<<3) + add
} else if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) {
(vec_shuffle{[8]u32, undefPromote{[8]u32, x}, range{8}>>1}<<sc) + add
(shuf{[8]u32, undefPromote{[8]u32, x}, range{8}>>1}<<sc) + add
} else {
def wd = widen{[k]D, x}
re_el{S, wd * [k]D**base{1<<width{S}, sc**sc}} + add
@ -122,11 +122,11 @@ def zip_halves{mode, E, nt, ni} = match(lut_gen{mode, w_h{E}, nt, ni}) { # e.g.
run_zip{mzip, is}
}
def me{is:([ni]u8) if hasarch{'X86_64'} and E==u64 and ni==16} = {
def is2 = vec_shuffle{u8, is, tr_iota{0,2,1,3}}
def is2 = shuf{u8, is, tr_iota{0,2,1,3}}
run_zip{mzip128, is2}
}
def me{is:([ni]u8) if hasarch{'X86_64'} and E==u16 and ni==32} = {
def is2 = vec_shuffle{u64, is, tr_iota{1,0}}
def is2 = shuf{u64, is, tr_iota{1,0}}
run_zip{mzip128, is2}
}
}}}
@ -149,13 +149,13 @@ def lut_gen{'c', E==u8, nt, ni if hasarch{'AVX2'} and nt<=64 and ni<=32} = { de
def top = [32]u8**128
def isp = each{{i} => is | (top &~ tree_fold{&, each{{m, o} => if (bit{o,i}!=0) m else ~m, bits, bi}}), range{vn}}
tup{tree_fold{|, each{vec_shuffle{[16]u8,.,.}, luts, isp}}}
tup{tree_fold{|, each{shuf{[16]u8,.,.}, luts, isp}}}
}
}}}}
def lut_gen{mode, E==u8, nt, ni if hasarch{'AVX2'} and nt<=16 and ni<=32} = tup{16, 32, loader{{TG} => {
lut:[32]u8 = [32]u8**TG{[16]u8, 0}
{is:([32]u8)} => tup{vec_shuffle{[16]u8, lut, is}}
{is:([32]u8)} => tup{shuf{[16]u8, lut, is}}
}}}
def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=32 and ni<=16} = blend_halves{mode, E, 32, 16}
@ -163,7 +163,7 @@ def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=16 and ni<=16} = ble
def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=8 and ni<=16} = tup{8, 16, loader{{TG} => {
def lut = TG{[8]u32, 0}
def me{'idxs', is:([16]u8)} = each{{wis} => tup{wis, vec_shuffle{[8]u32, lut, wis}}, widen_tup{u32,is}} # TODO inline, or properly outline
def me{'idxs', is:([16]u8)} = each{{wis} => tup{wis, shuf{[8]u32, lut, wis}}, widen_tup{u32,is}} # TODO inline, or properly outline
def me{is:([16]u8)} = each{{{_,v}}=>v, me{'idxs', is}}
}}}
@ -191,8 +191,8 @@ def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='c' and E>=u16} = 0
def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='i' and E==u64 and nt>16} = 0
def lut16{tab:([16]u8), idxs:([16]u8)} = vec_shuffle{[16]u8, tab, idxs}
def lut16{tab:([16]u8), idxs:([32]u8) if hasarch{'X86_64'}} = vec_shuffle{[16]u8, pair{tab, tab}, idxs}
def lut16{tab:([16]u8), idxs:([16]u8)} = shuf{[16]u8, tab, idxs}
def lut16{tab:([16]u8), idxs:([32]u8) if hasarch{'X86_64'}} = shuf{[16]u8, pair{tab, tab}, idxs}
def shuf_u8bits{inds:*u8, ni} = 0
def shuf_u8bits{inds:*u8, ni if has_sel} = {

View File

@ -103,7 +103,7 @@ rcsh4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}}
def read_shuf_vecs{l, ellw:(u64), shp:*V} = { # tuple of byte selectors in 1<<ellw
def double{x:X if hasarch{'AVX2'}} = {
s:=vec_shuffle{u64, x, 0,2,1,3}; s+=s
s:=shuf{u64, x, 0,2,1,3}; s+=s
r:=each{bind{~~,[32]i8},mzip128{s, s + X**1}}
r
}
@ -150,20 +150,20 @@ if_inline (hasarch{'AVX2'}) {
def rep_iter_from_sh{sh}{x, gen} = {
def l = length{sh}
def h = l>>1
def fs{v, s} = gen{vec_shuffle{[16]i8, v, s}}
a := vec_shuffle{u64, x, 0,1,0,1}; each{fs{a,.}, slice{sh,0,h}}
def fs{v, s} = gen{shuf{[16]i8, v, s}}
a := shuf{u64, x, 0,1,0,1}; each{fs{a,.}, slice{sh,0,h}}
if (l%2) fs{x, select{sh, h}}
b := vec_shuffle{u64, x, 2,3,2,3}; each{fs{b,.}, slice{sh,-h}}
b := shuf{u64, x, 2,3,2,3}; each{fs{b,.}, slice{sh,-h}}
}
def get_rep_iter{V, wv==2}{x, gen} = {
def s = vec_shuffle{u64, x, 0,2,1,3}
def s = shuf{u64, x, 0,2,1,3}
each{{q}=>gen{V~~q}, mzip128{s, s}}
}
def get_rep_iter{V==[4]u64, wv} = {
def step = 4
def sh = get_shufs{step, wv, wv}
{x, gen} => each{{s}=>gen{vec_shuffle{V, x, s}}, sh}
{x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh}
}
def rep_const_shuffle{wv, xv:*V, rv:*V, n:(u64)} = rep_const_shuffle{wv, get_rep_iter{V, wv}, xv, rv, n}
@ -171,7 +171,7 @@ if_inline (hasarch{'AVX2'}) {
} else { # has_bytesel_128
def rep_iter_from_sh{sh}{x, gen} = {
each{{s} => gen{vec_shuffle{[16]u8, x, s}}, sh}
each{{s} => gen{shuf{[16]u8, x, s}}, sh}
}
def rep_const_shuffle{wv==2, xv0:*V=[_]T, rv0:*V, n:(u64)} = {
@ -191,9 +191,9 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
def [step]_ = V # Bytes written
def wvb = wv << ellw
def hs = (h*step) / wvb # Actual step size in argument elements
def shufbase{i if hasarch{'AVX2'}} = vec_shuffle{u64, load{*V~~(x+i)}, 0,1,0,1}
def shufbase{i if hasarch{'AVX2'}} = shuf{u64, load{*V~~(x+i)}, 0,1,0,1}
def shufbase{i if has_bytesel_128} = load{*V~~(x+i)}
def shufrun{a, s} = vec_shuffle{[16]i8, a, s} # happens to be the same across AVX2 & NEON
def shufrun{a, s} = shuf{[16]i8, a, s} # happens to be the same across AVX2 & NEON
i:u64 = 0
re := r + n*wvb - h*step
@ -442,18 +442,18 @@ def rep_const_bool_div8{wv, x, r, nw if has_simd} = {
run8{{x} => z{z{z{x}}}}
}
} else { # any_sel
def selH = vec_shuffle{[16]u8, ., .}
def selH = shuf{[16]u8, ., .}
def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .}
def id{xv} = xv
if (wv == 2) {
def init = if (avx2) vec_shuffle{u64, ., 0,2,1,3} else id
def init = if (avx2) shuf{u64, ., 0,2,1,3} else id
# Expander for half byte
def tabr = makeTab{tr_iota{2*iota{4}} * 2b11}
m4 := V**0xf
run24{*V~~x, init, {x} => tabr{x & m4}, 4}
} else if (wv == 4) {
# Unzip 32-bit elements (result lanes) across AVX2 lanes
def pre = if (avx2) vec_shuffle{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id
def pre = if (avx2) shuf{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id
def init{xv} = { u:=pre{xv}; zip128{u,u,0} }
# Expander for two bits in either bottom or next-to-bottom position
def tabr = makeTab{tr_iota{0,4,0,4} * 2b1111}
@ -495,7 +495,7 @@ def swap_elts{x:V=[_]_, el_bytes} = {
def wd = min{el_bytes, 8}
def l = el_bytes/wd
def i = iota{l}
vec_shuffle{ty_u{wd*8}, x, merge{i+l, i}}
shuf{ty_u{wd*8}, x, merge{i+l, i}}
} else {
def sh = el_bytes*8
xe := re_el{ty_u{2*sh}, V} ~~ x
@ -510,7 +510,7 @@ def extract_modperm_mask{full:W, lane, l} = {
def n = (if (f) 256 else 128) / w # and number of elements
def o = l / w # Extract [o,2*o)
def i = o + iota{n}%o
vec_shuffle{ty_u{w}, if (f) full else lane, i}
shuf{ty_u{w}, if (f) full else lane, i}
}
def proc_mod_dat{swap_data:W} = {
def ww = width{W}; def avx2 = ww==256
@ -518,7 +518,7 @@ def proc_mod_dat{swap_data:W} = {
def lens = reverse{1 << slice{iota{lb{hi}}, lb{lo}}}
fold{{x,s}=>s{x}, ., each{get_swap, lens}}
}
swap_lane := if (avx2) vec_shuffle{swap_data, 0,1,0,1} else swap_data
swap_lane := if (avx2) shuf{swap_data, 0,1,0,1} else swap_data
# Transform width-w units with shifts only
def get_shiftperm{data, w} = {
dat := data
@ -535,20 +535,20 @@ def proc_mod_dat{swap_data:W} = {
# Within-byte transformation
def get_byteperm{} = {
def V = re_el{u8, W}
def sw_bytes = vec_shuffle{u8, swap_lane, 16**0}
def sw_bytes = shuf{u8, swap_lane, 16**0}
m4 := W~~V**0xf
t0 := fold{{v,a}=>modperm_shift_step{v,...a}, W~~make{V,iota{vcount{V}}%16}, tup{
tup{4, sw_bytes &~ m4},
tup{2, ({v} => v | v<<4){sw_bytes & W~~V**0xc}}
}}
t4 := (t0&m4)<<4 | (t0&~m4)>>4
def selI{v,i} = vec_shuffle{[16]u8, v, V~~i}
def selI{v,i} = shuf{[16]u8, v, V~~i}
{xv} => selI{t0, xv & m4} | selI{t4, (xv>>4) & m4}
}
def {partwidth, partperm} = {
def sh{w, d} = tup{w, get_shiftperm{d, w}}
if (ww==64) sh{64, swap_data}
else if (not any_sel) sh{32, vec_shuffle{u32, swap_data, 0,0,0,0}}
else if (not any_sel) sh{32, shuf{u32, swap_data, 0,0,0,0}}
else tup{8, get_byteperm{}}
}
# Fill in higher steps
@ -631,7 +631,7 @@ def rep_const_bool_odd_mask4{
def sub_carry{a, c} = match (M) {
{[l](u64)} => {
ca := if (hasarch{'SSE4.2'} or hasarch{'AARCH64'}) { def S = [l]i64; S~~c > S**0 }
else { cm := [2*l]i32~~c; cm != vec_shuffle{cm, 1,0} }
else { cm := [2*l]i32~~c; cm != shuf{cm, 1,0} }
a + M~~ca
}
{_} => a - promote{u64, c != 0}
@ -640,9 +640,9 @@ def rep_const_bool_odd_mask4{
while (1) {
x:M = get_modperm_x{}
def vrot1 = ifvec{{x} => {
if (w256{M}) vec_shuffle{x, 3,0,1,2}
if (w256{M}) shuf{x, 3,0,1,2}
else if (any_sel) vshl{x, x, vcount{type{x}}-1}
else vec_shuffle{x, 1,0}
else shuf{x, 1,0}
}}
xo := x<<k | vrot1{x>>(64-k)}
# Write result word given starting bits
@ -702,8 +702,8 @@ def rep_const_bool_small_odd{W=[wl](u64), max_wv, wv, get_perm_x, output} = {
def step{jj, oi, ind, mask} = {
def hk = (k-1) / 2
def getv = if (not lanes or jj==hk) ({x}=>x)
else vec_shuffle{[4]u64, ., 2*(jj>hk) + iota{4}%2}
def selx{x, i} = vec_shuffle{[128/ew]ty_u{ew}, getv{x}, i}
else shuf{[4]u64, ., 2*(jj>hk) + iota{4}%2}
def selx{x, i} = shuf{[128/ew]ty_u{ew}, getv{x}, i}
b := selx{xv, ind} & make{W, mask}
r := (b<<k) - b
def selx_nz{x, i} = { def nz = i != -1; selx{x, i * nz} & W~~make{[4]i32, -nz} }

View File

@ -141,7 +141,7 @@ fn scan_neq{if hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}}(init:u64, x:*u64, r:*u
xh := exor64{hb} # Exclusive xor of high bits (xh ^ hb for inclusive)
xc := xh ^ carry
v := x8 ^ V~~maskToHom{[64]u8, [64]u1~~extract{xc,0}}
carry = (xc ^ hb) ^ vec_shuffle{u64, xh, 1,1}
carry = (xc ^ hb) ^ shuf{u64, xh, 1,1}
rem:= nw - 8*i
if (rem < 8) {
maskStore{*V~~r+i, [8]u1~~(~(u8~~0xff<<rem)), v}
@ -169,19 +169,19 @@ fn bcs{T if hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = {
def ii32 = iota{32}; def bit{k}=bit{k,ii32}; def tail{k}=tail{k,ii32}
def sums{n} = (if (n==0) tup{0} else { def s=sums{n-1}; merge{s,s+1} })
def widen{v:T} = mzip128{vec_shuffle{[4]u64, v, 0,2,1,3}, T**0}
def widen{v:T} = mzip128{shuf{[4]u64, v, 0,2,1,3}, T**0}
def sumlanes{x:(u32)} = {
b:= [8]u32**x >> make{[8]u32, 4*tail{1, iota{8}}}
s:= vec_shuffle{16, [32]u8~~b, ii32>>3 + bit{2}}
s:= shuf{16, [32]u8~~b, ii32>>3 + bit{2}}
p:= s & make{[32]u8, (1<<(1+tail{2})) - 1} # Prefixes
d:= vec_shuffle{[16]u8, make{[32]u8, merge{sums{4},sums{4}}}, [32]i8~~p}
d+= vec_shuffle{16, d, bit{2}*(1+bit{3}>>2)-1}
d + vec_shuffle{16, d, bit{3}-1}
d:= shuf{[16]u8, make{[32]u8, merge{sums{4},sums{4}}}, [32]i8~~p}
d+= shuf{16, d, bit{2}*(1+bit{3}>>2)-1}
d + shuf{16, d, bit{3}-1}
}
def step{x:(u32), i, store1} = {
d:= sumlanes{x}
if (w==8) d+= [32]u8~~vec_shuffle{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 0,0,1,1}
if (w==8) d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 0,0,1,1}
j:= (w/8)*i
def out{v, k} = each{out, widen{v}, 2*k+iota{2}}
def out{v0:([vl]T), k} = {
@ -341,7 +341,7 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
@maskedLoop{4} (x in tup{V, xp},
r in tup{V, rp} over promote{u64,nw}) {
s := scan_words{x, m}
pc:= c; c = vec_shuffle{-(s>>63), 3,0,1,2}
pc:= c; c = shuf{-(s>>63), 3,0,1,2}
r = apply_carry{s, blend{c, pc, 1,0,0,0}, (m-V**1)&~m}
m = m>>d4 | m<<(l-d4)
}

View File

@ -20,13 +20,13 @@ def spread{a:[_]T, ...up} = {
def toLast{n:VT, up if hasarch{'X86_64'} and w128{VT}} = {
def l{v, w} = l{zip{up,v}, 2*w}
def l{v, w if hasarch{'SSSE3'}} = sel8{v, up*(16-w/8)+iota{16}%(w/8)}
def l{v, w==32} = vec_shuffle{[4]i32, v, 4**(up*3)}
def l{v, w==64} = vec_shuffle{[2]i64, v, 2** up }
def l{v, w==32} = shuf{[4]i32, v, 4**(up*3)}
def l{v, w==64} = shuf{[2]i64, v, 2** up }
l{n, elwidth{VT}}
}
def toLast{n:VT, up if hasarch{'AVX2'} and w256{VT}} = {
if (elwidth{VT}<=32) sel{[8]i32, spread{n,up}, [8]i32**(up*7)}
else vec_shuffle{[4]u64, n, 4**(up*3)}
else shuf{[4]u64, n, 4**(up*3)}
}
def toLast{n:VT} = toLast{n, 1}
@ -50,17 +50,17 @@ def make_scan_idem{T, op, up} = {
(if (up) vec_shift_right_128 else vec_shift_left_128){v, c} | id
}
def shb{v, k if hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}}
def shb{v, k if k>=32} = vec_shuffle{[4]u32, v, shift{k/32,4}}
def shb{v, k if k>=32} = shuf{[4]u32, v, shift{k/32,4}}
def shb{v, k if k==128 and hasarch{'AVX2'}} = {
# After lanewise scan, broadcast end of lane 0 to entire lane 1
vec_shuffle{[8]i32, spread{v,up}, rev{up,3*(3<iota{8})}}
shuf{[8]i32, spread{v,up}, rev{up,3*(3<iota{8})}}
}
prefix_byshift{op, shb}
}
def make_scan_idem{(f64), op, up} = {
def sc{a:[2]T} = op{a, zip{~up,a}}
def sc{a:T if hasarch{'AVX2'} and w256{T}} = {
def sh{s, a} = op{a, vec_shuffle{[4]u64, a, rev{up,s}}}
def sh{s, a} = op{a, shuf{[4]u64, a, rev{up,s}}}
sh{tup{0,1,1,1},sh{tup{0,0,2,2},a}}
}
sc

View File

@ -122,7 +122,7 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
tab:= *V~~(repeat_tab + 32 * (start-1))
def l0 = load{tab, 0}
def l1 = load{tab, 1}
v = vec_shuffle{V, v, l0}
v = shuf{V, v, l0}
v+= V~~(VU16~~l1 * broadcast{VU16, csz})
shufr = extract{l1, 15}+1
}
@ -278,24 +278,24 @@ exportT{'si_select_tab', join{table{select_fn,
def xrev = rbit{load{*VU ~~ x0}}
@maskedLoop{16}(cw0 in w, r in *u16~~r0, M in 'm' over i to wl) {
def cw = ty_u{wrapChk{cw0, xlf, M}}
def byte = vec_shuffle{[16]u8, xrev, cw>>3}
def byte = shuf{[16]u8, xrev, cw>>3}
r = homMask{ty_s{byte << (cw & VU**7)} < VI**0}
}
} else {
if (wl>32 and xl<=16) {
xb:= vec_shuffle{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
cw:= wrapChk{cw0, xlf, M}
sr = homMask{vec_shuffle{[16]i8, xb, cw}}
sr = homMask{shuf{[16]i8, xb, cw}}
}
} else {
x:= vec_shuffle{[4]u64, load{*VI ~~ x0}, 0,1,0,1}
x:= shuf{[4]u64, load{*VI ~~ x0}, 0,1,0,1}
low:= VI**7
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
cw:= wrapChk{cw0, xlf, M}
byte:= vec_shuffle{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
mask:= vec_shuffle{[16]i8, b, cw & low}
byte:= shuf{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
mask:= shuf{[16]i8, b, cw & low}
sr = homMask{(mask & byte) == mask}
}
}

View File

@ -26,7 +26,7 @@ def unpack_to{f, l, x} = {
pass{l, if (l==1) x else unpack_to{0, l/2, x}}
}
# Last pass for square kernel packed in halves
def shuf_pass{x} = each{{v} => vec_shuffle{[4]i64, v, 0,2,1,3}, x}
def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 0,2,1,3}, x}
# Square kernel where width is a full vector
def transpose_square{VT, l, x if hasarch{'AVX2'}} = unpack_to{1, l/2, x}

View File

@ -3,8 +3,8 @@
def vfold{F, x:V=[_]T if w128{V} and hasarch{'X86_64'}} = {
c:= x
def EW = width{T}
if (EW<=64) c = F{c, vec_shuffle{u64, c, 1,0}}
if (EW<=32) c = F{c, vec_shuffle{u32, c, 1,0}}
if (EW<=64) c = F{c, shuf{u64, c, 1,0}}
if (EW<=32) c = F{c, shuf{u32, c, 1,0}}
if (EW<=16) c = F{c, vec_shuffle16_lo{c, tup{1,0,3,2}}}
if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{T, v}, cast_i{T, v>>8}} }
else extract{c, 0}