From fa43ffd8826ced2aa2b036630a90bb46759e86c3 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 12 Oct 2024 20:48:33 -0400 Subject: [PATCH] Alias vec_shuffle to shuf --- src/singeli/src/base.singeli | 1 + src/singeli/src/bins.singeli | 16 +++++----- src/singeli/src/bitalign.singeli | 14 ++++----- src/singeli/src/bitops.singeli | 4 +-- src/singeli/src/fold.singeli | 2 +- src/singeli/src/lut.singeli | 20 ++++++------- src/singeli/src/replicate.singeli | 46 ++++++++++++++--------------- src/singeli/src/scan.singeli | 16 +++++----- src/singeli/src/scan_common.singeli | 12 ++++---- src/singeli/src/select.singeli | 14 ++++----- src/singeli/src/transpose.singeli | 2 +- src/singeli/src/vecfold.singeli | 4 +-- 12 files changed, 76 insertions(+), 75 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 603a27eb..224e4593 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -241,6 +241,7 @@ def broadcast{V=[_]T, v} = vec_broadcast{V, promote{T,v}} def make{V=[_]_, ...xs} = vec_make{V, ...xs} def iota{V=[k]_} = make{V, ...iota{k}} def absu{a:[_]_} = ty_u{abs{a}} +def shuf = vec_shuffle def floor = __floor def ceil = __ceil diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 5dc069b6..9fc1482d 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -20,7 +20,7 @@ def for_vec_overlap{vl}{vars,begin==0,n,iter} = { def shr16{v:V, n} = V~~(re_el{u16, v} >> n) # Copy half of a vector to the other half -def dup_half{vec,half} = vec_shuffle{[4]u64, vec, 2*half+tup{0,1,0,1}} +def dup_half{vec,half} = shuf{[4]u64, vec, 2*half+tup{0,1,0,1}} # Forward or backwards in-place max-scan # Assumes a whole number of vectors and minimum 0 @@ -44,15 +44,15 @@ fn max_scan{T, up}(x:*T, len:u64) : void = { def getsel{...x} = assert{'shuffling not supported', show{...x}} if_inline (hasarch{'AVX2'}) { def getsel{h:H=[16]T if width{T}==8} = { - vec_shuffle{H, pair{h,h}, .} + shuf{H, pair{h,h}, .} } def getsel{v:V=[vl==32]T if width{T}==8} = { def H = n_h{V} vtop := V**(vl/2) hs := each{dup_half{v, .}, tup{1,0}} - {i} => homBlend{...each{vec_shuffle{H,.,i}, hs}, V~~i homBlend{...each{shuf{H,.,i}, hs}, V~~i 16) { b := V~~(re_el{u16, i0} << (7 - lb{vl/2})) diff --git a/src/singeli/src/bitalign.singeli b/src/singeli/src/bitalign.singeli index bdd9fc1b..80f488a2 100644 --- a/src/singeli/src/bitalign.singeli +++ b/src/singeli/src/bitalign.singeli @@ -2,7 +2,7 @@ def xrange{s, e} = s + range{e-s} def __shl{(u64)}{a:T, b} = T~~(re_el{u64,a}<>b) -def shuf_imm = vec_shuffle +def shuf_imm = shuf def shuf_impl{rw, ...iw, data} = assert{0, 'shuffling failed', ...data{'info'}} def type0{vs} = type{select{vs,0}} @@ -56,7 +56,7 @@ local def perm_split{x:X, lanes, chunks} = { def si0 = join{each{index_of, u, gr}*b + lanes%b} def si = tern{join{lanes>=0}, si0, 0xff} # def si = join{each{{u,g,l} => l>=0) index_of{u,g}*b + lanes%b else 0xff, u, gr, lanes}} - vec_shuffle{[16]u8, + shuf{[16]u8, new_shuf{re_el{ty_u{b*8},X}, x, join{each{shiftright{.,chunks**0}, u}}}, make{X, si} } @@ -70,18 +70,18 @@ local def perm_shufb{x, is} = { } # TODO accept narrower inputs def shuf_impl{256, 256, data if hasarch{'AVX2'} and not same{perm_shufb{...data{u8}},'!'}} = perm_shufb{...data{u8}} -def shuf_impl{256, 128, data if hasarch{'AVX2'} and data{'h',u8}} = { def {x,is} = data{u8}; vec_shuffle{[16]u8, pair{x, x}, is} } +def shuf_impl{256, 128, data if hasarch{'AVX2'} and data{'h',u8}} = { def {x,is} = data{u8}; shuf{[16]u8, pair{x, x}, is} } def shuf_impl{512, 512, data if hasarch{'AVX512F'} and data{'h','nz',u8}} = { def {x,is} = data{u8}; sel{[64]u8, x, make{[64]u8, is}} } -def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u32}} = { def {x,is} = data{'nz',u32}; vec_shuffle{[8]u32, x, is} } +def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u32}} = { def {x,is} = data{'nz',u32}; shuf{[8]u32, x, is} } def shuf_impl{256, 256, data if hasarch{'AVX2'} and data{'h','nz',u64}} = { def {x,is} = data{'nz',u64}; shuf_imm{[4]u64, x, is} } def in_chunks{c, is} = { def ls = split{c, is}; all{{l, s} => all{(l<0) | ((l>=s) & (l>3 + bit{4, idxs}}} + d:= shuf{[16]u8, c, make{[32]i8, idxs>>3 + bit{4, idxs}}} e:= make{[32]u8, 1< { @@ -86,9 +86,9 @@ def raw_widen_inds{[k]D, x:[k0]S if k0>=k} = { # : [k*sc]S def add = make{RV, range{k*sc} % sc} if (hasarch{'AARCH64'} and [k]D == [2]u64 and S==u8) { def x16 = undefPromote{[16]u8, x} - (RV~~vec_shuffle{[16]u8, x16, range{16}>>3}<<3) + add + (RV~~shuf{[16]u8, x16, range{16}>>3}<<3) + add } else if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) { - (vec_shuffle{[8]u32, undefPromote{[8]u32, x}, range{8}>>1}<>1}< is | (top &~ tree_fold{&, each{{m, o} => if (bit{o,i}!=0) m else ~m, bits, bi}}), range{vn}} - tup{tree_fold{|, each{vec_shuffle{[16]u8,.,.}, luts, isp}}} + tup{tree_fold{|, each{shuf{[16]u8,.,.}, luts, isp}}} } }}}} def lut_gen{mode, E==u8, nt, ni if hasarch{'AVX2'} and nt<=16 and ni<=32} = tup{16, 32, loader{{TG} => { lut:[32]u8 = [32]u8**TG{[16]u8, 0} - {is:([32]u8)} => tup{vec_shuffle{[16]u8, lut, is}} + {is:([32]u8)} => tup{shuf{[16]u8, lut, is}} }}} def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=32 and ni<=16} = blend_halves{mode, E, 32, 16} @@ -163,7 +163,7 @@ def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=16 and ni<=16} = ble def lut_gen{mode, E==u32, nt, ni if hasarch{'AVX2'} and nt<=8 and ni<=16} = tup{8, 16, loader{{TG} => { def lut = TG{[8]u32, 0} - def me{'idxs', is:([16]u8)} = each{{wis} => tup{wis, vec_shuffle{[8]u32, lut, wis}}, widen_tup{u32,is}} # TODO inline, or properly outline + def me{'idxs', is:([16]u8)} = each{{wis} => tup{wis, shuf{[8]u32, lut, wis}}, widen_tup{u32,is}} # TODO inline, or properly outline def me{is:([16]u8)} = each{{{_,v}}=>v, me{'idxs', is}} }}} @@ -191,8 +191,8 @@ def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='c' and E>=u16} = 0 def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='i' and E==u64 and nt>16} = 0 -def lut16{tab:([16]u8), idxs:([16]u8)} = vec_shuffle{[16]u8, tab, idxs} -def lut16{tab:([16]u8), idxs:([32]u8) if hasarch{'X86_64'}} = vec_shuffle{[16]u8, pair{tab, tab}, idxs} +def lut16{tab:([16]u8), idxs:([16]u8)} = shuf{[16]u8, tab, idxs} +def lut16{tab:([16]u8), idxs:([32]u8) if hasarch{'X86_64'}} = shuf{[16]u8, pair{tab, tab}, idxs} def shuf_u8bits{inds:*u8, ni} = 0 def shuf_u8bits{inds:*u8, ni if has_sel} = { diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 3f40ce0f..f492361b 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -103,7 +103,7 @@ rcsh4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} def read_shuf_vecs{l, ellw:(u64), shp:*V} = { # tuple of byte selectors in 1<>1 - def fs{v, s} = gen{vec_shuffle{[16]i8, v, s}} - a := vec_shuffle{u64, x, 0,1,0,1}; each{fs{a,.}, slice{sh,0,h}} + def fs{v, s} = gen{shuf{[16]i8, v, s}} + a := shuf{u64, x, 0,1,0,1}; each{fs{a,.}, slice{sh,0,h}} if (l%2) fs{x, select{sh, h}} - b := vec_shuffle{u64, x, 2,3,2,3}; each{fs{b,.}, slice{sh,-h}} + b := shuf{u64, x, 2,3,2,3}; each{fs{b,.}, slice{sh,-h}} } def get_rep_iter{V, wv==2}{x, gen} = { - def s = vec_shuffle{u64, x, 0,2,1,3} + def s = shuf{u64, x, 0,2,1,3} each{{q}=>gen{V~~q}, mzip128{s, s}} } def get_rep_iter{V==[4]u64, wv} = { def step = 4 def sh = get_shufs{step, wv, wv} - {x, gen} => each{{s}=>gen{vec_shuffle{V, x, s}}, sh} + {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} } def rep_const_shuffle{wv, xv:*V, rv:*V, n:(u64)} = rep_const_shuffle{wv, get_rep_iter{V, wv}, xv, rv, n} @@ -171,7 +171,7 @@ if_inline (hasarch{'AVX2'}) { } else { # has_bytesel_128 def rep_iter_from_sh{sh}{x, gen} = { - each{{s} => gen{vec_shuffle{[16]u8, x, s}}, sh} + each{{s} => gen{shuf{[16]u8, x, s}}, sh} } def rep_const_shuffle{wv==2, xv0:*V=[_]T, rv0:*V, n:(u64)} = { @@ -191,9 +191,9 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { def [step]_ = V # Bytes written def wvb = wv << ellw def hs = (h*step) / wvb # Actual step size in argument elements - def shufbase{i if hasarch{'AVX2'}} = vec_shuffle{u64, load{*V~~(x+i)}, 0,1,0,1} + def shufbase{i if hasarch{'AVX2'}} = shuf{u64, load{*V~~(x+i)}, 0,1,0,1} def shufbase{i if has_bytesel_128} = load{*V~~(x+i)} - def shufrun{a, s} = vec_shuffle{[16]i8, a, s} # happens to be the same across AVX2 & NEON + def shufrun{a, s} = shuf{[16]i8, a, s} # happens to be the same across AVX2 & NEON i:u64 = 0 re := r + n*wvb - h*step @@ -442,18 +442,18 @@ def rep_const_bool_div8{wv, x, r, nw if has_simd} = { run8{{x} => z{z{z{x}}}} } } else { # any_sel - def selH = vec_shuffle{[16]u8, ., .} + def selH = shuf{[16]u8, ., .} def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .} def id{xv} = xv if (wv == 2) { - def init = if (avx2) vec_shuffle{u64, ., 0,2,1,3} else id + def init = if (avx2) shuf{u64, ., 0,2,1,3} else id # Expander for half byte def tabr = makeTab{tr_iota{2*iota{4}} * 2b11} m4 := V**0xf run24{*V~~x, init, {x} => tabr{x & m4}, 4} } else if (wv == 4) { # Unzip 32-bit elements (result lanes) across AVX2 lanes - def pre = if (avx2) vec_shuffle{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id + def pre = if (avx2) shuf{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id def init{xv} = { u:=pre{xv}; zip128{u,u,0} } # Expander for two bits in either bottom or next-to-bottom position def tabr = makeTab{tr_iota{0,4,0,4} * 2b1111} @@ -495,7 +495,7 @@ def swap_elts{x:V=[_]_, el_bytes} = { def wd = min{el_bytes, 8} def l = el_bytes/wd def i = iota{l} - vec_shuffle{ty_u{wd*8}, x, merge{i+l, i}} + shuf{ty_u{wd*8}, x, merge{i+l, i}} } else { def sh = el_bytes*8 xe := re_el{ty_u{2*sh}, V} ~~ x @@ -510,7 +510,7 @@ def extract_modperm_mask{full:W, lane, l} = { def n = (if (f) 256 else 128) / w # and number of elements def o = l / w # Extract [o,2*o) def i = o + iota{n}%o - vec_shuffle{ty_u{w}, if (f) full else lane, i} + shuf{ty_u{w}, if (f) full else lane, i} } def proc_mod_dat{swap_data:W} = { def ww = width{W}; def avx2 = ww==256 @@ -518,7 +518,7 @@ def proc_mod_dat{swap_data:W} = { def lens = reverse{1 << slice{iota{lb{hi}}, lb{lo}}} fold{{x,s}=>s{x}, ., each{get_swap, lens}} } - swap_lane := if (avx2) vec_shuffle{swap_data, 0,1,0,1} else swap_data + swap_lane := if (avx2) shuf{swap_data, 0,1,0,1} else swap_data # Transform width-w units with shifts only def get_shiftperm{data, w} = { dat := data @@ -535,20 +535,20 @@ def proc_mod_dat{swap_data:W} = { # Within-byte transformation def get_byteperm{} = { def V = re_el{u8, W} - def sw_bytes = vec_shuffle{u8, swap_lane, 16**0} + def sw_bytes = shuf{u8, swap_lane, 16**0} m4 := W~~V**0xf t0 := fold{{v,a}=>modperm_shift_step{v,...a}, W~~make{V,iota{vcount{V}}%16}, tup{ tup{4, sw_bytes &~ m4}, tup{2, ({v} => v | v<<4){sw_bytes & W~~V**0xc}} }} t4 := (t0&m4)<<4 | (t0&~m4)>>4 - def selI{v,i} = vec_shuffle{[16]u8, v, V~~i} + def selI{v,i} = shuf{[16]u8, v, V~~i} {xv} => selI{t0, xv & m4} | selI{t4, (xv>>4) & m4} } def {partwidth, partperm} = { def sh{w, d} = tup{w, get_shiftperm{d, w}} if (ww==64) sh{64, swap_data} - else if (not any_sel) sh{32, vec_shuffle{u32, swap_data, 0,0,0,0}} + else if (not any_sel) sh{32, shuf{u32, swap_data, 0,0,0,0}} else tup{8, get_byteperm{}} } # Fill in higher steps @@ -631,7 +631,7 @@ def rep_const_bool_odd_mask4{ def sub_carry{a, c} = match (M) { {[l](u64)} => { ca := if (hasarch{'SSE4.2'} or hasarch{'AARCH64'}) { def S = [l]i64; S~~c > S**0 } - else { cm := [2*l]i32~~c; cm != vec_shuffle{cm, 1,0} } + else { cm := [2*l]i32~~c; cm != shuf{cm, 1,0} } a + M~~ca } {_} => a - promote{u64, c != 0} @@ -640,9 +640,9 @@ def rep_const_bool_odd_mask4{ while (1) { x:M = get_modperm_x{} def vrot1 = ifvec{{x} => { - if (w256{M}) vec_shuffle{x, 3,0,1,2} + if (w256{M}) shuf{x, 3,0,1,2} else if (any_sel) vshl{x, x, vcount{type{x}}-1} - else vec_shuffle{x, 1,0} + else shuf{x, 1,0} }} xo := x<>(64-k)} # Write result word given starting bits @@ -702,8 +702,8 @@ def rep_const_bool_small_odd{W=[wl](u64), max_wv, wv, get_perm_x, output} = { def step{jj, oi, ind, mask} = { def hk = (k-1) / 2 def getv = if (not lanes or jj==hk) ({x}=>x) - else vec_shuffle{[4]u64, ., 2*(jj>hk) + iota{4}%2} - def selx{x, i} = vec_shuffle{[128/ew]ty_u{ew}, getv{x}, i} + else shuf{[4]u64, ., 2*(jj>hk) + iota{4}%2} + def selx{x, i} = shuf{[128/ew]ty_u{ew}, getv{x}, i} b := selx{xv, ind} & make{W, mask} r := (b<> make{[8]u32, 4*tail{1, iota{8}}} - s:= vec_shuffle{16, [32]u8~~b, ii32>>3 + bit{2}} + s:= shuf{16, [32]u8~~b, ii32>>3 + bit{2}} p:= s & make{[32]u8, (1<<(1+tail{2})) - 1} # Prefixes - d:= vec_shuffle{[16]u8, make{[32]u8, merge{sums{4},sums{4}}}, [32]i8~~p} - d+= vec_shuffle{16, d, bit{2}*(1+bit{3}>>2)-1} - d + vec_shuffle{16, d, bit{3}-1} + d:= shuf{[16]u8, make{[32]u8, merge{sums{4},sums{4}}}, [32]i8~~p} + d+= shuf{16, d, bit{2}*(1+bit{3}>>2)-1} + d + shuf{16, d, bit{3}-1} } def step{x:(u32), i, store1} = { d:= sumlanes{x} - if (w==8) d+= [32]u8~~vec_shuffle{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 0,0,1,1} + if (w==8) d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 0,0,1,1} j:= (w/8)*i def out{v, k} = each{out, widen{v}, 2*k+iota{2}} def out{v0:([vl]T), k} = { @@ -341,7 +341,7 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = { @maskedLoop{4} (x in tup{V, xp}, r in tup{V, rp} over promote{u64,nw}) { s := scan_words{x, m} - pc:= c; c = vec_shuffle{-(s>>63), 3,0,1,2} + pc:= c; c = shuf{-(s>>63), 3,0,1,2} r = apply_carry{s, blend{c, pc, 1,0,0,0}, (m-V**1)&~m} m = m>>d4 | m<<(l-d4) } diff --git a/src/singeli/src/scan_common.singeli b/src/singeli/src/scan_common.singeli index 21f6b045..bf703946 100644 --- a/src/singeli/src/scan_common.singeli +++ b/src/singeli/src/scan_common.singeli @@ -20,13 +20,13 @@ def spread{a:[_]T, ...up} = { def toLast{n:VT, up if hasarch{'X86_64'} and w128{VT}} = { def l{v, w} = l{zip{up,v}, 2*w} def l{v, w if hasarch{'SSSE3'}} = sel8{v, up*(16-w/8)+iota{16}%(w/8)} - def l{v, w==32} = vec_shuffle{[4]i32, v, 4**(up*3)} - def l{v, w==64} = vec_shuffle{[2]i64, v, 2** up } + def l{v, w==32} = shuf{[4]i32, v, 4**(up*3)} + def l{v, w==64} = shuf{[2]i64, v, 2** up } l{n, elwidth{VT}} } def toLast{n:VT, up if hasarch{'AVX2'} and w256{VT}} = { if (elwidth{VT}<=32) sel{[8]i32, spread{n,up}, [8]i32**(up*7)} - else vec_shuffle{[4]u64, n, 4**(up*3)} + else shuf{[4]u64, n, 4**(up*3)} } def toLast{n:VT} = toLast{n, 1} @@ -50,17 +50,17 @@ def make_scan_idem{T, op, up} = { (if (up) vec_shift_right_128 else vec_shift_left_128){v, c} | id } def shb{v, k if hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}} - def shb{v, k if k>=32} = vec_shuffle{[4]u32, v, shift{k/32,4}} + def shb{v, k if k>=32} = shuf{[4]u32, v, shift{k/32,4}} def shb{v, k if k==128 and hasarch{'AVX2'}} = { # After lanewise scan, broadcast end of lane 0 to entire lane 1 - vec_shuffle{[8]i32, spread{v,up}, rev{up,3*(3>3} + def byte = shuf{[16]u8, xrev, cw>>3} r = homMask{ty_s{byte << (cw & VU**7)} < VI**0} } } else { if (wl>32 and xl<=16) { - xb:= vec_shuffle{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 0,1,0,1} + xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 0,1,0,1} @maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) { cw:= wrapChk{cw0, xlf, M} - sr = homMask{vec_shuffle{[16]i8, xb, cw}} + sr = homMask{shuf{[16]i8, xb, cw}} } } else { - x:= vec_shuffle{[4]u64, load{*VI ~~ x0}, 0,1,0,1} + x:= shuf{[4]u64, load{*VI ~~ x0}, 0,1,0,1} low:= VI**7 b := VI~~make{[32]u8, 1 << (iota{32} & 7)} @maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) { cw:= wrapChk{cw0, xlf, M} - byte:= vec_shuffle{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)} - mask:= vec_shuffle{[16]i8, b, cw & low} + byte:= shuf{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)} + mask:= shuf{[16]i8, b, cw & low} sr = homMask{(mask & byte) == mask} } } diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 9a46d102..4cd59609 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -26,7 +26,7 @@ def unpack_to{f, l, x} = { pass{l, if (l==1) x else unpack_to{0, l/2, x}} } # Last pass for square kernel packed in halves -def shuf_pass{x} = each{{v} => vec_shuffle{[4]i64, v, 0,2,1,3}, x} +def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 0,2,1,3}, x} # Square kernel where width is a full vector def transpose_square{VT, l, x if hasarch{'AVX2'}} = unpack_to{1, l/2, x} diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli index ec6fbe85..e0849d52 100644 --- a/src/singeli/src/vecfold.singeli +++ b/src/singeli/src/vecfold.singeli @@ -3,8 +3,8 @@ def vfold{F, x:V=[_]T if w128{V} and hasarch{'X86_64'}} = { c:= x def EW = width{T} - if (EW<=64) c = F{c, vec_shuffle{u64, c, 1,0}} - if (EW<=32) c = F{c, vec_shuffle{u32, c, 1,0}} + if (EW<=64) c = F{c, shuf{u64, c, 1,0}} + if (EW<=32) c = F{c, shuf{u32, c, 1,0}} if (EW<=16) c = F{c, vec_shuffle16_lo{c, tup{1,0,3,2}}} if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{T, v}, cast_i{T, v>>8}} } else extract{c, 0}