(bit|top|hom)Blend → blend_(bit|top|hom)

This commit is contained in:
dzaima 2025-01-25 22:22:41 +02:00
parent 29b2b1c1f5
commit c4afe7b329
16 changed files with 40 additions and 49 deletions

View File

@ -165,8 +165,8 @@ Homogeneous definitions (i.e. ones with `hom` in their name) assume that each el
- `homAny{a:VI} : u1` - whether any element is set - `homAny{a:VI} : u1` - whether any element is set
- `topAny{a:VI} : u1` - whether all elements have their top bit set - `topAny{a:VI} : u1` - whether all elements have their top bit set
- `topAll{a:VI} : u1` - whether any element has its top bit set - `topAll{a:VI} : u1` - whether any element has its top bit set
- `homBlend{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1` - `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1`
- `topBlend{f:V, t:V, m:V} : V` - blend by top bit of `m` - `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m`
- `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal) - `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal)
- `homMask{...vs} : uint` - merged mask of `each{homMask,vs}` - `homMask{...vs} : uint` - merged mask of `each{homMask,vs}`
- `topMask{a:VI} : uint` - integer mask of the top bit of each element - `topMask{a:VI} : uint` - integer mask of the top bit of each element

View File

@ -36,8 +36,8 @@ def widen{T=[k]_, x:X=[l]_ if w256{X} and l>k} = widen{T, half{x,0}}
# structural operations # structural operations
def topBlend{f:T, t:T, m:M if w256{T,32} and w256i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, v2f{m}} def blend_top{f:T, t:T, m:M if w256i{T,32} and w256i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M if w256{T,64} and w256i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, v2d{m}} def blend_top{f:T, t:T, m:M if w256i{T,64} and w256i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, v2d{m}}
# shuffles # shuffles
# TODO rename to shuf, move shuf to shuf_imm or something # TODO rename to shuf, move shuf to shuf_imm or something

View File

@ -20,8 +20,6 @@ def blend{([8]u16), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[16]i
def blend{([8]u32), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 8]i32~~a, [ 8]i32~~b, m} def blend{([8]u32), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 8]i32~~a, [ 8]i32~~b, m}
def blend{([4]u64), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 4]f64~~a, [ 4]f64~~b, m} def blend{([4]u64), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 4]f64~~a, [ 4]f64~~b, m}
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = blend_top{f, t, m}
def sel{L, x:T, i:I if w256{T} and lvec{L,8,32} and w256{I,32}} = vec_shuffle{L, x, i} def sel{L, x:T, i:I if w256{T} and lvec{L,8,32} and w256{I,32}} = vec_shuffle{L, x, i}
def sel{L, x:T, i:I if w256{T} and lvec{L,16,8} and w256{I, 8}} = vec_shuffle{L, x, i} def sel{L, x:T, i:I if w256{T} and lvec{L,16,8} and w256{I, 8}} = vec_shuffle{L, x, i}

View File

@ -177,12 +177,13 @@ def {
homAll,homAny,bitAll,bitAny,homMask,homMaskStore,homMaskStoreF,loadBatchBit, homAll,homAny,bitAll,bitAny,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep, loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow, pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
topBlend,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper, blend_top,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,
multishift, multishift,
} }
def bitBlend{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m)) # extended by arch/*/select where applicable
def blend_hom{f:T, t:T, m:M} = bitBlend{f, t, m} # extended by arch/*/select def blend_bit{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m))
def blend_hom{f:T, t:T, m:M} = blend_bit{f, t, m}
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
@ -243,7 +244,6 @@ def iota{V=[k]_} = make{V, ...iota{k}}
def absu{a:[_]_} = ty_u{abs{a}} def absu{a:[_]_} = ty_u{abs{a}}
def shuf = vec_shuffle def shuf = vec_shuffle
def vshl = vec_merge_shift_right def vshl = vec_merge_shift_right
def homBlend = blend_hom
def floor = __floor def floor = __floor
def ceil = __ceil def ceil = __ceil

View File

@ -50,7 +50,7 @@ if_inline (hasarch{'AVX2'}) {
def H = n_h{V} def H = n_h{V}
vtop := V**(vl/2) vtop := V**(vl/2)
hs := each{dup_half{v, .}, tup{1,0}} hs := each{dup_half{v, .}, tup{1,0}}
{i} => homBlend{...each{shuf{H,.,i}, hs}, V~~i<vtop} {i} => blend_hom{...each{shuf{H,.,i}, hs}, V~~i<vtop}
} }
def getsel{v:V=[8]T if width{T}==32} = shuf{V, v, .} def getsel{v:V=[8]T if width{T}==32} = shuf{V, v, .}
} }
@ -191,8 +191,8 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
ind = isel{ui} ind = isel{ui}
if (nu > 16) { if (nu > 16) {
b := V~~(re_el{u16, i0} << (7 - lb{vl/2})) b := V~~(re_el{u16, i0} << (7 - lb{vl/2}))
ind = topBlend{ind, isel{ui1}, b} ind = blend_top{ind, isel{ui1}, b}
if (nu > 32) ind = homBlend{topBlend{...each{isel,ui2}, b}, ind, i0 < V**vl} if (nu > 32) ind = blend_hom{blend_top{...each{isel,ui2}, b}, ind, i0 < V**vl}
} }
} }
store{*V~~(*T~~rp+j), 0, ind} store{*V~~(*T~~rp+j), 0, ind}
@ -220,7 +220,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
# Fill with minimum value at the beginning # Fill with minimum value at the beginning
def pre = if (search) load{w} else (if (up) minvalue else maxvalue){T} def pre = if (search) load{w} else (if (up) minvalue else maxvalue){T}
wg := *V~~(w-gap) wg := *V~~(w-gap)
wv0:= homBlend{load{wg}, V**pre, maskOf{V,gap}} wv0:= blend_hom{load{wg}, V**pre, maskOf{V,gap}}
# For multiple lanes, interleave like transpose # For multiple lanes, interleave like transpose
def maxstep = lb{maxwn} def maxstep = lb{maxwn}
def lstep = lb{svl} def lstep = lb{svl}
@ -260,7 +260,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
def ltx = cmpx{lt}; def eqx = cmpx{==} def ltx = cmpx{lt}; def eqx = cmpx{==}
@unroll (j to klog) { @unroll (j to klog) {
m := s | select{bits,klog-1-j} m := s | select{bits,klog-1-j}
s = homBlend{m, s, ltx{selw, m}} s = blend_hom{m, s, ltx{selw, m}}
} }
r := if (isub==1) s else s>>(lb{isub}+wd-wi) r := if (isub==1) s else s>>(lb{isub}+wd-wi)
# b records if xv was found; c is added to the index # b records if xv was found; c is added to the index
@ -282,7 +282,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
} }
if (r_out) { if (r_out) {
r -= off r -= off
if (prim=='⊐') r = homBlend{U**cast_i{u8,wn}, r, b} if (prim=='⊐') r = blend_hom{U**cast_i{u8,wn}, r, b}
rn := if (T==i8) r rn := if (T==i8) r
else if (T==i16) half{narrow{u8, r}, 0} else if (T==i16) half{narrow{u8, r}, 0}
else extract{re_el{i64, narrow{u8, r}}, 0} else extract{re_el{i64, narrow{u8, r}}, 0}

View File

@ -125,7 +125,7 @@ def bitalign{s, 8 if s<7, a:V=[k](u8) if hasarch{'X86_64'}} = {
re_el{u8, d|e} re_el{u8, d|e}
# def d = re_el{u8, c >> (8 -s)} # def d = re_el{u8, c >> (8 -s)}
# def e = re_el{u8, c >> (16-s*2)} # def e = re_el{u8, c >> (16-s*2)}
# homBlend{e, d, make{V, cycle{k, tup{0, 0xff}}}} & V**tail{s} # blend_hom{e, d, make{V, cycle{k, tup{0, 0xff}}}} & V**tail{s}
} }
def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = { def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = {
def V16 = re_el{u16,V} def V16 = re_el{u16,V}
@ -133,7 +133,7 @@ def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = {
def b = V16~~new_shuf{a, range{k} - (((range{k}+2)/8)>>0)} def b = V16~~new_shuf{a, range{k} - (((range{k}+2)/8)>>0)}
def c = blend_units{b, b <<{u64} 4, tup{0,1,1,0}} def c = blend_units{b, b <<{u64} 4, tup{0,1,1,0}}
def d = (c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang def d = (c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang
homBlend{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f blend_hom{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f
} }
def bitalign{{2,8,s}, 8, G if hasarch{'X86_64'}} = { def bitalign{{2,8,s}, 8, G if hasarch{'X86_64'}} = {

View File

@ -11,7 +11,7 @@ def bitsel{VL, T, r, bits, e0, e1, len} = {
e0v:= VT**e0 e0v:= VT**e0
e1v:= VT**e1 e1v:= VT**e1
@maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{homBlend{e0v, e1v, b}} @maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{blend_hom{e0v, e1v, b}}
} }
fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = { fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
@ -31,7 +31,7 @@ fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : vo
def bulk = arch_defvw / width{E} def bulk = arch_defvw / width{E}
def VT = [bulk]E def VT = [bulk]E
def one = VT**cast_i{E, one0} def one = VT**cast_i{E, one0}
@maskedLoop{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = homBlend{zero, one, mask} @maskedLoop{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = blend_hom{zero, one, mask}
} }
} }

View File

@ -29,10 +29,10 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
@maskedLoopPositive{bulk}(M in 'm' over i to l) { @maskedLoopPositive{bulk}(M in 'm' over i to l) {
wu:= (if (hasarch{'AVX2'}) { wu:= (if (hasarch{'AVX2'}) {
cw:= b_getBatchLo{bulk, *u64~~w, i} cw:= b_getBatchLo{bulk, *u64~~w, i}
topBlend{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}} blend_top{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}}
} else { } else {
cw:= loadBatchBit{TU, *u64~~w, i} cw:= loadBatchBit{TU, *u64~~w, i}
homBlend{f0, f1, cw} blend_hom{f0, f1, cw}
}) })
cx:= load{*TF ~~ x, i} cx:= load{*TF ~~ x, i}
if (anynePositive{wu, cx, M}) return{0} if (anynePositive{wu, cx, M}) return{0}

View File

@ -141,7 +141,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
sii := mu16{li} sii := mu16{li}
def swap_ms = if (vl == 16) ({x}=>x) else { def swap_ms = if (vl == 16) ({x}=>x) else {
ms := (V**16 & sii) == (V**16 &~ iota{V}) ms := (V**16 & sii) == (V**16 &~ iota{V})
{x} => homBlend{x, shuf{[4]u64, x, 2,3,0,1}, ms} {x} => blend_hom{x, shuf{[4]u64, x, 2,3,0,1}, ms}
} }
# Blend masks # Blend masks
@ -164,16 +164,16 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
r := load{xv,0}; ++xv r := load{xv,0}; ++xv
if ((l & 2) != 0) { if ((l & 2) != 0) {
def {m0, _, m2} = m4s def {m0, _, m2} = m4s
re := homBlend{load{xv,0}, load{xv,1}, m2} re := blend_hom{load{xv,0}, load{xv,1}, m2}
r = homBlend{re, r, m0} r = blend_hom{re, r, m0}
xv += 2 xv += 2
} }
# Then the rest in groups of 4 # Then the rest in groups of 4
mh := mgo mh := mgo
@for (l/4) { @for (l/4) {
{l0, ...ls} := each{load{xv,.}, iota{4}} {l0, ...ls} := each{load{xv,.}, iota{4}}
r4 := fold{homBlend, l0, ls, m4s} r4 := fold{blend_hom, l0, ls, m4s}
r = homBlend{r, r4, mh < V**4} r = blend_hom{r, r4, mh < V**4}
mh -= V**4; xv += 4 mh -= V**4; xv += 4
} }
def write{r} = { def write{r} = {
@ -195,7 +195,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
o := V**(u8~~1<<el) o := V**(u8~~1<<el)
bl := i & elo < o bl := i & elo < o
sh := i - o sh := i - o
{r} => { ra = homBlend{shuf{[16]u8, ra, sh}, r, bl} } {r} => { ra = blend_hom{shuf{[16]u8, ra, sh}, r, bl} }
} }
il := make{V, iota{vl} % 16} il := make{V, iota{vl} % 16}
# Shuffle to undo interleaving of add_res # Shuffle to undo interleaving of add_res

View File

@ -59,17 +59,17 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
# each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}} # each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}}
# } # }
def me{is:[_](u8) if hasarch{'X86_64'} and E==u32} = { def me{is:[_](u8) if hasarch{'X86_64'} and E==u32} = {
each{{l,h,m} => topBlend{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}} each{{l,h,m} => blend_top{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}}
} }
def me{is:[_](u8) if hasarch{'X86_64'} and E==u8} = { def me{is:[_](u8) if hasarch{'X86_64'} and E==u8} = {
def bm = is <<{u16} (7 - lb{nth}) # TODO for outermost bit could do a cmpgt, increasing port diversity def bm = is <<{u16} (7 - lb{nth}) # TODO for outermost bit could do a cmpgt, increasing port diversity
each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}} each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}}
} }
# def me{is:[_](u8) if hasarch{'AARCH64'} and E==u8} = { # only for one outermost blend # def me{is:[_](u8) if hasarch{'AARCH64'} and E==u8} = { # only for one outermost blend
# def end = type{is}**(nth-1) # def end = type{is}**(nth-1)
# def bm = is > end # def bm = is > end
# each{{l,h} => homBlend{l,h,bm}, lo{is & end}, hi{is & end}} # each{{l,h} => blend_hom{l,h,bm}, lo{is & end}, hi{is & end}}
# } # }
# TODO xor-ing could still be worth it for lower repeated levels where the index transformation can be deduped; and outermost can do a cmpgt # TODO xor-ing could still be worth it for lower repeated levels where the index transformation can be deduped; and outermost can do a cmpgt

View File

@ -30,8 +30,6 @@ def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, n
def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes)) def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
def bitBlend{f:T=[_]_, t:T, m:M=[_]_} = blend_bit{f, t, m}
def addpw { x:T if nveci{T} and elwidth{T}<=32} = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening def addpw { x:T if nveci{T} and elwidth{T}<=32} = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
def addpwa{a:D==el_m{T}, x:T if nveci{T} and elwidth{T}<=32} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate def addpwa{a:D==el_m{T}, x:T if nveci{T} and elwidth{T}<=32} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
@ -178,4 +176,4 @@ def homMaskX{a:T=[k]E if E!=u64} = {
} }
def homMaskStoreF{p:*T, m:M, v:T if nveci{M} and nvec{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}} def homMaskStoreF{p:*T, m:M, v:T if nveci{M} and nvec{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}}

View File

@ -311,7 +311,7 @@ def isI64{x:T=[_](f64) if hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}}
def isI64{x:T=[_](f64) if hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53)) def isI64{x:T=[_](f64) if hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53))
def maskBlend{b:T, x:T, M} = x def maskBlend{b:T, x:T, M} = x
def maskBlend{b:T, x:T, M if M{0}} = homBlend{b, x, M{T, 'to homogeneous bits'}} def maskBlend{b:T, x:T, M if M{0}} = blend_hom{b, x, M{T, 'to homogeneous bits'}}
fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
assert{n>0} assert{n>0}

View File

@ -58,7 +58,7 @@ fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void =
def bulk = arch_defvw / width{TI} def bulk = arch_defvw / width{TI}
def VT = [bulk]TI def VT = [bulk]TI
@maskedLoop{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) { @maskedLoop{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
dst = homBlend{src, src + VT**cyc, src < VT**0} dst = blend_hom{src, src + VT**cyc, src < VT**0}
} }
} else { } else {
@for (src in *TI~~src, dst in *TI~~dst over n) dst = tern{src<0, src+cyc, src} @for (src in *TI~~src, dst in *TI~~dst over n) dst = tern{src<0, src+cyc, src}

View File

@ -20,10 +20,5 @@ def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16
# mask stuff # mask stuff
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y} def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m} def blend_top{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128i{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m} def blend_top{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128i{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = blend_top{ f , t , m}
### SSE4.2 ###

View File

@ -67,7 +67,7 @@ def packQ{a:T,b:T if w128i{T}} = packs{a,b}
def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4} def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4}
def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3} def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3}
def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}} def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}}
def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = { def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = {
def s{v} = s{mzip{v,v,0}} def s{v} = s{mzip{v,v,0}}

View File

@ -273,14 +273,14 @@ def rotcol{xs, mg:I} = {
def w = length{xs} def w = length{xs}
@unroll (kl to ceil_log2{w}) { def k = 1<<kl @unroll (kl to ceil_log2{w}) { def k = 1<<kl
def vk = I**k; def m = (mg & vk) == vk def vk = I**k; def m = (mg & vk) == vk
def bl{x,y} = { x = homBlend{x,y,m} } def bl{x,y} = { x = blend_hom{x,y,m} }
x0 := select{xs, 0} x0 := select{xs, 0}
def xord = select{xs, k*iota{w} % w} def xord = select{xs, k*iota{w} % w}
each{bl, xord, shiftleft{xord, tup{x0}}} each{bl, xord, shiftleft{xord, tup{x0}}}
} }
} }
def get_modperm_lane_shuf{c} = { def get_modperm_lane_shuf{c} = {
def cross{s, i} = homBlend{s, shuf{[4]u64, s, 2,3,0,1}, i&c == c} def cross{s, i} = blend_hom{s, shuf{[4]u64, s, 2,3,0,1}, i&c == c}
{x, i} => cross{shuf{16, x, i}, i} {x, i} => cross{shuf{16, x, i}, i}
} }
def tr_quads = match { {1}=>({x}=>x); {2}=>shuf{[4]u64, ., 0,2,1,3} } def tr_quads = match { {1}=>({x}=>x); {2}=>shuf{[4]u64, ., 0,2,1,3} }