(bit|top|hom)Blend → blend_(bit|top|hom)
This commit is contained in:
parent
29b2b1c1f5
commit
c4afe7b329
@ -165,8 +165,8 @@ Homogeneous definitions (i.e. ones with `hom` in their name) assume that each el
|
||||
- `homAny{a:VI} : u1` - whether any element is set
|
||||
- `topAny{a:VI} : u1` - whether all elements have their top bit set
|
||||
- `topAll{a:VI} : u1` - whether any element has its top bit set
|
||||
- `homBlend{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1`
|
||||
- `topBlend{f:V, t:V, m:V} : V` - blend by top bit of `m`
|
||||
- `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1`
|
||||
- `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m`
|
||||
- `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal)
|
||||
- `homMask{...vs} : uint` - merged mask of `each{homMask,vs}`
|
||||
- `topMask{a:VI} : uint` - integer mask of the top bit of each element
|
||||
|
||||
@ -36,8 +36,8 @@ def widen{T=[k]_, x:X=[l]_ if w256{X} and l>k} = widen{T, half{x,0}}
|
||||
|
||||
|
||||
# structural operations
|
||||
def topBlend{f:T, t:T, m:M if w256{T,32} and w256i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M if w256{T,64} and w256i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, v2d{m}}
|
||||
def blend_top{f:T, t:T, m:M if w256i{T,32} and w256i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, v2f{m}}
|
||||
def blend_top{f:T, t:T, m:M if w256i{T,64} and w256i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, v2d{m}}
|
||||
|
||||
# shuffles
|
||||
# TODO rename to shuf, move shuf to shuf_imm or something
|
||||
|
||||
@ -20,8 +20,6 @@ def blend{([8]u16), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[16]i
|
||||
def blend{([8]u32), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 8]i32~~a, [ 8]i32~~b, m}
|
||||
def blend{([4]u64), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 4]f64~~a, [ 4]f64~~b, m}
|
||||
|
||||
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = blend_top{f, t, m}
|
||||
|
||||
def sel{L, x:T, i:I if w256{T} and lvec{L,8,32} and w256{I,32}} = vec_shuffle{L, x, i}
|
||||
def sel{L, x:T, i:I if w256{T} and lvec{L,16,8} and w256{I, 8}} = vec_shuffle{L, x, i}
|
||||
|
||||
|
||||
@ -177,12 +177,13 @@ def {
|
||||
homAll,homAny,bitAll,bitAny,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
|
||||
pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
|
||||
topBlend,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,
|
||||
blend_top,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,
|
||||
multishift,
|
||||
}
|
||||
|
||||
def bitBlend{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m))
|
||||
def blend_hom{f:T, t:T, m:M} = bitBlend{f, t, m} # extended by arch/*/select
|
||||
# extended by arch/*/select where applicable
|
||||
def blend_bit{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m))
|
||||
def blend_hom{f:T, t:T, m:M} = blend_bit{f, t, m}
|
||||
|
||||
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
|
||||
def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
|
||||
@ -243,7 +244,6 @@ def iota{V=[k]_} = make{V, ...iota{k}}
|
||||
def absu{a:[_]_} = ty_u{abs{a}}
|
||||
def shuf = vec_shuffle
|
||||
def vshl = vec_merge_shift_right
|
||||
def homBlend = blend_hom
|
||||
|
||||
def floor = __floor
|
||||
def ceil = __ceil
|
||||
|
||||
@ -50,7 +50,7 @@ if_inline (hasarch{'AVX2'}) {
|
||||
def H = n_h{V}
|
||||
vtop := V**(vl/2)
|
||||
hs := each{dup_half{v, .}, tup{1,0}}
|
||||
{i} => homBlend{...each{shuf{H,.,i}, hs}, V~~i<vtop}
|
||||
{i} => blend_hom{...each{shuf{H,.,i}, hs}, V~~i<vtop}
|
||||
}
|
||||
def getsel{v:V=[8]T if width{T}==32} = shuf{V, v, .}
|
||||
}
|
||||
@ -191,8 +191,8 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
|
||||
ind = isel{ui}
|
||||
if (nu > 16) {
|
||||
b := V~~(re_el{u16, i0} << (7 - lb{vl/2}))
|
||||
ind = topBlend{ind, isel{ui1}, b}
|
||||
if (nu > 32) ind = homBlend{topBlend{...each{isel,ui2}, b}, ind, i0 < V**vl}
|
||||
ind = blend_top{ind, isel{ui1}, b}
|
||||
if (nu > 32) ind = blend_hom{blend_top{...each{isel,ui2}, b}, ind, i0 < V**vl}
|
||||
}
|
||||
}
|
||||
store{*V~~(*T~~rp+j), 0, ind}
|
||||
@ -220,7 +220,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
|
||||
# Fill with minimum value at the beginning
|
||||
def pre = if (search) load{w} else (if (up) minvalue else maxvalue){T}
|
||||
wg := *V~~(w-gap)
|
||||
wv0:= homBlend{load{wg}, V**pre, maskOf{V,gap}}
|
||||
wv0:= blend_hom{load{wg}, V**pre, maskOf{V,gap}}
|
||||
# For multiple lanes, interleave like transpose
|
||||
def maxstep = lb{maxwn}
|
||||
def lstep = lb{svl}
|
||||
@ -260,7 +260,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
|
||||
def ltx = cmpx{lt}; def eqx = cmpx{==}
|
||||
@unroll (j to klog) {
|
||||
m := s | select{bits,klog-1-j}
|
||||
s = homBlend{m, s, ltx{selw, m}}
|
||||
s = blend_hom{m, s, ltx{selw, m}}
|
||||
}
|
||||
r := if (isub==1) s else s>>(lb{isub}+wd-wi)
|
||||
# b records if xv was found; c is added to the index
|
||||
@ -282,7 +282,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
|
||||
}
|
||||
if (r_out) {
|
||||
r -= off
|
||||
if (prim=='⊐') r = homBlend{U**cast_i{u8,wn}, r, b}
|
||||
if (prim=='⊐') r = blend_hom{U**cast_i{u8,wn}, r, b}
|
||||
rn := if (T==i8) r
|
||||
else if (T==i16) half{narrow{u8, r}, 0}
|
||||
else extract{re_el{i64, narrow{u8, r}}, 0}
|
||||
|
||||
@ -125,7 +125,7 @@ def bitalign{s, 8 if s<7, a:V=[k](u8) if hasarch{'X86_64'}} = {
|
||||
re_el{u8, d|e}
|
||||
# def d = re_el{u8, c >> (8 -s)}
|
||||
# def e = re_el{u8, c >> (16-s*2)}
|
||||
# homBlend{e, d, make{V, cycle{k, tup{0, 0xff}}}} & V**tail{s}
|
||||
# blend_hom{e, d, make{V, cycle{k, tup{0, 0xff}}}} & V**tail{s}
|
||||
}
|
||||
def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = {
|
||||
def V16 = re_el{u16,V}
|
||||
@ -133,7 +133,7 @@ def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = {
|
||||
def b = V16~~new_shuf{a, range{k} - (((range{k}+2)/8)>>0)}
|
||||
def c = blend_units{b, b <<{u64} 4, tup{0,1,1,0}}
|
||||
def d = (c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang
|
||||
homBlend{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f
|
||||
blend_hom{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f
|
||||
}
|
||||
|
||||
def bitalign{{2,8,s}, 8, G if hasarch{'X86_64'}} = {
|
||||
|
||||
@ -11,7 +11,7 @@ def bitsel{VL, T, r, bits, e0, e1, len} = {
|
||||
|
||||
e0v:= VT**e0
|
||||
e1v:= VT**e1
|
||||
@maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{homBlend{e0v, e1v, b}}
|
||||
@maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{blend_hom{e0v, e1v, b}}
|
||||
}
|
||||
|
||||
fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
|
||||
@ -31,7 +31,7 @@ fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : vo
|
||||
def bulk = arch_defvw / width{E}
|
||||
def VT = [bulk]E
|
||||
def one = VT**cast_i{E, one0}
|
||||
@maskedLoop{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = homBlend{zero, one, mask}
|
||||
@maskedLoop{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = blend_hom{zero, one, mask}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -29,10 +29,10 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
@maskedLoopPositive{bulk}(M in 'm' over i to l) {
|
||||
wu:= (if (hasarch{'AVX2'}) {
|
||||
cw:= b_getBatchLo{bulk, *u64~~w, i}
|
||||
topBlend{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}}
|
||||
blend_top{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}}
|
||||
} else {
|
||||
cw:= loadBatchBit{TU, *u64~~w, i}
|
||||
homBlend{f0, f1, cw}
|
||||
blend_hom{f0, f1, cw}
|
||||
})
|
||||
cx:= load{*TF ~~ x, i}
|
||||
if (anynePositive{wu, cx, M}) return{0}
|
||||
|
||||
@ -141,7 +141,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
|
||||
sii := mu16{li}
|
||||
def swap_ms = if (vl == 16) ({x}=>x) else {
|
||||
ms := (V**16 & sii) == (V**16 &~ iota{V})
|
||||
{x} => homBlend{x, shuf{[4]u64, x, 2,3,0,1}, ms}
|
||||
{x} => blend_hom{x, shuf{[4]u64, x, 2,3,0,1}, ms}
|
||||
}
|
||||
|
||||
# Blend masks
|
||||
@ -164,16 +164,16 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
|
||||
r := load{xv,0}; ++xv
|
||||
if ((l & 2) != 0) {
|
||||
def {m0, _, m2} = m4s
|
||||
re := homBlend{load{xv,0}, load{xv,1}, m2}
|
||||
r = homBlend{re, r, m0}
|
||||
re := blend_hom{load{xv,0}, load{xv,1}, m2}
|
||||
r = blend_hom{re, r, m0}
|
||||
xv += 2
|
||||
}
|
||||
# Then the rest in groups of 4
|
||||
mh := mgo
|
||||
@for (l/4) {
|
||||
{l0, ...ls} := each{load{xv,.}, iota{4}}
|
||||
r4 := fold{homBlend, l0, ls, m4s}
|
||||
r = homBlend{r, r4, mh < V**4}
|
||||
r4 := fold{blend_hom, l0, ls, m4s}
|
||||
r = blend_hom{r, r4, mh < V**4}
|
||||
mh -= V**4; xv += 4
|
||||
}
|
||||
def write{r} = {
|
||||
@ -195,7 +195,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
|
||||
o := V**(u8~~1<<el)
|
||||
bl := i & elo < o
|
||||
sh := i - o
|
||||
{r} => { ra = homBlend{shuf{[16]u8, ra, sh}, r, bl} }
|
||||
{r} => { ra = blend_hom{shuf{[16]u8, ra, sh}, r, bl} }
|
||||
}
|
||||
il := make{V, iota{vl} % 16}
|
||||
# Shuffle to undo interleaving of add_res
|
||||
|
||||
@ -59,17 +59,17 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
|
||||
# each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}}
|
||||
# }
|
||||
def me{is:[_](u8) if hasarch{'X86_64'} and E==u32} = {
|
||||
each{{l,h,m} => topBlend{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}}
|
||||
each{{l,h,m} => blend_top{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}}
|
||||
}
|
||||
|
||||
def me{is:[_](u8) if hasarch{'X86_64'} and E==u8} = {
|
||||
def bm = is <<{u16} (7 - lb{nth}) # TODO for outermost bit could do a cmpgt, increasing port diversity
|
||||
each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}}
|
||||
each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}}
|
||||
}
|
||||
# def me{is:[_](u8) if hasarch{'AARCH64'} and E==u8} = { # only for one outermost blend
|
||||
# def end = type{is}**(nth-1)
|
||||
# def bm = is > end
|
||||
# each{{l,h} => homBlend{l,h,bm}, lo{is & end}, hi{is & end}}
|
||||
# each{{l,h} => blend_hom{l,h,bm}, lo{is & end}, hi{is & end}}
|
||||
# }
|
||||
|
||||
# TODO xor-ing could still be worth it for lower repeated levels where the index transformation can be deduped; and outermost can do a cmpgt
|
||||
|
||||
@ -26,11 +26,9 @@ def mulwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b}
|
||||
def mulwHi{a:T,b:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b}
|
||||
def mulw {a:T,b:T if w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}}
|
||||
|
||||
def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed
|
||||
def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
|
||||
def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
|
||||
|
||||
def bitBlend{f:T=[_]_, t:T, m:M=[_]_} = blend_bit{f, t, m}
|
||||
def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed
|
||||
def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
|
||||
def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
|
||||
|
||||
def addpw { x:T if nveci{T} and elwidth{T}<=32} = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
|
||||
def addpwa{a:D==el_m{T}, x:T if nveci{T} and elwidth{T}<=32} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
|
||||
@ -178,4 +176,4 @@ def homMaskX{a:T=[k]E if E!=u64} = {
|
||||
}
|
||||
|
||||
|
||||
def homMaskStoreF{p:*T, m:M, v:T if nveci{M} and nvec{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
def homMaskStoreF{p:*T, m:M, v:T if nveci{M} and nvec{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}}
|
||||
|
||||
@ -311,7 +311,7 @@ def isI64{x:T=[_](f64) if hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}}
|
||||
def isI64{x:T=[_](f64) if hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53))
|
||||
|
||||
def maskBlend{b:T, x:T, M} = x
|
||||
def maskBlend{b:T, x:T, M if M{0}} = homBlend{b, x, M{T, 'to homogeneous bits'}}
|
||||
def maskBlend{b:T, x:T, M if M{0}} = blend_hom{b, x, M{T, 'to homogeneous bits'}}
|
||||
|
||||
fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
|
||||
assert{n>0}
|
||||
|
||||
@ -58,7 +58,7 @@ fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void =
|
||||
def bulk = arch_defvw / width{TI}
|
||||
def VT = [bulk]TI
|
||||
@maskedLoop{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
|
||||
dst = homBlend{src, src + VT**cyc, src < VT**0}
|
||||
dst = blend_hom{src, src + VT**cyc, src < VT**0}
|
||||
}
|
||||
} else {
|
||||
@for (src in *TI~~src, dst in *TI~~dst over n) dst = tern{src<0, src+cyc, src}
|
||||
|
||||
@ -20,10 +20,5 @@ def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16
|
||||
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = blend_top{ f , t , m}
|
||||
|
||||
|
||||
|
||||
### SSE4.2 ###
|
||||
def blend_top{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128i{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m}
|
||||
def blend_top{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128i{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m}
|
||||
|
||||
@ -67,7 +67,7 @@ def packQ{a:T,b:T if w128i{T}} = packs{a,b}
|
||||
def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4}
|
||||
def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3}
|
||||
|
||||
def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}}
|
||||
|
||||
def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = {
|
||||
def s{v} = s{mzip{v,v,0}}
|
||||
|
||||
@ -273,14 +273,14 @@ def rotcol{xs, mg:I} = {
|
||||
def w = length{xs}
|
||||
@unroll (kl to ceil_log2{w}) { def k = 1<<kl
|
||||
def vk = I**k; def m = (mg & vk) == vk
|
||||
def bl{x,y} = { x = homBlend{x,y,m} }
|
||||
def bl{x,y} = { x = blend_hom{x,y,m} }
|
||||
x0 := select{xs, 0}
|
||||
def xord = select{xs, k*iota{w} % w}
|
||||
each{bl, xord, shiftleft{xord, tup{x0}}}
|
||||
}
|
||||
}
|
||||
def get_modperm_lane_shuf{c} = {
|
||||
def cross{s, i} = homBlend{s, shuf{[4]u64, s, 2,3,0,1}, i&c == c}
|
||||
def cross{s, i} = blend_hom{s, shuf{[4]u64, s, 2,3,0,1}, i&c == c}
|
||||
{x, i} => cross{shuf{16, x, i}, i}
|
||||
}
|
||||
def tr_quads = match { {1}=>({x}=>x); {2}=>shuf{[4]u64, ., 0,2,1,3} }
|
||||
|
||||
Loading…
Reference in New Issue
Block a user