From c4afe7b329800d386f1e9876aa1e074dd3809d1c Mon Sep 17 00:00:00 2001 From: dzaima Date: Sat, 25 Jan 2025 22:22:41 +0200 Subject: [PATCH] =?UTF-8?q?(bit|top|hom)Blend=20=E2=86=92=20blend=5F(bit|t?= =?UTF-8?q?op|hom)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/README.md | 4 ++-- src/singeli/src/avx.singeli | 4 ++-- src/singeli/src/avx2.singeli | 2 -- src/singeli/src/base.singeli | 8 ++++---- src/singeli/src/bins.singeli | 12 ++++++------ src/singeli/src/bitalign.singeli | 4 ++-- src/singeli/src/bits.singeli | 4 ++-- src/singeli/src/equal.singeli | 4 ++-- src/singeli/src/fold.singeli | 12 ++++++------ src/singeli/src/lut.singeli | 6 +++--- src/singeli/src/neon.singeli | 10 ++++------ src/singeli/src/search.singeli | 2 +- src/singeli/src/select.singeli | 2 +- src/singeli/src/sse.singeli | 9 ++------- src/singeli/src/sse2.singeli | 2 +- src/singeli/src/transpose.singeli | 4 ++-- 16 files changed, 40 insertions(+), 49 deletions(-) diff --git a/src/singeli/README.md b/src/singeli/README.md index 71b0c37b..6ce54da4 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -165,8 +165,8 @@ Homogeneous definitions (i.e. ones with `hom` in their name) assume that each el - `homAny{a:VI} : u1` - whether any element is set - `topAny{a:VI} : u1` - whether all elements have their top bit set - `topAll{a:VI} : u1` - whether any element has its top bit set -- `homBlend{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1` -- `topBlend{f:V, t:V, m:V} : V` - blend by top bit of `m` +- `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1` +- `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m` - `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal) - `homMask{...vs} : uint` - merged mask of `each{homMask,vs}` - `topMask{a:VI} : uint` - integer mask of the top bit of each element diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index b8cc1b4d..7466cee7 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -36,8 +36,8 @@ def widen{T=[k]_, x:X=[l]_ if w256{X} and l>k} = widen{T, half{x,0}} # structural operations -def topBlend{f:T, t:T, m:M if w256{T,32} and w256i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, v2f{m}} -def topBlend{f:T, t:T, m:M if w256{T,64} and w256i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, v2d{m}} +def blend_top{f:T, t:T, m:M if w256i{T,32} and w256i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, v2f{m}} +def blend_top{f:T, t:T, m:M if w256i{T,64} and w256i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, v2d{m}} # shuffles # TODO rename to shuf, move shuf to shuf_imm or something diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 733c1b60..ecda0a3e 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -20,8 +20,6 @@ def blend{([8]u16), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[16]i def blend{([8]u32), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 8]i32~~a, [ 8]i32~~b, m} def blend{([4]u64), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 4]f64~~a, [ 4]f64~~b, m} -def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = blend_top{f, t, m} - def sel{L, x:T, i:I if w256{T} and lvec{L,8,32} and w256{I,32}} = vec_shuffle{L, x, i} def sel{L, x:T, i:I if w256{T} and lvec{L,16,8} and w256{I, 8}} = vec_shuffle{L, x, i} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 8c1907c8..86d4da48 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -177,12 +177,13 @@ def { homAll,homAny,bitAll,bitAny,homMask,homMaskStore,homMaskStoreF,loadBatchBit, loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep, pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow, - topBlend,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper, + blend_top,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper, multishift, } -def bitBlend{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m)) -def blend_hom{f:T, t:T, m:M} = bitBlend{f, t, m} # extended by arch/*/select +# extended by arch/*/select where applicable +def blend_bit{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m)) +def blend_hom{f:T, t:T, m:M} = blend_bit{f, t, m} def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX @@ -243,7 +244,6 @@ def iota{V=[k]_} = make{V, ...iota{k}} def absu{a:[_]_} = ty_u{abs{a}} def shuf = vec_shuffle def vshl = vec_merge_shift_right -def homBlend = blend_hom def floor = __floor def ceil = __ceil diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 110548a1..4478408a 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -50,7 +50,7 @@ if_inline (hasarch{'AVX2'}) { def H = n_h{V} vtop := V**(vl/2) hs := each{dup_half{v, .}, tup{1,0}} - {i} => homBlend{...each{shuf{H,.,i}, hs}, V~~i blend_hom{...each{shuf{H,.,i}, hs}, V~~i 16) { b := V~~(re_el{u16, i0} << (7 - lb{vl/2})) - ind = topBlend{ind, isel{ui1}, b} - if (nu > 32) ind = homBlend{topBlend{...each{isel,ui2}, b}, ind, i0 < V**vl} + ind = blend_top{ind, isel{ui1}, b} + if (nu > 32) ind = blend_hom{blend_top{...each{isel,ui2}, b}, ind, i0 < V**vl} } } store{*V~~(*T~~rp+j), 0, ind} @@ -220,7 +220,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} = # Fill with minimum value at the beginning def pre = if (search) load{w} else (if (up) minvalue else maxvalue){T} wg := *V~~(w-gap) - wv0:= homBlend{load{wg}, V**pre, maskOf{V,gap}} + wv0:= blend_hom{load{wg}, V**pre, maskOf{V,gap}} # For multiple lanes, interleave like transpose def maxstep = lb{maxwn} def lstep = lb{svl} @@ -260,7 +260,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} = def ltx = cmpx{lt}; def eqx = cmpx{==} @unroll (j to klog) { m := s | select{bits,klog-1-j} - s = homBlend{m, s, ltx{selw, m}} + s = blend_hom{m, s, ltx{selw, m}} } r := if (isub==1) s else s>>(lb{isub}+wd-wi) # b records if xv was found; c is added to the index @@ -282,7 +282,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} = } if (r_out) { r -= off - if (prim=='⊐') r = homBlend{U**cast_i{u8,wn}, r, b} + if (prim=='⊐') r = blend_hom{U**cast_i{u8,wn}, r, b} rn := if (T==i8) r else if (T==i16) half{narrow{u8, r}, 0} else extract{re_el{i64, narrow{u8, r}}, 0} diff --git a/src/singeli/src/bitalign.singeli b/src/singeli/src/bitalign.singeli index 80f488a2..dfc0677e 100644 --- a/src/singeli/src/bitalign.singeli +++ b/src/singeli/src/bitalign.singeli @@ -125,7 +125,7 @@ def bitalign{s, 8 if s<7, a:V=[k](u8) if hasarch{'X86_64'}} = { re_el{u8, d|e} # def d = re_el{u8, c >> (8 -s)} # def e = re_el{u8, c >> (16-s*2)} - # homBlend{e, d, make{V, cycle{k, tup{0, 0xff}}}} & V**tail{s} + # blend_hom{e, d, make{V, cycle{k, tup{0, 0xff}}}} & V**tail{s} } def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = { def V16 = re_el{u16,V} @@ -133,7 +133,7 @@ def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = { def b = V16~~new_shuf{a, range{k} - (((range{k}+2)/8)>>0)} def c = blend_units{b, b <<{u64} 4, tup{0,1,1,0}} def d = (c * make{V16, cycle{k/2, 1<> 2 # TODO ofence constant for clang - homBlend{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f + blend_hom{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f } def bitalign{{2,8,s}, 8, G if hasarch{'X86_64'}} = { diff --git a/src/singeli/src/bits.singeli b/src/singeli/src/bits.singeli index 9234657c..1069c64a 100644 --- a/src/singeli/src/bits.singeli +++ b/src/singeli/src/bits.singeli @@ -11,7 +11,7 @@ def bitsel{VL, T, r, bits, e0, e1, len} = { e0v:= VT**e0 e1v:= VT**e1 - @maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{homBlend{e0v, e1v, b}} + @maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{blend_hom{e0v, e1v, b}} } fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = { @@ -31,7 +31,7 @@ fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : vo def bulk = arch_defvw / width{E} def VT = [bulk]E def one = VT**cast_i{E, one0} - @maskedLoop{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = homBlend{zero, one, mask} + @maskedLoop{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = blend_hom{zero, one, mask} } } diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 3bdd7ff2..6444f402 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -29,10 +29,10 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { @maskedLoopPositive{bulk}(M in 'm' over i to l) { wu:= (if (hasarch{'AVX2'}) { cw:= b_getBatchLo{bulk, *u64~~w, i} - topBlend{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}} + blend_top{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}} } else { cw:= loadBatchBit{TU, *u64~~w, i} - homBlend{f0, f1, cw} + blend_hom{f0, f1, cw} }) cx:= load{*TF ~~ x, i} if (anynePositive{wu, cx, M}) return{0} diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 4b5f1073..39bb508c 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -141,7 +141,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = { sii := mu16{li} def swap_ms = if (vl == 16) ({x}=>x) else { ms := (V**16 & sii) == (V**16 &~ iota{V}) - {x} => homBlend{x, shuf{[4]u64, x, 2,3,0,1}, ms} + {x} => blend_hom{x, shuf{[4]u64, x, 2,3,0,1}, ms} } # Blend masks @@ -164,16 +164,16 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = { r := load{xv,0}; ++xv if ((l & 2) != 0) { def {m0, _, m2} = m4s - re := homBlend{load{xv,0}, load{xv,1}, m2} - r = homBlend{re, r, m0} + re := blend_hom{load{xv,0}, load{xv,1}, m2} + r = blend_hom{re, r, m0} xv += 2 } # Then the rest in groups of 4 mh := mgo @for (l/4) { {l0, ...ls} := each{load{xv,.}, iota{4}} - r4 := fold{homBlend, l0, ls, m4s} - r = homBlend{r, r4, mh < V**4} + r4 := fold{blend_hom, l0, ls, m4s} + r = blend_hom{r, r4, mh < V**4} mh -= V**4; xv += 4 } def write{r} = { @@ -195,7 +195,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = { o := V**(u8~~1< { ra = homBlend{shuf{[16]u8, ra, sh}, r, bl} } + {r} => { ra = blend_hom{shuf{[16]u8, ra, sh}, r, bl} } } il := make{V, iota{vl} % 16} # Shuffle to undo interleaving of add_res diff --git a/src/singeli/src/lut.singeli b/src/singeli/src/lut.singeli index 6f3a4e5b..3bd6f383 100644 --- a/src/singeli/src/lut.singeli +++ b/src/singeli/src/lut.singeli @@ -59,17 +59,17 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => { # each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}} # } def me{is:[_](u8) if hasarch{'X86_64'} and E==u32} = { - each{{l,h,m} => topBlend{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}} + each{{l,h,m} => blend_top{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}} } def me{is:[_](u8) if hasarch{'X86_64'} and E==u8} = { def bm = is <<{u16} (7 - lb{nth}) # TODO for outermost bit could do a cmpgt, increasing port diversity - each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}} + each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}} } # def me{is:[_](u8) if hasarch{'AARCH64'} and E==u8} = { # only for one outermost blend # def end = type{is}**(nth-1) # def bm = is > end - # each{{l,h} => homBlend{l,h,bm}, lo{is & end}, hi{is & end}} + # each{{l,h} => blend_hom{l,h,bm}, lo{is & end}, hi{is & end}} # } # TODO xor-ing could still be worth it for lower repeated levels where the index transformation can be deduped; and outermost can do a cmpgt diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 3815fd5e..7d67b172 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -26,11 +26,9 @@ def mulwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b} def mulwHi{a:T,b:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b} def mulw {a:T,b:T if w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}} -def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed -def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) -def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed +def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) +def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<0} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 62125fc1..a0dce71e 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -58,7 +58,7 @@ fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void = def bulk = arch_defvw / width{TI} def VT = [bulk]TI @maskedLoop{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) { - dst = homBlend{src, src + VT**cyc, src < VT**0} + dst = blend_hom{src, src + VT**cyc, src < VT**0} } } else { @for (src in *TI~~src, dst in *TI~~dst over n) dst = tern{src<0, src+cyc, src} diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index e0857302..b143fc97 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -20,10 +20,5 @@ def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16 # mask stuff def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y} -def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m} -def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m} -def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = blend_top{ f , t , m} - - - -### SSE4.2 ### +def blend_top{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128i{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m} +def blend_top{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128i{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 480d52b0..4dd333d3 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -67,7 +67,7 @@ def packQ{a:T,b:T if w128i{T}} = packs{a,b} def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4} def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3} -def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}} +def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}} def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = { def s{v} = s{mzip{v,v,0}} diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 7fc67f9b..9c7db648 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -273,14 +273,14 @@ def rotcol{xs, mg:I} = { def w = length{xs} @unroll (kl to ceil_log2{w}) { def k = 1< cross{shuf{16, x, i}, i} } def tr_quads = match { {1}=>({x}=>x); {2}=>shuf{[4]u64, ., 0,2,1,3} }