From b6578e43a1f2623ce9b8d3d829d7c15373547f7e Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 2 Feb 2025 19:01:08 +0200 Subject: [PATCH] renames of the rest of base.singeli --- src/singeli/README.md | 18 +++++++----------- src/singeli/src/avx.singeli | 10 +++++----- src/singeli/src/avx2.singeli | 4 ++-- src/singeli/src/base.singeli | 24 +++++++++++------------- src/singeli/src/cmp.singeli | 4 ++-- src/singeli/src/copy.singeli | 2 +- src/singeli/src/lut.singeli | 4 ++-- src/singeli/src/mask.singeli | 2 +- src/singeli/src/neon.singeli | 8 ++++---- src/singeli/src/scan.singeli | 2 +- src/singeli/src/search.singeli | 2 +- src/singeli/src/select.singeli | 2 +- src/singeli/src/slash.singeli | 6 +++--- src/singeli/src/squeeze.singeli | 12 ++++++------ src/singeli/src/sse2.singeli | 4 ++-- 15 files changed, 49 insertions(+), 55 deletions(-) diff --git a/src/singeli/README.md b/src/singeli/README.md index f9399aa2..4b77be2b 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -43,19 +43,17 @@ The SIMD operations listed aren't guaranteed to be supported on all targets, nor ## Loops & branches - `@for` - generate body once, with indices `s,s+1,...,e-2,e-1` -- `@forNZ` - `@for`, but assumed to run at least once -- `@for_backwards` - `@for`, but indices go backwards `e-1,e-2,...,s+1,s` -- `@forUnroll{exp, unr}` - generate a main loop body that takes a tuple of `unr` indices to process. Handle the tail via processing one index at a time; if `exp==1`, that's done via `unr` generated bodies, otherwise it's a loop +- `@for_nz` - `@for`, but assumed to run at least once +- `@for_unroll_block{exp, unr}` - generate a main loop body that takes a tuple of `unr` indices to process. Handle the tail via processing one index at a time; if `exp==1`, that's done via `unr` generated bodies, otherwise it's a loop - `@unroll(n)` - generate body `n` times -- `@collect(n)` - generate body `n` times, and collect the results as a tuple -- `makeBranch{Ts, G=={...x:Ts}=>{}} == {...x:Ts}=>{}` - make a code segment that takes arguments of types `Ts`, and can be jumped to by invoking the result of the `makeBranch` -- `makeOptBranch{enable, Ts, G}` - `makeBranch` but only generated & invokable when `enable==1` +- `make_branch{Ts, G=={...x:Ts}=>{}} == {...x:Ts}=>{}` - make a code segment that takes arguments of types `Ts`, and can be jumped to by invoking the result of the `make_branch` +- `make_opt_ranch{enable, Ts, G}` - `make_branch` but only generated & invokable when `enable==1` ## Scalar operations - `popc{x:EI} : uint` - population count -- `popcRand{x:EI} : uint` - population count; under valgrind, return a random value in the set of possible ones if the input is partly undefined +- `popc_rand{x:EI} : uint` - population count; under valgrind, return a random value in the set of possible ones if the input is partly undefined - `ctz{x:EI} : uint` - count trailing zeroes; if `x==0`, the result is undefined - `clz{x:EI} : uint` - count leading zeroes; if `x==0`, the result is undefined - `clzc{x:EI} : uint` - `width{EI}-clz{x}`; if `x==0`, the result is undefined @@ -142,7 +140,7 @@ Some may also support one scalar argument or arguments with different widths. - `insert{a:V, i, b:eltype{V}} : V` - a copy of `a` with the `i`-th element replaced with `b` - `half{a:V, n} : count_half{V}` - extract either the low (`n==0`) or high (`n==1`) half of `a` - `pair{a:V, b:V} : count_dbl{V}` - make a vector from halves -- `undefPromote{V2, a:V} : V` - promote `a` to a wider type, leaving the upper part undefined +- `undef_promote{V2, a:V} : V` - promote `a` to a wider type, leaving the upper part undefined - `vshl{a:V, b:V, n} : V` - `vshl{[0,1,2,3], [4,5,6,7], 1}` → `[1,2,3,4]` - `sel{VI1, a:V, b:VI2} : V2` - shuffle `a` by indices `b` in `VI1`-long lanes; arch-specific behavior on out-of-bounds values @@ -234,9 +232,7 @@ For float conversions, the used rounding mode is unspecified. - `addp` - add pairwise - `addpw` - add pairwise, widening - `addpwa` - add pairwise, widening; accumulate -- `addwLo` - widening add -- `subwLo` - widening subtract -- `mulwLo`, `mulwHi`, `mulw` - widening multiply +- `addw`, `subw`, `mulw` - widening add/subtract/multiply - `andnz` - element-wise (a&b)!=0? ~0 : 0 - `clz` - count leading zeroes diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 2b68ccdd..5f975a0b 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -3,11 +3,11 @@ def v2i{x:T=[_]E if w256{T}} = if(isint{E}) x else [32]u8 ~~ x def v2f{x:T if w256{T}} = [8]f32 ~~ x def v2d{x:T if w256{T}} = [4]f64 ~~ x -def undefPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} -def zeroPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_zextsi128_si256', v2i{x}} +def undef_promote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} +def zero_promote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_zextsi128_si256', v2i{x}} # load & store -def load_low{ptr:*V, w if w256{V} and w<=128} = undefPromote{V, load_low{*n_h{V} ~~ ptr, w}} +def load_low{ptr:*V, w if w256{V} and w<=128} = undef_promote{V, load_low{*n_h{V} ~~ ptr, w}} def load_low{ptr:*V, w if w256{V} and w==256} = load{ptr} def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = store_low{ptr, w, half{x, 0}} @@ -41,10 +41,10 @@ def blend_top{f:T, t:T, m:M if w256i{T,64} and w256i{M,64}} = T ~~ blend_top{v2d # shuffles # TODO rename to shuf, move shuf to shuf_imm or something -def shufInd{a:T, b:T=[8]E, {...is} if width{E}==32 and length{is}==8 and ({l,h} => same{l,h-4} and x86_shufps_range{l, 8}){slice{is,0,4}, slice{is,4,8}}} = { +def shuf_ind{a:T, b:T=[8]E, {...is} if width{E}==32 and length{is}==8 and ({l,h} => same{l,h-4} and x86_shufps_range{l, 8}){slice{is,0,4}, slice{is,4,8}}} = { vec_shuffle{[4]f32, tup{a, b}, slice{is,0,4} & 3} } -def shufInd{a:T, b:T=[4]E, {...is} if width{E}==64 and length{is}==4} = T~~shufInd{re_el{u32,a}, re_el{u32,b}, merge{...each{{i} => tup{i*2, i*2+1}, is}}} +def shuf_ind{a:T, b:T=[4]E, {...is} if width{E}==64 and length{is}==4} = T~~shuf_ind{re_el{u32,a}, re_el{u32,b}, merge{...each{{i} => tup{i*2, i*2+1}, is}}} # mask stuff def and_bit_none{x:T, y:T if w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 3f2e6e17..2b7502c0 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -91,8 +91,8 @@ def unzip128{a:[k]E, b:[k]E, 1 if hasarch{'X86_64'} and isunsigned{E}} = { def ew = width{E} packQ{re_el{ED,a} >> ew, re_el{ED,b} >> ew} } -def unzip128{a:T, b:T==[8](u32), k} = shufInd{a, b, tup{0,2,8,10, 4,6,12,14}+k} -def unzip128{a:T, b:T==[4](u64), k} = shufInd{a, b, tup{0,4,2,6}+k} +def unzip128{a:T, b:T==[8](u32), k} = shuf_ind{a, b, tup{0,2,8,10, 4,6,12,14}+k} +def unzip128{a:T, b:T==[4](u64), k} = shuf_ind{a, b, tup{0,4,2,6}+k} def unzip{a:T, b:T, k if w128u{T} and hasarch{'SSE2'}} = unzip128{a, b, k} def unzip{a:T, b:T, k if w256u{T} and hasarch{'AVX2'}} = vec_shuffle{[4]u64, unzip128{a, b, k}, 0,2,1,3} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 1f40855f..7fda4d40 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -176,7 +176,7 @@ def { all_hom,any_hom,blend_hom,hom_to_int,store_masked_hom,store_blended_hom, all_top,any_top,blend_top,top_to_int,store_masked_top,store_blended_top, load_expand_bits,load_low,make,mask_to_hom,mulw_split,mulh,narrow,narrow_trunc,narrow_pair, - pair,pdep,pext,rbit,sel,shufInd,store_low,reverse_units, + pair,pdep,pext,rbit,sel,shuf_ind,store_low,reverse_units, unord,unzip,vfold,vec_select,vec_shuffle,widen,widen_upper,multishift, } @@ -233,8 +233,8 @@ def pair{{a, b}} = pair{a, b} def widen{T, x:T} = x def narrow{T, x:[_]T} = x -def undefPromote{T, x:T} = x -def zeroPromote{T, x:T} = x +def undef_promote{T, x:T} = x +def zero_promote{T, x:T} = x def cvt{T, x:[_]T} = x def broadcast{V=[_]T, v} = vec_broadcast{V, promote{T,v}} @@ -302,21 +302,19 @@ def tr_iota{...bs} = { def tr_iota{{...bs}} = tr_iota{...bs} # range logic -def inRangeLen{x:TS=[_]T, start, count if issigned{T}} = { # ∊ [start;start+count) +def in_range_len{x:TS=[_]T, start, count if issigned{T}} = { # ∊ [start;start+count) def TU = ty_u{TS} (TU~~(x-TS**start)) < TU**count } -def inRangeLen{x:TU=[_]T, start, count if isunsigned{T}} = { # ∊ [start;start+count) +def in_range_len{x:TU=[_]T, start, count if isunsigned{T}} = { # ∊ [start;start+count) def TS = ty_s{TU} def h = 1 << (width{T}-1) (TS~~(x-TU**(start-h))) < TS**(count-h) } -def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1} # ∊ [start;end] -def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start} # ∊ [start;end) # check if all tuple elements are in a given range -def inrange{t, min, max} = tree_fold{&, (t>=min) & (t=min) & (ti+j, iota{unr}}, vars} @@ -372,7 +370,7 @@ def for_blocks{blsz}{vars,begin,end,iter} = { } def for_blocks{0}{vars,begin,end,iter} = iter{tup{begin,end}, vars} -def makeBranch{Ts, F} = { +def make_branch{Ts, F} = { def args = undef{Ts} def skip = makelabel{}; goto{skip} def start = setlabel{} @@ -380,7 +378,7 @@ def makeBranch{Ts, F} = { setlabel{skip} {...vs} => { args = vs; goto{start} } } -def makeOptBranch{enable, Ts, F} = if (enable) makeBranch{Ts, F} else 'not defined' +def make_opt_branch{enable, Ts, F} = if (enable) make_branch{Ts, F} else 'not defined' def undef{T, n if istype{T}} = @collect(n) undef{T} def undef{Ts if istup{Ts}} = each{undef, Ts} diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index 14bbf63e..fc244e64 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -75,7 +75,7 @@ def any2bit{VT=[k]T, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = { } def mask = if (same{op0, op}) hom_to_int else ({x} => ~hom_to_int{x}) - @forNZ (ri to cdiv{len,bulk}) { + @for_nz (ri to cdiv{len,bulk}) { store_bits{bulk, dst, ri, mask{each{{j}=>op{wV{xi+j}, xV{xi+j}}, iota{unr}}}} xi+= unr } @@ -93,7 +93,7 @@ fn as2bit{VT=[_]T, unr, op}(dst:*u64, wr:*void, x:u64, len:ux) : void = { } fn bitAA{bitop}(dst:*u64, wr:*void, xr:*void, len:ux) : void = { - @forNZ (dst, w in *u64~~wr, x in *u64~~xr over cdiv{len,64}) dst = bitop{w,x} + @for_nz (dst, w in *u64~~wr, x in *u64~~xr over cdiv{len,64}) dst = bitop{w,x} } fn bit_not(dst:*u64, x:*u64, len:ux) : void = { am:=cdiv{len,64}; emit{void, 'bit_negatePtr', dst, x, am} } diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index de5487bb..97855549 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -48,7 +48,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = { def unr = if (hasarch{'X86_64'} and width{X}==16) 2 else max{8/bulk, 1} def bulk2 = bulk*unr xi:ux = 0 - @forNZ (i to cdiv{l,bulk2}) { + @for_nz (i to cdiv{l,bulk2}) { store_bits{bulk2, rp, i, hom_to_int{each{{i} => op{loadBatch{xp, xi+i, XV}}, iota{unr}}}} xi+= unr } diff --git a/src/singeli/src/lut.singeli b/src/singeli/src/lut.singeli index 9d7fd95c..ebc8331c 100644 --- a/src/singeli/src/lut.singeli +++ b/src/singeli/src/lut.singeli @@ -85,10 +85,10 @@ def raw_widen_inds{[k]D, x:[k0]S if k0>=k} = { # : [k*sc]S def RV = [k*sc]S def add = make{RV, range{k*sc} % sc} if (hasarch{'AARCH64'} and [k]D == [2]u64 and S==u8) { - def x16 = undefPromote{[16]u8, x} + def x16 = undef_promote{[16]u8, x} (RV~~shuf{[16]u8, x16, range{16}>>3}<<3) + add } else if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) { - (shuf{[8]u32, undefPromote{[8]u32, x}, range{8}>>1}<>1}< width{XE}* def widen{R=[rn]RE, x:X=[xn]XE if w64{X} and isfloat{RE}!=isfloat{XE} and width{RE}>width{XE}} = cvt{RE, widen{[rn]to_w{XE,width{RE}}, x}} def widen{R=[rn]RE, x:X=[xn]XE if w128{X} and xn>rn} = widen{R, half{x,0}} -def narrow {T, x:X=[_]E if w128{X} and eqqi{T,E} and width{T}*2< width{E}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{E}, x}}} +def narrow {T, x:X=[_]E if w128{X} and eqqi{T,E} and width{T}*2< width{E}} = narrow{T, undef_promote{el_s{X}, narrow{w_h{E}, x}}} def narrow_trunc{T, x:X=[_]E if w128{X} and eqqi{T,E} and width{T}*2==width{E}} = emit{el_h{X}, ntyp0{'vmovn', X}, x} def narrow {T, x:X=[_]E if w128{X} and eqqi{T,E} and width{T}*2==width{E}} = narrow_trunc{T, x} def narrow {T, x:X=[_]E if w128{X} and isfloat{T}!=isfloat{E} and width{T} { def {b,n} = addChk{c, promote{R, v}} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index c6f2cd47..ae9b51fa 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -21,7 +21,7 @@ def findFirst{C, M, F, ...v1} = { def search{E, x, n:(u64), OP} = { def bulk = arch_defvw/width{E} def VT = [bulk]E - def end = makeBranch{ + def end = make_branch{ tup{u64, ty_u{VT}}, {i,c} => return{i*bulk + promote{u64, ctz_ext{hom_to_int_ext{c}}}} } diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index f7486789..a8144922 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -32,7 +32,7 @@ def wrapChk{cw0:VI, xlf, M} = { def masked_multistore{r0, vs, M, end} = { # returns bumped-forwards r r:= r0 def left = if (M{0}) { left:ux = M{'count'} } else 0 - def lastMaskedStore = makeOptBranch{M{0}, tup{one_type{vs}}, {c} => { + def lastMaskedStore = make_opt_branch{M{0}, tup{one_type{vs}}, {c} => { storeBatch{r, 0, c, maskAfter{left}} end{} }} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index be79c506..d3df3e5f 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -5,8 +5,8 @@ if_inline (hasarch{'X86_64'}) { } include './mask' -def popcRand{x:T if isint{T} and width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones -def popcRand{x:T if isint{T} and width{T}<=32} = emit{u8, 'rand_popc64', x} +def popc_rand{x:T if isint{T} and width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones +def popc_rand{x:T if isint{T} and width{T}<=32} = emit{u8, 'rand_popc64', x} # Table from l bits to w-bit indices, shifted left by s, and G applied afterwards def maketab{l,w,s,G} = { @@ -226,7 +226,7 @@ fn slash{c, T if hasarch{'AVX2'} and T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, s } else { h := w&0xf step{r, h} - step{r+popcRand{h}, w>>4} + step{r+popc_rand{h}, w>>4} } r += pc } diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 664dbb1b..db71b89f 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -3,7 +3,7 @@ include './mask' include './cbqnDefs' include './vecfold' -def is_sNaN{x:[_](u64)} = tup{any_hom, inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2}} +def is_sNaN{x:[_](u64)} = tup{any_hom, in_range_len{x<<1, (0xFFE<<52)+2, (1<<52)-2}} def is_sNaN{x:[2](u64) if hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = { # avoiding i64 comparisons def nan = unord{[2]f64~~x, [2]f64~~x} def qnan = re_el{u64, [4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000)} @@ -14,7 +14,7 @@ def any_sNaN{M, ...xs} = { one_val{any}{M{tree_fold{|, are}}} } -def any_nonC32{M, x:[_](u64)} = any_hom{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} +def any_nonC32{M, x:[_](u64)} = any_hom{M{~in_range_len{x, cbqn_c32Tag{}<<48, 1<<48}}} def any_nonC32{M, x:T=[_]_ if hasarch{'X86_64'}} = { def H = re_el{u32, T} def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16} @@ -22,7 +22,7 @@ def any_nonC32{M, x:T=[_]_ if hasarch{'X86_64'}} = { } def any_nonC32{(maskNone), x:[k](u64), y:[k](u64)} = { def T32 = [k*2]u32 - def hi = shufInd{T32~~x, T32~~y, match(k) { + def hi = shuf_ind{T32~~x, T32~~y, match(k) { {2} => tup{1,3,5,7} {4} => tup{1,3,9,11,5,7,13,15} # all odd indices, in the order that vshufps can handle }} @@ -54,8 +54,8 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { maxv = max{maxv, M{maxc}} } def me{M, vs} = { - minc:= zeroPromote{T, tree_fold{min, vs}} # could pack pairs in v to low & high halves, but an extra min costs the same or less than an insert - maxc:= zeroPromote{T, tree_fold{max, vs}} + minc:= zero_promote{T, tree_fold{min, vs}} # could pack pairs in v to low & high halves, but an extra min costs the same or less than an insert + maxc:= zero_promote{T, tree_fold{max, vs}} me{M, minc, maxc} } @@ -119,7 +119,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { } 0 } else if (X==f64) { - def case_B = makeBranch{tup{ux}, {bulkCont} => { + def case_B = make_branch{tup{ux}, {bulkCont} => { def i0 = bulkCont*bulk x:= i0 + *u64~~x0 if (B) @muLoop{bulk, 2}(xs in tup{[bulk]u64, x}, M in 'm' over len-i0) { diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 7efaae86..926afb0f 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -64,8 +64,8 @@ def packs{a:T,b:T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b} def packs{a:T,b:T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b} def packQ{a:T,b:T if w128i{T}} = packs{a,b} -def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4} -def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3} +def x86_shufps_range{is, hi} = in_range{is,0,2, 0,4} and in_range{is,2,4, hi,hi+4} +def shuf_ind{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = vec_shuffle{[4]f32, tup{a, b}, is&3} def store_blended_hom{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}}