From 426c1b4bf8d07b0f613e5fb793f8a00d1cee3c2d Mon Sep 17 00:00:00 2001 From: dzaima Date: Sat, 25 Jan 2025 23:01:23 +0200 Subject: [PATCH] =?UTF-8?q?(bit|top|hom)(Any|All)=20=E2=86=92=20(any|all)?= =?UTF-8?q?=5F(bit|top|hom)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/README.md | 18 +++++++++--------- src/singeli/src/avx.singeli | 8 ++++---- src/singeli/src/avx2.singeli | 18 +++++++++--------- src/singeli/src/base.singeli | 2 +- src/singeli/src/count.singeli | 6 +++--- src/singeli/src/dyarith.singeli | 22 +++++++++++----------- src/singeli/src/lut.singeli | 4 ++-- src/singeli/src/mask.singeli | 10 +++++----- src/singeli/src/monarith.singeli | 2 +- src/singeli/src/neon.singeli | 26 +++++++++++++------------- src/singeli/src/scan.singeli | 4 ++-- src/singeli/src/search.singeli | 10 +++++----- src/singeli/src/select.singeli | 2 +- src/singeli/src/squeeze.singeli | 18 +++++++++--------- src/singeli/src/sse2.singeli | 14 +++++++------- 15 files changed, 82 insertions(+), 82 deletions(-) diff --git a/src/singeli/README.md b/src/singeli/README.md index 6ce54da4..b9ffa4de 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -161,12 +161,15 @@ Some of them define variants which operate within 128-bit lanes, via a `128` pos Homogeneous definitions (i.e. ones with `hom` in their name) assume that each element in the mask type has all its bits equal. -- `homAll{a:VI} : u1` - whether all elements are set -- `homAny{a:VI} : u1` - whether any element is set -- `topAny{a:VI} : u1` - whether all elements have their top bit set -- `topAll{a:VI} : u1` - whether any element has its top bit set +- `all_hom{a:VI} : u1` - whether all elements are set +- `any_hom{a:VI} : u1` - whether any element is set +- `any_top{a:VI} : u1` - whether all elements have their top bit set +- `all_top{a:VI} : u1` - whether any element has its top bit set +- `any_bit{a:VI} : u1` - whether any bit in any element is set +- `all_bit{a:VI} : u1` - whether all bits in all elements is set - `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1` - `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m` +- `blend_bit{f:V, t:V, m:M} : V` - bitwise blend - `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal) - `homMask{...vs} : uint` - merged mask of `each{homMask,vs}` - `topMask{a:VI} : uint` - integer mask of the top bit of each element @@ -236,9 +239,6 @@ For float conversions, the used rounding mode is unspecified. - `mulwLo`, `mulwHi`, `mulw` - widening multiply - `andnz` - element-wise (a&b)!=0? ~0 : 0 -- `bitAll` - are all bits 1s -- `bitAny` - is any bit a 1 -- `bitBlend` - blend by bits - `clz` - count leading zeroes - `cls` - count leading sign bits - `copyLane` @@ -308,7 +308,7 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @maskedLoop ) { # x, r, and b now have type VT r1:= r + x - b # subtracting b because it's all 1s for a one - if (homAny{M{r1 < r}}) return{0} # detect overflow + if (any_hom{M{r1 < r}}) return{0} # detect overflow r = r1 # will be mask-stored if needed } 1 @@ -327,7 +327,7 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x un ) { def r0 = r{} def r1 = each{-, each{+,r0,x}, b} - if (homAny{M{tree_fold{|, each{<,r1,r0}}}}) return{0} + if (any_hom{M{tree_fold{|, each{<,r1,r0}}}}) return{0} r{r1} } 1 diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 7466cee7..7dd7b73e 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -53,8 +53,8 @@ def topMask{x:T if w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}} def topMask{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} def homMask{x:T if w256{T}} = topMask{x} -def homAny{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0 -def homAll{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff +def any_hom{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0 +def all_hom{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff -def topAny{x:T=[_]E if w256i{T} and width{E}>=32} = topMask{x} != 0 -def topAll{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<=32} = topMask{x} != 0 +def all_top{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<>8) } -def homAny{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}} -def homAll{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff -def topAny{x:T if w256i{T}} = topMask{x} != 0 -def topAll{x:T=[k]_ if w256i{T}} = topMask{x} == (1<=32} = topAny{x} +def any_top{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}} +def any_top{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}} +def any_hom{x:T if w256i{T} and elwidth{T}>=32} = any_top{x} -def topAny{x:T if w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0} -def topAll{x:T if w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0} +def any_top{x:T if w256i{T,16}} = any_hom{[16]i16~~x < [16]i16**0} +def all_top{x:T if w256i{T,16}} = all_hom{[16]i16~~x < [16]i16**0} # conversion diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 86d4da48..3558fbf8 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -174,7 +174,7 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 } # base cases def { absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half, - homAll,homAny,bitAll,bitAny,homMask,homMaskStore,homMaskStoreF,loadBatchBit, + all_hom,any_hom,all_bit,any_bit,all_top,any_top,homMask,homMaskStore,homMaskStoreF,loadBatchBit, loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep, pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow, blend_top,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper, diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 5ed41fca..729ca72d 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -36,8 +36,8 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { ne := jv != load{*V~~(x+1)}; dc := -ne # Quickly skip ahead if initial values are all equal a:u64 = 1 - if (not homAny{ne} and b>=4) { - def eq_k{k} = homAll{tree_fold{&, @unroll(x in xv+a over k) x==jv}} + if (not any_hom{ne} and b>=4) { + def eq_k{k} = all_hom{tree_fold{&, @unroll(x in xv+a over k) x==jv}} def skip_eq{k} = if (eq_k{k}) { a=2*k; skip_eq{2*k} } def skip_eq{k==4} = while (a<=b-k and eq_k{k}) a+=k skip_eq{1} @@ -86,7 +86,7 @@ fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = { def V = [vl]u16 def bot = 1<<15 - 1 on:usz = 0 - @for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{topAny{t}}) { + @for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{any_top{t}}) { o := if (hasarch{'X86_64'}) topMask{t} else homMask{t > V**bot} if (jv == n/vl) o &= type{o}~~1<<(n%vl) - 1 while (o > 0) { diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 2e917bb8..f03bface 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -18,8 +18,8 @@ def bqn_or{a, b} = (a+b)-(a*b) # + & - -def arithChk1{(__add), M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}} -def arithChk1{(__sub), M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}} +def arithChk1{(__add), M, w:T, x:T, r:T} = tup{'any_top', M{(w^r) & (x^r)}} +def arithChk1{(__sub), M, w:T, x:T, r:T} = tup{'any_top', M{(w^x) & (w^r)}} def arithChk1{(__add), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', adds{w,x}, r} def arithChk1{(__sub), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', subs{w,x}, r} @@ -37,7 +37,7 @@ def arithChk2{F=(__mul), M, w:T=[_](i8), x:T if hasarch{'X86_64'}} = { def rp = each{__mul, wp, xp} if (M{0}) { # masked check def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp} - tup{packQ{rp}, tup{'homAny', M{packQ{bad}}}} + tup{packQ{rp}, tup{'any_hom', M{packQ{bad}}}} } else { # unmasked check; can do check in a simpler way def bad = each{{v} => ty_s{v ^ (v>>15)}, rp} def RU = re_el{u16,T} @@ -53,7 +53,7 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = { max:= re_el{f32, (ty_u{T})**0x4efffffe} def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x} f32mul:= cf32{w} * cf32{x} - tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}} + tup{w*x, tup{'any_hom', M{abs{f32mul} >= max}}} # TODO fallback to the below if the above fails # TODO don't do this, but instead shuffle one half, do math, unshuffle that half # def wp = mzip128{w, T**0} @@ -63,12 +63,12 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = { # def bad = each{{v} => { # ((T2~~v + T2**0x80000000) ^ T2**(cast{i64,1}<<63)) > T2**cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF} # }, rp} - # tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'homAny', tree_fold{|,bad}}} this doesn't use M + # tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'any_hom', tree_fold{|,bad}}} this doesn't use M } def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = { def {rl, rh} = pack{...mulw{w, x}} - tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}} + tup{rl, tup{'any_hom', M{rh != (rl >> (width{E}-1))}}} } @@ -97,22 +97,22 @@ def runner{u, R, F} = { def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} or same{F,__sub})} = { # 'a'+1, 'a'-1 r:= F{ty_u{w}, ty_u{x}} - tup{re_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}} + tup{re_el{R, VW}~~r, tup{'any_hom', M{r > type{r}**1114111}}} } run } -# homAny, topAny already give masked vals; anyne doesn't, and ~andAllZero assumes no masking +# any_hom, any_top already give masked vals; anyne doesn't, and ~andAllZero assumes no masking def runChecks_any{F, vals} = { F{tree_fold{|, each{select{.,1}, vals}}} } -def runChecks{'homAny', vals, M} = runChecks_any{homAny, vals} -def runChecks{'topAny', vals, M} = runChecks_any{topAny, vals} +def runChecks{'any_hom', vals, M} = runChecks_any{any_hom, vals} +def runChecks{'any_top', vals, M} = runChecks_any{any_top, vals} def runChecks{'none', vals, M} = 0 def runChecks{'~andAllZero', vals, M if ~M{0}} = ~tree_fold{&, each{andAllZero, ...slice{flip{vals}, 1}}} def runChecks{'anyne', vals, M} = { def i{vals} = { def {_,xs,ys} = flip{vals} assert{M{0} == 0} - ~homAll{tree_fold{&, each{==, xs, ys}}} + ~all_hom{tree_fold{&, each{==, xs, ys}}} } def i{{{_,x,y}}} = anyne{x, y, M} i{vals} diff --git a/src/singeli/src/lut.singeli b/src/singeli/src/lut.singeli index 3bd6f383..6d4cf2b6 100644 --- a/src/singeli/src/lut.singeli +++ b/src/singeli/src/lut.singeli @@ -56,7 +56,7 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => { # def [_]IE = oneType{is} # def shl = if (IE==u8) __shl{u16} else __shl # def bm = shl{is, width{IE}-1 - lb{nth}} - # each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}} + # each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}} # } def me{is:[_](u8) if hasarch{'X86_64'} and E==u32} = { each{{l,h,m} => blend_top{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}} @@ -76,7 +76,7 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => { # if (mode=='c') hi{'xor', lo} # def me{is:I=[_](u8) if mode=='c'} = { # def bm = (is & I**lb{nth}) - # each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}} + # each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}} # } }}} diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 4d4abc00..24c51db8 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -14,8 +14,8 @@ def maskOf{T,n if w256{T}} = maskOfImpl{T, n, 256} def maskOf{T,n if w128{T}} = maskOfImpl{T, n, 128} def maskOf{T,n if w64{T}} = maskOfImpl{T, n, 64} -def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~homAll{x==y} -def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = homAny{M{x!=y}} +def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~all_hom{x==y} +def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = any_hom{M{x!=y}} def anyne{x:T, y:T, M if M{0}==0 and anyInt{x}} = x!=y def anyne{x:T, y:T, M if M{0}==1 and anyInt{x}} = M{x^y} != 0 def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} @@ -64,9 +64,9 @@ def loadBatch{ptr:*E0, i, [k]E1} = { def loadBatch {ptr:*E, {...ns}, T } = each{loadBatch {ptr, ., T }, ns} def storeBatch{ptr:*E, {...ns}, xs, M} = each{storeBatch{ptr, ., ., M}, ns, xs} -# TODO also similar homAny & use those more -def homAll{(maskNone), ...xs} = homAll{...xs} -def homAll{M, x:T if kgen{M}} = ~homAny{M{~x}} # TODO better +# TODO also similar any_hom & use those more +def all_hom{(maskNone), ...xs} = all_hom{...xs} +def all_hom{M, x:T if kgen{M}} = ~any_hom{M{~x}} # TODO better # "harmless" pointer cast that'll only cast void* def hCast{T,p} = assert{0, 'expected pointer with element',T,'or void but got ',p} diff --git a/src/singeli/src/monarith.singeli b/src/singeli/src/monarith.singeli index 17f7772e..59ab73e9 100644 --- a/src/singeli/src/monarith.singeli +++ b/src/singeli/src/monarith.singeli @@ -9,7 +9,7 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { def bulk = arch_defvw/width{T} def VT = [bulk]T @muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) { - if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk} + if (T!=f64 and any_hom{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk} sr{each{abs, cx}} } len diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 7d67b172..ed12c861 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -116,27 +116,27 @@ def narrowPair{a:T=[_]E, b:T if isint{E}} = pack{a, b, 0} def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x} def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}} -def homAll{x:V if nvec{V}} = bitAll{x} +def all_hom{x:V if nvec{V}} = all_bit{x} local def bit_any_lo64{x:V} = extract{re_el{u64,x}, 0} != 0 local def hom_any_lo64{x:V} = extract{re_el{f64,x}, 0} != 0.0 # safe to use float comparison as homogeneity guarantees not hitting -0.0; https://lemire.me/blog/2025/01/20/checking-whether-an-arm-neon-register-is-zero/ local def bit_all_lo64{x:V} = extract{re_el{i64,x}, 0} == -1 # no special hom_all_lo64 possible with float comparison local def min_pack{x} = re_el{u64, minp{...2**re_el{u32,x}}} local def max_pack{x} = re_el{u64, maxp{...2**re_el{u32,x}}} -def bitAny{x:V if w64{V}} = bit_any_lo64{x} -def bitAny{x:V if w128{V}} = bit_any_lo64{max_pack{x}} -def homAny{x:V if w64{V}} = hom_any_lo64{x} -def homAny{x:V if w128{V}} = hom_any_lo64{max_pack{x}} -def bitAll{x:V if w64{V}} = bit_all_lo64{x} -def bitAll{x:V if w128{V}} = bit_all_lo64{min_pack{x}} +def any_bit{x:V if w64{V}} = bit_any_lo64{x} +def any_bit{x:V if w128{V}} = bit_any_lo64{max_pack{x}} +def any_hom{x:V if w64{V}} = hom_any_lo64{x} +def any_hom{x:V if w128{V}} = hom_any_lo64{max_pack{x}} +def all_bit{x:V if w64{V}} = bit_all_lo64{x} +def all_bit{x:V if w128{V}} = bit_all_lo64{min_pack{x}} -def homAll{x:V if w128{V} and elwidth{V}>=16} = ~hom_any_lo64{subnHi{[8]u16**16r2b1, [8]u16~~x}} -def homAny{x:V if w128{V} and elwidth{V}>=16} = hom_any_lo64{narrow{u8,[8]u16~~x}} # narrow probably better than maxp -# TODO multi-vector homAny for ew≥16 via addhn? +def all_hom{x:V if w128{V} and elwidth{V}>=16} = ~hom_any_lo64{subnHi{[8]u16**16r2b1, [8]u16~~x}} +def any_hom{x:V if w128{V} and elwidth{V}>=16} = hom_any_lo64{narrow{u8,[8]u16~~x}} # narrow probably better than maxp +# TODO multi-vector any_hom for ew≥16 via addhn? -def topAny{x:V if nvec{V}} = fold_min{ty_s{x}}<0 -def topAll{x:V if nvec{V}} = fold_max{ty_s{x}}<0 +def any_top{x:V if nvec{V}} = fold_min{ty_s{x}}<0 +def all_top{x:V if nvec{V}} = fold_max{ty_s{x}}<0 def homMask{x:T=[k]E if nvecu{T} and width{E}>=k} = { @@ -168,7 +168,7 @@ def homMask{a:T,b:T=[k]E if k*2<=width{E}} = { truncBits{k*2, fold_add{shrm{a,width{E}-k,b} & make{T, (1< absu{c} >= (ty_u{T}**maxFastE), cx}}}}) goto{'end'} + if (rare{any_hom{tree_fold{|, each{{c:T} => absu{c} >= (ty_u{T}**maxFastE), cx}}}}) goto{'end'} } def s0 = each{scan_plus, cx} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 505b1868..6c2c8e74 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -27,9 +27,9 @@ def search{E, x, n:(u64), OP} = { } @muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) { eq:= each{OP, x} - if (homAny{M{tree_fold{|, eq}}}) { + if (any_hom{M{tree_fold{|, eq}}}) { findFirst{ - {i,c} => homAny{c}, + {i,c} => any_hom{c}, {i,c} => tup{i,c}, end, is, eq @@ -54,7 +54,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = { def bulk = arch_defvw/width{E} def VT = [bulk]E @maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) { - if (homAny{M{x!=x}}) return{1} + if (any_hom{M{x!=x}}) return{1} sr{x + VT**0} } 0 @@ -325,7 +325,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { def minA = acc{2, VT**min1} def maxA = acc{2, VT**min1} @muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) { - if (E==f64 and homAny{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0} + if (E==f64 and any_hom{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0} minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend } @@ -637,7 +637,7 @@ def try_vec_memb{T==u32, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done @unroll (f in fp+u*i over a to u) c |= V**(1<<(rw*a)) & test{f} r = vswap ^ vfold{|, c} } - @for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ homAny{test{fp}}} + @for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ any_hom{test{fp}}} goto{done} } def try{nv} = { diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index a0dce71e..9189d087 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -25,7 +25,7 @@ if_inline (hasarch{'AVX2'}) { def wrapChk{cw0:VI, xlf, M} = { cw:= cw0 + (xlf & VI~~(cw0= ty_u{xlf}}}) return{0} + if (any_hom{M{ty_u{cw} >= ty_u{xlf}}}) return{0} cw } diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index b4507004..02d316ed 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -3,22 +3,22 @@ include './mask' include './cbqnDefs' include './vecfold' -def is_sNaN{x:[_](u64)} = tup{homAny, inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2}} +def is_sNaN{x:[_](u64)} = tup{any_hom, inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2}} def is_sNaN{x:[2](u64) if hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = { # avoiding i64 comparisons def nan = unord{[2]f64~~x, [2]f64~~x} def qnan = re_el{u64, [4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000)} - tup{topAny, nan &~ qnan} + tup{any_top, nan &~ qnan} } def any_sNaN{M, ...xs} = { def {any, are} = flip{each{is_sNaN, xs}} oneVal{any}{M{tree_fold{|, are}}} } -def any_nonC32{M, x:[_](u64)} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} +def any_nonC32{M, x:[_](u64)} = any_hom{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} def any_nonC32{M, x:T=[_]_ if hasarch{'X86_64'}} = { def H = re_el{u32, T} def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16} - topAny{M{T~~ne}} + any_top{M{T~~ne}} } def any_nonC32{(maskNone), x:[k](u64), y:[k](u64)} = { def T32 = [k*2]u32 @@ -86,7 +86,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { if (B) { if (any_nonC32{M, ...xs}) return{3} } else { - def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}} + def bad = if (hasarch{'AARCH64'}) any_bit{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}} else ~andAllZero{orx, ~XV**maxvalue{w_h{X}}} if (bad) return{lb{width{X}}-3} } @@ -103,13 +103,13 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { @muLoop{bulk, 4}(xs in xb, M in 'm' over len) { minc:= tree_fold{min, xs} maxc:= tree_fold{max, xs} - if (M{0}==0 and homAny{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff} + if (M{0}==0 and any_hom{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff} acc{M, minc, maxc} } } else { @muLoop{bulk, 2}(xs in xb, M in 'm' over len) { def mixed = acc{M, xs} - if (M{0}==0 and homAny{mixed > XV**maxvalue{EH}}) return{0xffff_ffff} + if (M{0}==0 and any_hom{mixed > XV**maxvalue{EH}}) return{0xffff_ffff} } } acc{} @@ -132,7 +132,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { @muLoop{bulk, 2}(xs in xb, M in 'm' over is to len) { if (hasarch{'X86_64'}) { def ns = each{narrow{i32,.}, xs} - if (homAny{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}} + if (any_hom{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}} acc{M, ns} } else { assert{hasarch{'AARCH64'}} @@ -141,7 +141,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { def int32 = if (k==2) pack{...int, 0} else pack{...int, [bulk]i64**0, 0} def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widenUpper}, 0, k}} - if (~homAll{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}} + if (~all_hom{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}} acc{M, tup{int32}} } } diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 4dd333d3..f13f6d59 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -40,7 +40,7 @@ def rsqrtE{a:([4]f32)} = emit{[4]f32, '_mm_rsqrt_ps', a} def rcpE{a:([4]f32)} = emit{[4]f32, '_mm_rcp_ps', a} # mask stuff -def andAllZero{x:T, y:T if w128i{T}} = homAll{(x & y) == T**0} +def andAllZero{x:T, y:T if w128i{T}} = all_hom{(x & y) == T**0} def topMask{x:T if w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}} @@ -50,13 +50,13 @@ def homMask{x:T if w128{T}} = topMask{x} def homMaskX{a:[_]T if width{T}==16} = tup{2, homMask{re_el{u8,a}}} def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}} -def homAny{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0 -def homAll{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff +def any_hom{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0 +def all_hom{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff -def topAny{x:T if w128i{T}} = topMask{x} != 0 -def topAll{x:T=[k]_ if w128i{T}} = topMask{x} == (1<