(bit|top|hom)(Any|All) → (any|all)_(bit|top|hom)

This commit is contained in:
dzaima 2025-01-25 23:01:23 +02:00
parent c4afe7b329
commit 426c1b4bf8
15 changed files with 82 additions and 82 deletions

View File

@ -161,12 +161,15 @@ Some of them define variants which operate within 128-bit lanes, via a `128` pos
Homogeneous definitions (i.e. ones with `hom` in their name) assume that each element in the mask type has all its bits equal. Homogeneous definitions (i.e. ones with `hom` in their name) assume that each element in the mask type has all its bits equal.
- `homAll{a:VI} : u1` - whether all elements are set - `all_hom{a:VI} : u1` - whether all elements are set
- `homAny{a:VI} : u1` - whether any element is set - `any_hom{a:VI} : u1` - whether any element is set
- `topAny{a:VI} : u1` - whether all elements have their top bit set - `any_top{a:VI} : u1` - whether all elements have their top bit set
- `topAll{a:VI} : u1` - whether any element has its top bit set - `all_top{a:VI} : u1` - whether any element has its top bit set
- `any_bit{a:VI} : u1` - whether any bit in any element is set
- `all_bit{a:VI} : u1` - whether all bits in all elements is set
- `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1` - `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1`
- `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m` - `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m`
- `blend_bit{f:V, t:V, m:M} : V` - bitwise blend
- `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal) - `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal)
- `homMask{...vs} : uint` - merged mask of `each{homMask,vs}` - `homMask{...vs} : uint` - merged mask of `each{homMask,vs}`
- `topMask{a:VI} : uint` - integer mask of the top bit of each element - `topMask{a:VI} : uint` - integer mask of the top bit of each element
@ -236,9 +239,6 @@ For float conversions, the used rounding mode is unspecified.
- `mulwLo`, `mulwHi`, `mulw` - widening multiply - `mulwLo`, `mulwHi`, `mulw` - widening multiply
- `andnz` - element-wise (a&b)!=0? ~0 : 0 - `andnz` - element-wise (a&b)!=0? ~0 : 0
<!-- --> <!-- -->
- `bitAll` - are all bits 1s
- `bitAny` - is any bit a 1
- `bitBlend` - blend by bits
- `clz` - count leading zeroes - `clz` - count leading zeroes
- `cls` - count leading sign bits - `cls` - count leading sign bits
- `copyLane` - `copyLane`
@ -308,7 +308,7 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @maskedLoop
) { ) {
# x, r, and b now have type VT # x, r, and b now have type VT
r1:= r + x - b # subtracting b because it's all 1s for a one r1:= r + x - b # subtracting b because it's all 1s for a one
if (homAny{M{r1 < r}}) return{0} # detect overflow if (any_hom{M{r1 < r}}) return{0} # detect overflow
r = r1 # will be mask-stored if needed r = r1 # will be mask-stored if needed
} }
1 1
@ -327,7 +327,7 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x un
) { ) {
def r0 = r{} def r0 = r{}
def r1 = each{-, each{+,r0,x}, b} def r1 = each{-, each{+,r0,x}, b}
if (homAny{M{tree_fold{|, each{<,r1,r0}}}}) return{0} if (any_hom{M{tree_fold{|, each{<,r1,r0}}}}) return{0}
r{r1} r{r1}
} }
1 1

View File

@ -53,8 +53,8 @@ def topMask{x:T if w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
def topMask{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} def topMask{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
def homMask{x:T if w256{T}} = topMask{x} def homMask{x:T if w256{T}} = topMask{x}
def homAny{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0 def any_hom{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0
def homAll{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff def all_hom{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff
def topAny{x:T=[_]E if w256i{T} and width{E}>=32} = topMask{x} != 0 def any_top{x:T=[_]E if w256i{T} and width{E}>=32} = topMask{x} != 0
def topAll{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<<k)-1 def all_top{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<<k)-1

View File

@ -39,18 +39,18 @@ def topMask{x:T if w256{T, 16}} = {
(msk&255) | (msk>>8) (msk&255) | (msk>>8)
} }
def homAny{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}} def any_hom{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
def homAll{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff def all_hom{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
def topAny{x:T if w256i{T}} = topMask{x} != 0 def any_top{x:T if w256i{T}} = topMask{x} != 0
def topAll{x:T=[k]_ if w256i{T}} = topMask{x} == (1<<k)-1 def all_top{x:T=[k]_ if w256i{T}} = topMask{x} == (1<<k)-1
def homMask{a:T, b:T if w256i{T,16}} = homMask{vec_shuffle{[4]u64, packQ{ty_s{a},ty_s{b}}, 0,2,1,3}} def homMask{a:T, b:T if w256i{T,16}} = homMask{vec_shuffle{[4]u64, packQ{ty_s{a},ty_s{b}}, 0,2,1,3}}
def topAny{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}} def any_top{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
def topAny{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}} def any_top{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
def homAny{x:T if w256i{T} and elwidth{T}>=32} = topAny{x} def any_hom{x:T if w256i{T} and elwidth{T}>=32} = any_top{x}
def topAny{x:T if w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0} def any_top{x:T if w256i{T,16}} = any_hom{[16]i16~~x < [16]i16**0}
def topAll{x:T if w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0} def all_top{x:T if w256i{T,16}} = all_hom{[16]i16~~x < [16]i16**0}
# conversion # conversion

View File

@ -174,7 +174,7 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
# base cases # base cases
def { def {
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half, absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
homAll,homAny,bitAll,bitAny,homMask,homMaskStore,homMaskStoreF,loadBatchBit, all_hom,any_hom,all_bit,any_bit,all_top,any_top,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep, loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow, pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
blend_top,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper, blend_top,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,

View File

@ -36,8 +36,8 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = {
ne := jv != load{*V~~(x+1)}; dc := -ne ne := jv != load{*V~~(x+1)}; dc := -ne
# Quickly skip ahead if initial values are all equal # Quickly skip ahead if initial values are all equal
a:u64 = 1 a:u64 = 1
if (not homAny{ne} and b>=4) { if (not any_hom{ne} and b>=4) {
def eq_k{k} = homAll{tree_fold{&, @unroll(x in xv+a over k) x==jv}} def eq_k{k} = all_hom{tree_fold{&, @unroll(x in xv+a over k) x==jv}}
def skip_eq{k} = if (eq_k{k}) { a=2*k; skip_eq{2*k} } def skip_eq{k} = if (eq_k{k}) { a=2*k; skip_eq{2*k} }
def skip_eq{k==4} = while (a<=b-k and eq_k{k}) a+=k def skip_eq{k==4} = while (a<=b-k and eq_k{k}) a+=k
skip_eq{1} skip_eq{1}
@ -86,7 +86,7 @@ fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = {
def V = [vl]u16 def V = [vl]u16
def bot = 1<<15 - 1 def bot = 1<<15 - 1
on:usz = 0 on:usz = 0
@for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{topAny{t}}) { @for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{any_top{t}}) {
o := if (hasarch{'X86_64'}) topMask{t} else homMask{t > V**bot} o := if (hasarch{'X86_64'}) topMask{t} else homMask{t > V**bot}
if (jv == n/vl) o &= type{o}~~1<<(n%vl) - 1 if (jv == n/vl) o &= type{o}~~1<<(n%vl) - 1
while (o > 0) { while (o > 0) {

View File

@ -18,8 +18,8 @@ def bqn_or{a, b} = (a+b)-(a*b)
# + & - # + & -
def arithChk1{(__add), M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}} def arithChk1{(__add), M, w:T, x:T, r:T} = tup{'any_top', M{(w^r) & (x^r)}}
def arithChk1{(__sub), M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}} def arithChk1{(__sub), M, w:T, x:T, r:T} = tup{'any_top', M{(w^x) & (w^r)}}
def arithChk1{(__add), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', adds{w,x}, r} def arithChk1{(__add), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', adds{w,x}, r}
def arithChk1{(__sub), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', subs{w,x}, r} def arithChk1{(__sub), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', subs{w,x}, r}
@ -37,7 +37,7 @@ def arithChk2{F=(__mul), M, w:T=[_](i8), x:T if hasarch{'X86_64'}} = {
def rp = each{__mul, wp, xp} def rp = each{__mul, wp, xp}
if (M{0}) { # masked check if (M{0}) { # masked check
def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp} def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp}
tup{packQ{rp}, tup{'homAny', M{packQ{bad}}}} tup{packQ{rp}, tup{'any_hom', M{packQ{bad}}}}
} else { # unmasked check; can do check in a simpler way } else { # unmasked check; can do check in a simpler way
def bad = each{{v} => ty_s{v ^ (v>>15)}, rp} def bad = each{{v} => ty_s{v ^ (v>>15)}, rp}
def RU = re_el{u16,T} def RU = re_el{u16,T}
@ -53,7 +53,7 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = {
max:= re_el{f32, (ty_u{T})**0x4efffffe} max:= re_el{f32, (ty_u{T})**0x4efffffe}
def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x} def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x}
f32mul:= cf32{w} * cf32{x} f32mul:= cf32{w} * cf32{x}
tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}} tup{w*x, tup{'any_hom', M{abs{f32mul} >= max}}}
# TODO fallback to the below if the above fails # TODO fallback to the below if the above fails
# TODO don't do this, but instead shuffle one half, do math, unshuffle that half # TODO don't do this, but instead shuffle one half, do math, unshuffle that half
# def wp = mzip128{w, T**0} # def wp = mzip128{w, T**0}
@ -63,12 +63,12 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = {
# def bad = each{{v} => { # def bad = each{{v} => {
# ((T2~~v + T2**0x80000000) ^ T2**(cast{i64,1}<<63)) > T2**cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF} # ((T2~~v + T2**0x80000000) ^ T2**(cast{i64,1}<<63)) > T2**cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF}
# }, rp} # }, rp}
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'homAny', tree_fold{|,bad}}} this doesn't use M # tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'any_hom', tree_fold{|,bad}}} this doesn't use M
} }
def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = { def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = {
def {rl, rh} = pack{...mulw{w, x}} def {rl, rh} = pack{...mulw{w, x}}
tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}} tup{rl, tup{'any_hom', M{rh != (rl >> (width{E}-1))}}}
} }
@ -97,22 +97,22 @@ def runner{u, R, F} = {
def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} or same{F,__sub})} = { # 'a'+1, 'a'-1 def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} or same{F,__sub})} = { # 'a'+1, 'a'-1
r:= F{ty_u{w}, ty_u{x}} r:= F{ty_u{w}, ty_u{x}}
tup{re_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}} tup{re_el{R, VW}~~r, tup{'any_hom', M{r > type{r}**1114111}}}
} }
run run
} }
# homAny, topAny already give masked vals; anyne doesn't, and ~andAllZero assumes no masking # any_hom, any_top already give masked vals; anyne doesn't, and ~andAllZero assumes no masking
def runChecks_any{F, vals} = { F{tree_fold{|, each{select{.,1}, vals}}} } def runChecks_any{F, vals} = { F{tree_fold{|, each{select{.,1}, vals}}} }
def runChecks{'homAny', vals, M} = runChecks_any{homAny, vals} def runChecks{'any_hom', vals, M} = runChecks_any{any_hom, vals}
def runChecks{'topAny', vals, M} = runChecks_any{topAny, vals} def runChecks{'any_top', vals, M} = runChecks_any{any_top, vals}
def runChecks{'none', vals, M} = 0 def runChecks{'none', vals, M} = 0
def runChecks{'~andAllZero', vals, M if ~M{0}} = ~tree_fold{&, each{andAllZero, ...slice{flip{vals}, 1}}} def runChecks{'~andAllZero', vals, M if ~M{0}} = ~tree_fold{&, each{andAllZero, ...slice{flip{vals}, 1}}}
def runChecks{'anyne', vals, M} = { def runChecks{'anyne', vals, M} = {
def i{vals} = { def i{vals} = {
def {_,xs,ys} = flip{vals} def {_,xs,ys} = flip{vals}
assert{M{0} == 0} assert{M{0} == 0}
~homAll{tree_fold{&, each{==, xs, ys}}} ~all_hom{tree_fold{&, each{==, xs, ys}}}
} }
def i{{{_,x,y}}} = anyne{x, y, M} def i{{{_,x,y}}} = anyne{x, y, M}
i{vals} i{vals}

View File

@ -56,7 +56,7 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
# def [_]IE = oneType{is} # def [_]IE = oneType{is}
# def shl = if (IE==u8) __shl{u16} else __shl # def shl = if (IE==u8) __shl{u16} else __shl
# def bm = shl{is, width{IE}-1 - lb{nth}} # def bm = shl{is, width{IE}-1 - lb{nth}}
# each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}} # each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}}
# } # }
def me{is:[_](u8) if hasarch{'X86_64'} and E==u32} = { def me{is:[_](u8) if hasarch{'X86_64'} and E==u32} = {
each{{l,h,m} => blend_top{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}} each{{l,h,m} => blend_top{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}}
@ -76,7 +76,7 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
# if (mode=='c') hi{'xor', lo} # if (mode=='c') hi{'xor', lo}
# def me{is:I=[_](u8) if mode=='c'} = { # def me{is:I=[_](u8) if mode=='c'} = {
# def bm = (is & I**lb{nth}) # def bm = (is & I**lb{nth})
# each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}} # each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}}
# } # }
}}} }}}

View File

@ -14,8 +14,8 @@ def maskOf{T,n if w256{T}} = maskOfImpl{T, n, 256}
def maskOf{T,n if w128{T}} = maskOfImpl{T, n, 128} def maskOf{T,n if w128{T}} = maskOfImpl{T, n, 128}
def maskOf{T,n if w64{T}} = maskOfImpl{T, n, 64} def maskOf{T,n if w64{T}} = maskOfImpl{T, n, 64}
def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~homAll{x==y} def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~all_hom{x==y}
def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = homAny{M{x!=y}} def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = any_hom{M{x!=y}}
def anyne{x:T, y:T, M if M{0}==0 and anyInt{x}} = x!=y def anyne{x:T, y:T, M if M{0}==0 and anyInt{x}} = x!=y
def anyne{x:T, y:T, M if M{0}==1 and anyInt{x}} = M{x^y} != 0 def anyne{x:T, y:T, M if M{0}==1 and anyInt{x}} = M{x^y} != 0
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
@ -64,9 +64,9 @@ def loadBatch{ptr:*E0, i, [k]E1} = {
def loadBatch {ptr:*E, {...ns}, T } = each{loadBatch {ptr, ., T }, ns} def loadBatch {ptr:*E, {...ns}, T } = each{loadBatch {ptr, ., T }, ns}
def storeBatch{ptr:*E, {...ns}, xs, M} = each{storeBatch{ptr, ., ., M}, ns, xs} def storeBatch{ptr:*E, {...ns}, xs, M} = each{storeBatch{ptr, ., ., M}, ns, xs}
# TODO also similar homAny & use those more # TODO also similar any_hom & use those more
def homAll{(maskNone), ...xs} = homAll{...xs} def all_hom{(maskNone), ...xs} = all_hom{...xs}
def homAll{M, x:T if kgen{M}} = ~homAny{M{~x}} # TODO better def all_hom{M, x:T if kgen{M}} = ~any_hom{M{~x}} # TODO better
# "harmless" pointer cast that'll only cast void* # "harmless" pointer cast that'll only cast void*
def hCast{T,p} = assert{0, 'expected pointer with element',T,'or void but got ',p} def hCast{T,p} = assert{0, 'expected pointer with element',T,'or void but got ',p}

View File

@ -9,7 +9,7 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
def bulk = arch_defvw/width{T} def bulk = arch_defvw/width{T}
def VT = [bulk]T def VT = [bulk]T
@muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) { @muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk} if (T!=f64 and any_hom{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk}
sr{each{abs, cx}} sr{each{abs, cx}}
} }
len len

View File

@ -116,27 +116,27 @@ def narrowPair{a:T=[_]E, b:T if isint{E}} = pack{a, b, 0}
def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x} def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x}
def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}} def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}}
def homAll{x:V if nvec{V}} = bitAll{x} def all_hom{x:V if nvec{V}} = all_bit{x}
local def bit_any_lo64{x:V} = extract{re_el{u64,x}, 0} != 0 local def bit_any_lo64{x:V} = extract{re_el{u64,x}, 0} != 0
local def hom_any_lo64{x:V} = extract{re_el{f64,x}, 0} != 0.0 # safe to use float comparison as homogeneity guarantees not hitting -0.0; https://lemire.me/blog/2025/01/20/checking-whether-an-arm-neon-register-is-zero/ local def hom_any_lo64{x:V} = extract{re_el{f64,x}, 0} != 0.0 # safe to use float comparison as homogeneity guarantees not hitting -0.0; https://lemire.me/blog/2025/01/20/checking-whether-an-arm-neon-register-is-zero/
local def bit_all_lo64{x:V} = extract{re_el{i64,x}, 0} == -1 # no special hom_all_lo64 possible with float comparison local def bit_all_lo64{x:V} = extract{re_el{i64,x}, 0} == -1 # no special hom_all_lo64 possible with float comparison
local def min_pack{x} = re_el{u64, minp{...2**re_el{u32,x}}} local def min_pack{x} = re_el{u64, minp{...2**re_el{u32,x}}}
local def max_pack{x} = re_el{u64, maxp{...2**re_el{u32,x}}} local def max_pack{x} = re_el{u64, maxp{...2**re_el{u32,x}}}
def bitAny{x:V if w64{V}} = bit_any_lo64{x} def any_bit{x:V if w64{V}} = bit_any_lo64{x}
def bitAny{x:V if w128{V}} = bit_any_lo64{max_pack{x}} def any_bit{x:V if w128{V}} = bit_any_lo64{max_pack{x}}
def homAny{x:V if w64{V}} = hom_any_lo64{x} def any_hom{x:V if w64{V}} = hom_any_lo64{x}
def homAny{x:V if w128{V}} = hom_any_lo64{max_pack{x}} def any_hom{x:V if w128{V}} = hom_any_lo64{max_pack{x}}
def bitAll{x:V if w64{V}} = bit_all_lo64{x} def all_bit{x:V if w64{V}} = bit_all_lo64{x}
def bitAll{x:V if w128{V}} = bit_all_lo64{min_pack{x}} def all_bit{x:V if w128{V}} = bit_all_lo64{min_pack{x}}
def homAll{x:V if w128{V} and elwidth{V}>=16} = ~hom_any_lo64{subnHi{[8]u16**16r2b1, [8]u16~~x}} def all_hom{x:V if w128{V} and elwidth{V}>=16} = ~hom_any_lo64{subnHi{[8]u16**16r2b1, [8]u16~~x}}
def homAny{x:V if w128{V} and elwidth{V}>=16} = hom_any_lo64{narrow{u8,[8]u16~~x}} # narrow probably better than maxp def any_hom{x:V if w128{V} and elwidth{V}>=16} = hom_any_lo64{narrow{u8,[8]u16~~x}} # narrow probably better than maxp
# TODO multi-vector homAny for ew≥16 via addhn? # TODO multi-vector any_hom for ew≥16 via addhn?
def topAny{x:V if nvec{V}} = fold_min{ty_s{x}}<0 def any_top{x:V if nvec{V}} = fold_min{ty_s{x}}<0
def topAll{x:V if nvec{V}} = fold_max{ty_s{x}}<0 def all_top{x:V if nvec{V}} = fold_max{ty_s{x}}<0
def homMask{x:T=[k]E if nvecu{T} and width{E}>=k} = { def homMask{x:T=[k]E if nvecu{T} and width{E}>=k} = {
@ -168,7 +168,7 @@ def homMask{a:T,b:T=[k]E if k*2<=width{E}} = {
truncBits{k*2, fold_add{shrm{a,width{E}-k,b} & make{T, (1<<iota{k}) | (1<<(iota{k}+k))}}} truncBits{k*2, fold_add{shrm{a,width{E}-k,b} & make{T, (1<<iota{k}) | (1<<(iota{k}+k))}}}
} }
def andAllZero{x:T, y:T if nveci{T}} = ~bitAny{x&y} def andAllZero{x:T, y:T if nveci{T}} = ~any_bit{x&y}
def homMaskX{a:T=[k]E if E!=u64} = { def homMaskX{a:T=[k]E if E!=u64} = {
def h = width{E}/2 def h = width{E}/2

View File

@ -50,7 +50,7 @@ fn scan_idem{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = {
@for (ik to ek) { i := ik * k @for (ik to ek) { i := ik * k
def ii = iota{k} def ii = iota{k}
xvi := each{load{xv + i, .}, ii} xvi := each{load{xv + i, .}, ii}
if (not homAny{cmp{p, tree_fold{op, xvi}}}) { if (not any_hom{cmp{p, tree_fold{op, xvi}}}) {
each{store{rv+i, ., p}, ii} each{store{rv+i, ., p}, ii}
} else @unroll (rv in rv+i over j to k) { } else @unroll (rv in rv+i over j to k) {
rv = scan{select{xvi,j}, p} rv = scan{select{xvi,j}, p}
@ -278,7 +278,7 @@ def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
def cx0 = tup{load{*[bulk]X~~(x+i)}} def cx0 = tup{load{*[bulk]X~~(x+i)}}
def cx = if(wd) widenFull{WE,cx0} else cx0 def cx = if(wd) widenFull{WE,cx0} else cx0
if (~wd) { # within-vector overflow check; widening gives range space for this to not happen if (~wd) { # within-vector overflow check; widening gives range space for this to not happen
if (rare{homAny{tree_fold{|, each{{c:T} => absu{c} >= (ty_u{T}**maxFastE), cx}}}}) goto{'end'} if (rare{any_hom{tree_fold{|, each{{c:T} => absu{c} >= (ty_u{T}**maxFastE), cx}}}}) goto{'end'}
} }
def s0 = each{scan_plus, cx} def s0 = each{scan_plus, cx}

View File

@ -27,9 +27,9 @@ def search{E, x, n:(u64), OP} = {
} }
@muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) { @muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
eq:= each{OP, x} eq:= each{OP, x}
if (homAny{M{tree_fold{|, eq}}}) { if (any_hom{M{tree_fold{|, eq}}}) {
findFirst{ findFirst{
{i,c} => homAny{c}, {i,c} => any_hom{c},
{i,c} => tup{i,c}, {i,c} => tup{i,c},
end, end,
is, eq is, eq
@ -54,7 +54,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
def bulk = arch_defvw/width{E} def bulk = arch_defvw/width{E}
def VT = [bulk]E def VT = [bulk]E
@maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) { @maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
if (homAny{M{x!=x}}) return{1} if (any_hom{M{x!=x}}) return{1}
sr{x + VT**0} sr{x + VT**0}
} }
0 0
@ -325,7 +325,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
def minA = acc{2, VT**min1} def minA = acc{2, VT**min1}
def maxA = acc{2, VT**min1} def maxA = acc{2, VT**min1}
@muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) { @muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
if (E==f64 and homAny{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0} if (E==f64 and any_hom{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0}
minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend
maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend
} }
@ -637,7 +637,7 @@ def try_vec_memb{T==u32, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done
@unroll (f in fp+u*i over a to u) c |= V**(1<<(rw*a)) & test{f} @unroll (f in fp+u*i over a to u) c |= V**(1<<(rw*a)) & test{f}
r = vswap ^ vfold{|, c} r = vswap ^ vfold{|, c}
} }
@for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ homAny{test{fp}}} @for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ any_hom{test{fp}}}
goto{done} goto{done}
} }
def try{nv} = { def try{nv} = {

View File

@ -25,7 +25,7 @@ if_inline (hasarch{'AVX2'}) {
def wrapChk{cw0:VI, xlf, M} = { def wrapChk{cw0:VI, xlf, M} = {
cw:= cw0 + (xlf & VI~~(cw0<VI**0)) cw:= cw0 + (xlf & VI~~(cw0<VI**0))
if (homAny{M{ty_u{cw} >= ty_u{xlf}}}) return{0} if (any_hom{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
cw cw
} }

View File

@ -3,22 +3,22 @@ include './mask'
include './cbqnDefs' include './cbqnDefs'
include './vecfold' include './vecfold'
def is_sNaN{x:[_](u64)} = tup{homAny, inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2}} def is_sNaN{x:[_](u64)} = tup{any_hom, inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2}}
def is_sNaN{x:[2](u64) if hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = { # avoiding i64 comparisons def is_sNaN{x:[2](u64) if hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = { # avoiding i64 comparisons
def nan = unord{[2]f64~~x, [2]f64~~x} def nan = unord{[2]f64~~x, [2]f64~~x}
def qnan = re_el{u64, [4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000)} def qnan = re_el{u64, [4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000)}
tup{topAny, nan &~ qnan} tup{any_top, nan &~ qnan}
} }
def any_sNaN{M, ...xs} = { def any_sNaN{M, ...xs} = {
def {any, are} = flip{each{is_sNaN, xs}} def {any, are} = flip{each{is_sNaN, xs}}
oneVal{any}{M{tree_fold{|, are}}} oneVal{any}{M{tree_fold{|, are}}}
} }
def any_nonC32{M, x:[_](u64)} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} def any_nonC32{M, x:[_](u64)} = any_hom{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
def any_nonC32{M, x:T=[_]_ if hasarch{'X86_64'}} = { def any_nonC32{M, x:T=[_]_ if hasarch{'X86_64'}} = {
def H = re_el{u32, T} def H = re_el{u32, T}
def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16} def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16}
topAny{M{T~~ne}} any_top{M{T~~ne}}
} }
def any_nonC32{(maskNone), x:[k](u64), y:[k](u64)} = { def any_nonC32{(maskNone), x:[k](u64), y:[k](u64)} = {
def T32 = [k*2]u32 def T32 = [k*2]u32
@ -86,7 +86,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
if (B) { if (B) {
if (any_nonC32{M, ...xs}) return{3} if (any_nonC32{M, ...xs}) return{3}
} else { } else {
def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}} def bad = if (hasarch{'AARCH64'}) any_bit{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}}
else ~andAllZero{orx, ~XV**maxvalue{w_h{X}}} else ~andAllZero{orx, ~XV**maxvalue{w_h{X}}}
if (bad) return{lb{width{X}}-3} if (bad) return{lb{width{X}}-3}
} }
@ -103,13 +103,13 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
@muLoop{bulk, 4}(xs in xb, M in 'm' over len) { @muLoop{bulk, 4}(xs in xb, M in 'm' over len) {
minc:= tree_fold{min, xs} minc:= tree_fold{min, xs}
maxc:= tree_fold{max, xs} maxc:= tree_fold{max, xs}
if (M{0}==0 and homAny{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff} if (M{0}==0 and any_hom{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff}
acc{M, minc, maxc} acc{M, minc, maxc}
} }
} else { } else {
@muLoop{bulk, 2}(xs in xb, M in 'm' over len) { @muLoop{bulk, 2}(xs in xb, M in 'm' over len) {
def mixed = acc{M, xs} def mixed = acc{M, xs}
if (M{0}==0 and homAny{mixed > XV**maxvalue{EH}}) return{0xffff_ffff} if (M{0}==0 and any_hom{mixed > XV**maxvalue{EH}}) return{0xffff_ffff}
} }
} }
acc{} acc{}
@ -132,7 +132,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
@muLoop{bulk, 2}(xs in xb, M in 'm' over is to len) { @muLoop{bulk, 2}(xs in xb, M in 'm' over is to len) {
if (hasarch{'X86_64'}) { if (hasarch{'X86_64'}) {
def ns = each{narrow{i32,.}, xs} def ns = each{narrow{i32,.}, xs}
if (homAny{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}} if (any_hom{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}}
acc{M, ns} acc{M, ns}
} else { } else {
assert{hasarch{'AARCH64'}} assert{hasarch{'AARCH64'}}
@ -141,7 +141,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
def int32 = if (k==2) pack{...int, 0} else pack{...int, [bulk]i64**0, 0} def int32 = if (k==2) pack{...int, 0} else pack{...int, [bulk]i64**0, 0}
def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widenUpper}, 0, k}} def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widenUpper}, 0, k}}
if (~homAll{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}} if (~all_hom{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}}
acc{M, tup{int32}} acc{M, tup{int32}}
} }
} }

View File

@ -40,7 +40,7 @@ def rsqrtE{a:([4]f32)} = emit{[4]f32, '_mm_rsqrt_ps', a}
def rcpE{a:([4]f32)} = emit{[4]f32, '_mm_rcp_ps', a} def rcpE{a:([4]f32)} = emit{[4]f32, '_mm_rcp_ps', a}
# mask stuff # mask stuff
def andAllZero{x:T, y:T if w128i{T}} = homAll{(x & y) == T**0} def andAllZero{x:T, y:T if w128i{T}} = all_hom{(x & y) == T**0}
def topMask{x:T if w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} def topMask{x:T if w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}} def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
@ -50,13 +50,13 @@ def homMask{x:T if w128{T}} = topMask{x}
def homMaskX{a:[_]T if width{T}==16} = tup{2, homMask{re_el{u8,a}}} def homMaskX{a:[_]T if width{T}==16} = tup{2, homMask{re_el{u8,a}}}
def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}} def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}}
def homAny{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0 def any_hom{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0
def homAll{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff def all_hom{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
def topAny{x:T if w128i{T}} = topMask{x} != 0 def any_top{x:T if w128i{T}} = topMask{x} != 0
def topAll{x:T=[k]_ if w128i{T}} = topMask{x} == (1<<k)-1 def all_top{x:T=[k]_ if w128i{T}} = topMask{x} == (1<<k)-1
def topAny{x:T if w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0} def any_top{x:T if w128i{T, 16}} = any_hom{[8]i16~~x < [8]i16**0}
def topAll{x:T if w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0} def all_top{x:T if w128i{T, 16}} = all_hom{[8]i16~~x < [8]i16**0}
# bits of other things SSE2 has # bits of other things SSE2 has
def packs{a:T,b:T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b} def packs{a:T,b:T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}