(bit|top|hom)(Any|All) → (any|all)_(bit|top|hom)
This commit is contained in:
parent
c4afe7b329
commit
426c1b4bf8
@ -161,12 +161,15 @@ Some of them define variants which operate within 128-bit lanes, via a `128` pos
|
||||
|
||||
Homogeneous definitions (i.e. ones with `hom` in their name) assume that each element in the mask type has all its bits equal.
|
||||
|
||||
- `homAll{a:VI} : u1` - whether all elements are set
|
||||
- `homAny{a:VI} : u1` - whether any element is set
|
||||
- `topAny{a:VI} : u1` - whether all elements have their top bit set
|
||||
- `topAll{a:VI} : u1` - whether any element has its top bit set
|
||||
- `all_hom{a:VI} : u1` - whether all elements are set
|
||||
- `any_hom{a:VI} : u1` - whether any element is set
|
||||
- `any_top{a:VI} : u1` - whether all elements have their top bit set
|
||||
- `all_top{a:VI} : u1` - whether any element has its top bit set
|
||||
- `any_bit{a:VI} : u1` - whether any bit in any element is set
|
||||
- `all_bit{a:VI} : u1` - whether all bits in all elements is set
|
||||
- `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1`
|
||||
- `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m`
|
||||
- `blend_bit{f:V, t:V, m:M} : V` - bitwise blend
|
||||
- `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal)
|
||||
- `homMask{...vs} : uint` - merged mask of `each{homMask,vs}`
|
||||
- `topMask{a:VI} : uint` - integer mask of the top bit of each element
|
||||
@ -236,9 +239,6 @@ For float conversions, the used rounding mode is unspecified.
|
||||
- `mulwLo`, `mulwHi`, `mulw` - widening multiply
|
||||
- `andnz` - element-wise (a&b)!=0? ~0 : 0
|
||||
<!-- -->
|
||||
- `bitAll` - are all bits 1s
|
||||
- `bitAny` - is any bit a 1
|
||||
- `bitBlend` - blend by bits
|
||||
- `clz` - count leading zeroes
|
||||
- `cls` - count leading sign bits
|
||||
- `copyLane`
|
||||
@ -308,7 +308,7 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @maskedLoop
|
||||
) {
|
||||
# x, r, and b now have type VT
|
||||
r1:= r + x - b # subtracting b because it's all 1s for a one
|
||||
if (homAny{M{r1 < r}}) return{0} # detect overflow
|
||||
if (any_hom{M{r1 < r}}) return{0} # detect overflow
|
||||
r = r1 # will be mask-stored if needed
|
||||
}
|
||||
1
|
||||
@ -327,7 +327,7 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x un
|
||||
) {
|
||||
def r0 = r{}
|
||||
def r1 = each{-, each{+,r0,x}, b}
|
||||
if (homAny{M{tree_fold{|, each{<,r1,r0}}}}) return{0}
|
||||
if (any_hom{M{tree_fold{|, each{<,r1,r0}}}}) return{0}
|
||||
r{r1}
|
||||
}
|
||||
1
|
||||
|
||||
@ -53,8 +53,8 @@ def topMask{x:T if w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
|
||||
def topMask{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
|
||||
def homMask{x:T if w256{T}} = topMask{x}
|
||||
|
||||
def homAny{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0
|
||||
def homAll{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff
|
||||
def any_hom{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0
|
||||
def all_hom{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff
|
||||
|
||||
def topAny{x:T=[_]E if w256i{T} and width{E}>=32} = topMask{x} != 0
|
||||
def topAll{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<<k)-1
|
||||
def any_top{x:T=[_]E if w256i{T} and width{E}>=32} = topMask{x} != 0
|
||||
def all_top{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<<k)-1
|
||||
|
||||
@ -39,18 +39,18 @@ def topMask{x:T if w256{T, 16}} = {
|
||||
(msk&255) | (msk>>8)
|
||||
}
|
||||
|
||||
def homAny{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
|
||||
def homAll{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
|
||||
def topAny{x:T if w256i{T}} = topMask{x} != 0
|
||||
def topAll{x:T=[k]_ if w256i{T}} = topMask{x} == (1<<k)-1
|
||||
def any_hom{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
|
||||
def all_hom{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
|
||||
def any_top{x:T if w256i{T}} = topMask{x} != 0
|
||||
def all_top{x:T=[k]_ if w256i{T}} = topMask{x} == (1<<k)-1
|
||||
def homMask{a:T, b:T if w256i{T,16}} = homMask{vec_shuffle{[4]u64, packQ{ty_s{a},ty_s{b}}, 0,2,1,3}}
|
||||
|
||||
def topAny{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
|
||||
def topAny{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
|
||||
def homAny{x:T if w256i{T} and elwidth{T}>=32} = topAny{x}
|
||||
def any_top{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
|
||||
def any_top{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
|
||||
def any_hom{x:T if w256i{T} and elwidth{T}>=32} = any_top{x}
|
||||
|
||||
def topAny{x:T if w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0}
|
||||
def topAll{x:T if w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0}
|
||||
def any_top{x:T if w256i{T,16}} = any_hom{[16]i16~~x < [16]i16**0}
|
||||
def all_top{x:T if w256i{T,16}} = all_hom{[16]i16~~x < [16]i16**0}
|
||||
|
||||
|
||||
# conversion
|
||||
|
||||
@ -174,7 +174,7 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
|
||||
# base cases
|
||||
def {
|
||||
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
|
||||
homAll,homAny,bitAll,bitAny,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
all_hom,any_hom,all_bit,any_bit,all_top,any_top,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
|
||||
pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
|
||||
blend_top,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,
|
||||
|
||||
@ -36,8 +36,8 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = {
|
||||
ne := jv != load{*V~~(x+1)}; dc := -ne
|
||||
# Quickly skip ahead if initial values are all equal
|
||||
a:u64 = 1
|
||||
if (not homAny{ne} and b>=4) {
|
||||
def eq_k{k} = homAll{tree_fold{&, @unroll(x in xv+a over k) x==jv}}
|
||||
if (not any_hom{ne} and b>=4) {
|
||||
def eq_k{k} = all_hom{tree_fold{&, @unroll(x in xv+a over k) x==jv}}
|
||||
def skip_eq{k} = if (eq_k{k}) { a=2*k; skip_eq{2*k} }
|
||||
def skip_eq{k==4} = while (a<=b-k and eq_k{k}) a+=k
|
||||
skip_eq{1}
|
||||
@ -86,7 +86,7 @@ fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = {
|
||||
def V = [vl]u16
|
||||
def bot = 1<<15 - 1
|
||||
on:usz = 0
|
||||
@for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{topAny{t}}) {
|
||||
@for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{any_top{t}}) {
|
||||
o := if (hasarch{'X86_64'}) topMask{t} else homMask{t > V**bot}
|
||||
if (jv == n/vl) o &= type{o}~~1<<(n%vl) - 1
|
||||
while (o > 0) {
|
||||
|
||||
@ -18,8 +18,8 @@ def bqn_or{a, b} = (a+b)-(a*b)
|
||||
|
||||
|
||||
# + & -
|
||||
def arithChk1{(__add), M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}}
|
||||
def arithChk1{(__sub), M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}}
|
||||
def arithChk1{(__add), M, w:T, x:T, r:T} = tup{'any_top', M{(w^r) & (x^r)}}
|
||||
def arithChk1{(__sub), M, w:T, x:T, r:T} = tup{'any_top', M{(w^x) & (w^r)}}
|
||||
def arithChk1{(__add), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', adds{w,x}, r}
|
||||
def arithChk1{(__sub), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', subs{w,x}, r}
|
||||
|
||||
@ -37,7 +37,7 @@ def arithChk2{F=(__mul), M, w:T=[_](i8), x:T if hasarch{'X86_64'}} = {
|
||||
def rp = each{__mul, wp, xp}
|
||||
if (M{0}) { # masked check
|
||||
def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp}
|
||||
tup{packQ{rp}, tup{'homAny', M{packQ{bad}}}}
|
||||
tup{packQ{rp}, tup{'any_hom', M{packQ{bad}}}}
|
||||
} else { # unmasked check; can do check in a simpler way
|
||||
def bad = each{{v} => ty_s{v ^ (v>>15)}, rp}
|
||||
def RU = re_el{u16,T}
|
||||
@ -53,7 +53,7 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = {
|
||||
max:= re_el{f32, (ty_u{T})**0x4efffffe}
|
||||
def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x}
|
||||
f32mul:= cf32{w} * cf32{x}
|
||||
tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}}
|
||||
tup{w*x, tup{'any_hom', M{abs{f32mul} >= max}}}
|
||||
# TODO fallback to the below if the above fails
|
||||
# TODO don't do this, but instead shuffle one half, do math, unshuffle that half
|
||||
# def wp = mzip128{w, T**0}
|
||||
@ -63,12 +63,12 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = {
|
||||
# def bad = each{{v} => {
|
||||
# ((T2~~v + T2**0x80000000) ^ T2**(cast{i64,1}<<63)) > T2**cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF}
|
||||
# }, rp}
|
||||
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'homAny', tree_fold{|,bad}}} this doesn't use M
|
||||
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'any_hom', tree_fold{|,bad}}} this doesn't use M
|
||||
}
|
||||
|
||||
def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = {
|
||||
def {rl, rh} = pack{...mulw{w, x}}
|
||||
tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}}
|
||||
tup{rl, tup{'any_hom', M{rh != (rl >> (width{E}-1))}}}
|
||||
}
|
||||
|
||||
|
||||
@ -97,22 +97,22 @@ def runner{u, R, F} = {
|
||||
|
||||
def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} or same{F,__sub})} = { # 'a'+1, 'a'-1
|
||||
r:= F{ty_u{w}, ty_u{x}}
|
||||
tup{re_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}}
|
||||
tup{re_el{R, VW}~~r, tup{'any_hom', M{r > type{r}**1114111}}}
|
||||
}
|
||||
run
|
||||
}
|
||||
|
||||
# homAny, topAny already give masked vals; anyne doesn't, and ~andAllZero assumes no masking
|
||||
# any_hom, any_top already give masked vals; anyne doesn't, and ~andAllZero assumes no masking
|
||||
def runChecks_any{F, vals} = { F{tree_fold{|, each{select{.,1}, vals}}} }
|
||||
def runChecks{'homAny', vals, M} = runChecks_any{homAny, vals}
|
||||
def runChecks{'topAny', vals, M} = runChecks_any{topAny, vals}
|
||||
def runChecks{'any_hom', vals, M} = runChecks_any{any_hom, vals}
|
||||
def runChecks{'any_top', vals, M} = runChecks_any{any_top, vals}
|
||||
def runChecks{'none', vals, M} = 0
|
||||
def runChecks{'~andAllZero', vals, M if ~M{0}} = ~tree_fold{&, each{andAllZero, ...slice{flip{vals}, 1}}}
|
||||
def runChecks{'anyne', vals, M} = {
|
||||
def i{vals} = {
|
||||
def {_,xs,ys} = flip{vals}
|
||||
assert{M{0} == 0}
|
||||
~homAll{tree_fold{&, each{==, xs, ys}}}
|
||||
~all_hom{tree_fold{&, each{==, xs, ys}}}
|
||||
}
|
||||
def i{{{_,x,y}}} = anyne{x, y, M}
|
||||
i{vals}
|
||||
|
||||
@ -56,7 +56,7 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
|
||||
# def [_]IE = oneType{is}
|
||||
# def shl = if (IE==u8) __shl{u16} else __shl
|
||||
# def bm = shl{is, width{IE}-1 - lb{nth}}
|
||||
# each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}}
|
||||
# each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}}
|
||||
# }
|
||||
def me{is:[_](u8) if hasarch{'X86_64'} and E==u32} = {
|
||||
each{{l,h,m} => blend_top{l,h,m}, lo{is}, hi{is}, each{{c} => c << (31 - lb{nth}), widen_tup{E, is}}}
|
||||
@ -76,7 +76,7 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
|
||||
# if (mode=='c') hi{'xor', lo}
|
||||
# def me{is:I=[_](u8) if mode=='c'} = {
|
||||
# def bm = (is & I**lb{nth})
|
||||
# each{{l,h} => topBlend{l,h,bm}, lo{is}, hi{is}}
|
||||
# each{{l,h} => blend_top{l,h,bm}, lo{is}, hi{is}}
|
||||
# }
|
||||
}}}
|
||||
|
||||
|
||||
@ -14,8 +14,8 @@ def maskOf{T,n if w256{T}} = maskOfImpl{T, n, 256}
|
||||
def maskOf{T,n if w128{T}} = maskOfImpl{T, n, 128}
|
||||
def maskOf{T,n if w64{T}} = maskOfImpl{T, n, 64}
|
||||
|
||||
def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~homAll{x==y}
|
||||
def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = homAny{M{x!=y}}
|
||||
def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~all_hom{x==y}
|
||||
def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = any_hom{M{x!=y}}
|
||||
def anyne{x:T, y:T, M if M{0}==0 and anyInt{x}} = x!=y
|
||||
def anyne{x:T, y:T, M if M{0}==1 and anyInt{x}} = M{x^y} != 0
|
||||
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
||||
@ -64,9 +64,9 @@ def loadBatch{ptr:*E0, i, [k]E1} = {
|
||||
def loadBatch {ptr:*E, {...ns}, T } = each{loadBatch {ptr, ., T }, ns}
|
||||
def storeBatch{ptr:*E, {...ns}, xs, M} = each{storeBatch{ptr, ., ., M}, ns, xs}
|
||||
|
||||
# TODO also similar homAny & use those more
|
||||
def homAll{(maskNone), ...xs} = homAll{...xs}
|
||||
def homAll{M, x:T if kgen{M}} = ~homAny{M{~x}} # TODO better
|
||||
# TODO also similar any_hom & use those more
|
||||
def all_hom{(maskNone), ...xs} = all_hom{...xs}
|
||||
def all_hom{M, x:T if kgen{M}} = ~any_hom{M{~x}} # TODO better
|
||||
|
||||
# "harmless" pointer cast that'll only cast void*
|
||||
def hCast{T,p} = assert{0, 'expected pointer with element',T,'or void but got ',p}
|
||||
|
||||
@ -9,7 +9,7 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
|
||||
def bulk = arch_defvw/width{T}
|
||||
def VT = [bulk]T
|
||||
@muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
|
||||
if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk}
|
||||
if (T!=f64 and any_hom{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk}
|
||||
sr{each{abs, cx}}
|
||||
}
|
||||
len
|
||||
|
||||
@ -116,27 +116,27 @@ def narrowPair{a:T=[_]E, b:T if isint{E}} = pack{a, b, 0}
|
||||
def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x}
|
||||
def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}}
|
||||
|
||||
def homAll{x:V if nvec{V}} = bitAll{x}
|
||||
def all_hom{x:V if nvec{V}} = all_bit{x}
|
||||
|
||||
local def bit_any_lo64{x:V} = extract{re_el{u64,x}, 0} != 0
|
||||
local def hom_any_lo64{x:V} = extract{re_el{f64,x}, 0} != 0.0 # safe to use float comparison as homogeneity guarantees not hitting -0.0; https://lemire.me/blog/2025/01/20/checking-whether-an-arm-neon-register-is-zero/
|
||||
local def bit_all_lo64{x:V} = extract{re_el{i64,x}, 0} == -1 # no special hom_all_lo64 possible with float comparison
|
||||
local def min_pack{x} = re_el{u64, minp{...2**re_el{u32,x}}}
|
||||
local def max_pack{x} = re_el{u64, maxp{...2**re_el{u32,x}}}
|
||||
def bitAny{x:V if w64{V}} = bit_any_lo64{x}
|
||||
def bitAny{x:V if w128{V}} = bit_any_lo64{max_pack{x}}
|
||||
def homAny{x:V if w64{V}} = hom_any_lo64{x}
|
||||
def homAny{x:V if w128{V}} = hom_any_lo64{max_pack{x}}
|
||||
def bitAll{x:V if w64{V}} = bit_all_lo64{x}
|
||||
def bitAll{x:V if w128{V}} = bit_all_lo64{min_pack{x}}
|
||||
def any_bit{x:V if w64{V}} = bit_any_lo64{x}
|
||||
def any_bit{x:V if w128{V}} = bit_any_lo64{max_pack{x}}
|
||||
def any_hom{x:V if w64{V}} = hom_any_lo64{x}
|
||||
def any_hom{x:V if w128{V}} = hom_any_lo64{max_pack{x}}
|
||||
def all_bit{x:V if w64{V}} = bit_all_lo64{x}
|
||||
def all_bit{x:V if w128{V}} = bit_all_lo64{min_pack{x}}
|
||||
|
||||
def homAll{x:V if w128{V} and elwidth{V}>=16} = ~hom_any_lo64{subnHi{[8]u16**16r2b1, [8]u16~~x}}
|
||||
def homAny{x:V if w128{V} and elwidth{V}>=16} = hom_any_lo64{narrow{u8,[8]u16~~x}} # narrow probably better than maxp
|
||||
# TODO multi-vector homAny for ew≥16 via addhn?
|
||||
def all_hom{x:V if w128{V} and elwidth{V}>=16} = ~hom_any_lo64{subnHi{[8]u16**16r2b1, [8]u16~~x}}
|
||||
def any_hom{x:V if w128{V} and elwidth{V}>=16} = hom_any_lo64{narrow{u8,[8]u16~~x}} # narrow probably better than maxp
|
||||
# TODO multi-vector any_hom for ew≥16 via addhn?
|
||||
|
||||
|
||||
def topAny{x:V if nvec{V}} = fold_min{ty_s{x}}<0
|
||||
def topAll{x:V if nvec{V}} = fold_max{ty_s{x}}<0
|
||||
def any_top{x:V if nvec{V}} = fold_min{ty_s{x}}<0
|
||||
def all_top{x:V if nvec{V}} = fold_max{ty_s{x}}<0
|
||||
|
||||
|
||||
def homMask{x:T=[k]E if nvecu{T} and width{E}>=k} = {
|
||||
@ -168,7 +168,7 @@ def homMask{a:T,b:T=[k]E if k*2<=width{E}} = {
|
||||
truncBits{k*2, fold_add{shrm{a,width{E}-k,b} & make{T, (1<<iota{k}) | (1<<(iota{k}+k))}}}
|
||||
}
|
||||
|
||||
def andAllZero{x:T, y:T if nveci{T}} = ~bitAny{x&y}
|
||||
def andAllZero{x:T, y:T if nveci{T}} = ~any_bit{x&y}
|
||||
|
||||
def homMaskX{a:T=[k]E if E!=u64} = {
|
||||
def h = width{E}/2
|
||||
|
||||
@ -50,7 +50,7 @@ fn scan_idem{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
@for (ik to ek) { i := ik * k
|
||||
def ii = iota{k}
|
||||
xvi := each{load{xv + i, .}, ii}
|
||||
if (not homAny{cmp{p, tree_fold{op, xvi}}}) {
|
||||
if (not any_hom{cmp{p, tree_fold{op, xvi}}}) {
|
||||
each{store{rv+i, ., p}, ii}
|
||||
} else @unroll (rv in rv+i over j to k) {
|
||||
rv = scan{select{xvi,j}, p}
|
||||
@ -278,7 +278,7 @@ def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
|
||||
def cx0 = tup{load{*[bulk]X~~(x+i)}}
|
||||
def cx = if(wd) widenFull{WE,cx0} else cx0
|
||||
if (~wd) { # within-vector overflow check; widening gives range space for this to not happen
|
||||
if (rare{homAny{tree_fold{|, each{{c:T} => absu{c} >= (ty_u{T}**maxFastE), cx}}}}) goto{'end'}
|
||||
if (rare{any_hom{tree_fold{|, each{{c:T} => absu{c} >= (ty_u{T}**maxFastE), cx}}}}) goto{'end'}
|
||||
}
|
||||
|
||||
def s0 = each{scan_plus, cx}
|
||||
|
||||
@ -27,9 +27,9 @@ def search{E, x, n:(u64), OP} = {
|
||||
}
|
||||
@muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
|
||||
eq:= each{OP, x}
|
||||
if (homAny{M{tree_fold{|, eq}}}) {
|
||||
if (any_hom{M{tree_fold{|, eq}}}) {
|
||||
findFirst{
|
||||
{i,c} => homAny{c},
|
||||
{i,c} => any_hom{c},
|
||||
{i,c} => tup{i,c},
|
||||
end,
|
||||
is, eq
|
||||
@ -54,7 +54,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
|
||||
def bulk = arch_defvw/width{E}
|
||||
def VT = [bulk]E
|
||||
@maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
|
||||
if (homAny{M{x!=x}}) return{1}
|
||||
if (any_hom{M{x!=x}}) return{1}
|
||||
sr{x + VT**0}
|
||||
}
|
||||
0
|
||||
@ -325,7 +325,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
|
||||
def minA = acc{2, VT**min1}
|
||||
def maxA = acc{2, VT**min1}
|
||||
@muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
|
||||
if (E==f64 and homAny{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0}
|
||||
if (E==f64 and any_hom{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0}
|
||||
minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend
|
||||
maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend
|
||||
}
|
||||
@ -637,7 +637,7 @@ def try_vec_memb{T==u32, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done
|
||||
@unroll (f in fp+u*i over a to u) c |= V**(1<<(rw*a)) & test{f}
|
||||
r = vswap ^ vfold{|, c}
|
||||
}
|
||||
@for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ homAny{test{fp}}}
|
||||
@for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ any_hom{test{fp}}}
|
||||
goto{done}
|
||||
}
|
||||
def try{nv} = {
|
||||
|
||||
@ -25,7 +25,7 @@ if_inline (hasarch{'AVX2'}) {
|
||||
|
||||
def wrapChk{cw0:VI, xlf, M} = {
|
||||
cw:= cw0 + (xlf & VI~~(cw0<VI**0))
|
||||
if (homAny{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
if (any_hom{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
cw
|
||||
}
|
||||
|
||||
|
||||
@ -3,22 +3,22 @@ include './mask'
|
||||
include './cbqnDefs'
|
||||
include './vecfold'
|
||||
|
||||
def is_sNaN{x:[_](u64)} = tup{homAny, inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2}}
|
||||
def is_sNaN{x:[_](u64)} = tup{any_hom, inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2}}
|
||||
def is_sNaN{x:[2](u64) if hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = { # avoiding i64 comparisons
|
||||
def nan = unord{[2]f64~~x, [2]f64~~x}
|
||||
def qnan = re_el{u64, [4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000)}
|
||||
tup{topAny, nan &~ qnan}
|
||||
tup{any_top, nan &~ qnan}
|
||||
}
|
||||
def any_sNaN{M, ...xs} = {
|
||||
def {any, are} = flip{each{is_sNaN, xs}}
|
||||
oneVal{any}{M{tree_fold{|, are}}}
|
||||
}
|
||||
|
||||
def any_nonC32{M, x:[_](u64)} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
|
||||
def any_nonC32{M, x:[_](u64)} = any_hom{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
|
||||
def any_nonC32{M, x:T=[_]_ if hasarch{'X86_64'}} = {
|
||||
def H = re_el{u32, T}
|
||||
def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16}
|
||||
topAny{M{T~~ne}}
|
||||
any_top{M{T~~ne}}
|
||||
}
|
||||
def any_nonC32{(maskNone), x:[k](u64), y:[k](u64)} = {
|
||||
def T32 = [k*2]u32
|
||||
@ -86,7 +86,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
if (B) {
|
||||
if (any_nonC32{M, ...xs}) return{3}
|
||||
} else {
|
||||
def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}}
|
||||
def bad = if (hasarch{'AARCH64'}) any_bit{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}}
|
||||
else ~andAllZero{orx, ~XV**maxvalue{w_h{X}}}
|
||||
if (bad) return{lb{width{X}}-3}
|
||||
}
|
||||
@ -103,13 +103,13 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
@muLoop{bulk, 4}(xs in xb, M in 'm' over len) {
|
||||
minc:= tree_fold{min, xs}
|
||||
maxc:= tree_fold{max, xs}
|
||||
if (M{0}==0 and homAny{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff}
|
||||
if (M{0}==0 and any_hom{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff}
|
||||
acc{M, minc, maxc}
|
||||
}
|
||||
} else {
|
||||
@muLoop{bulk, 2}(xs in xb, M in 'm' over len) {
|
||||
def mixed = acc{M, xs}
|
||||
if (M{0}==0 and homAny{mixed > XV**maxvalue{EH}}) return{0xffff_ffff}
|
||||
if (M{0}==0 and any_hom{mixed > XV**maxvalue{EH}}) return{0xffff_ffff}
|
||||
}
|
||||
}
|
||||
acc{}
|
||||
@ -132,7 +132,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
@muLoop{bulk, 2}(xs in xb, M in 'm' over is to len) {
|
||||
if (hasarch{'X86_64'}) {
|
||||
def ns = each{narrow{i32,.}, xs}
|
||||
if (homAny{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}}
|
||||
if (any_hom{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}}
|
||||
acc{M, ns}
|
||||
} else {
|
||||
assert{hasarch{'AARCH64'}}
|
||||
@ -141,7 +141,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
def int32 = if (k==2) pack{...int, 0} else pack{...int, [bulk]i64**0, 0}
|
||||
def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widenUpper}, 0, k}}
|
||||
|
||||
if (~homAll{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}}
|
||||
if (~all_hom{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}}
|
||||
acc{M, tup{int32}}
|
||||
}
|
||||
}
|
||||
|
||||
@ -40,7 +40,7 @@ def rsqrtE{a:([4]f32)} = emit{[4]f32, '_mm_rsqrt_ps', a}
|
||||
def rcpE{a:([4]f32)} = emit{[4]f32, '_mm_rcp_ps', a}
|
||||
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T if w128i{T}} = homAll{(x & y) == T**0}
|
||||
def andAllZero{x:T, y:T if w128i{T}} = all_hom{(x & y) == T**0}
|
||||
|
||||
def topMask{x:T if w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
|
||||
def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
|
||||
@ -50,13 +50,13 @@ def homMask{x:T if w128{T}} = topMask{x}
|
||||
def homMaskX{a:[_]T if width{T}==16} = tup{2, homMask{re_el{u8,a}}}
|
||||
def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}}
|
||||
|
||||
def homAny{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0
|
||||
def homAll{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
|
||||
def any_hom{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0
|
||||
def all_hom{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
|
||||
|
||||
def topAny{x:T if w128i{T}} = topMask{x} != 0
|
||||
def topAll{x:T=[k]_ if w128i{T}} = topMask{x} == (1<<k)-1
|
||||
def topAny{x:T if w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
|
||||
def topAll{x:T if w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}
|
||||
def any_top{x:T if w128i{T}} = topMask{x} != 0
|
||||
def all_top{x:T=[k]_ if w128i{T}} = topMask{x} == (1<<k)-1
|
||||
def any_top{x:T if w128i{T, 16}} = any_hom{[8]i16~~x < [8]i16**0}
|
||||
def all_top{x:T if w128i{T, 16}} = all_hom{[8]i16~~x < [8]i16**0}
|
||||
|
||||
# bits of other things SSE2 has
|
||||
def packs{a:T,b:T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user