(hom|top)Mask → (hom|top)_to_int
This commit is contained in:
parent
e9e1574d28
commit
e3b30e5db7
@ -170,11 +170,11 @@ Homogeneous definitions (i.e. ones with `hom` in their name) assume that each el
|
|||||||
- `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1`
|
- `blend_hom{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1`
|
||||||
- `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m`
|
- `blend_top{f:V, t:V, m:V} : V` - blend by top bit of `m`
|
||||||
- `blend_bit{f:V, t:V, m:M} : V` - bitwise blend
|
- `blend_bit{f:V, t:V, m:M} : V` - bitwise blend
|
||||||
- `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal)
|
- `hom_to_int{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal)
|
||||||
- `homMask{...vs} : uint` - merged mask of `each{homMask,vs}`
|
- `hom_to_int{...vs} : uint` - merged mask of `each{hom_to_int,vs}`
|
||||||
- `topMask{a:VI} : uint` - integer mask of the top bit of each element
|
- `top_to_int{a:VI} : uint` - integer mask of the top bit of each element
|
||||||
- `homMaskX{a:VI} : tup{knum, uint}` - integer mask where each element is represented by `knum` bits (possibly more efficient to calculate than `homMask`)
|
- `hom_to_int_ext{a:VI} : tup{knum, uint}` - integer mask where each element is represented by `knum` bits (possibly more efficient to calculate than `hom_to_int`)
|
||||||
- `ctzX{tup{knum, uint}}` - count trailing zeroes from a result of `homMaskX`
|
- `ctz_ext{tup{knum, uint}}` - count trailing zeroes from a result of `hom_to_int_ext`
|
||||||
|
|
||||||
## Load/store
|
## Load/store
|
||||||
|
|
||||||
|
|||||||
@ -49,12 +49,12 @@ def shufInd{a:T, b:T=[4]E, {...is} if width{E}==64 and length{is}==4} = T~~shufI
|
|||||||
# mask stuff
|
# mask stuff
|
||||||
def andAllZero{x:T, y:T if w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
|
def andAllZero{x:T, y:T if w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
|
||||||
|
|
||||||
def topMask{x:T if w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
|
def top_to_int{x:T if w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
|
||||||
def topMask{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
|
def top_to_int{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
|
||||||
def homMask{x:T if w256{T}} = topMask{x}
|
def hom_to_int{x:T if w256{T}} = top_to_int{x}
|
||||||
|
|
||||||
def any_hom{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0
|
def any_hom{x:T if w256i{T} and elwidth{T}>=32} = hom_to_int{[8]u32 ~~ x} != 0
|
||||||
def all_hom{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff
|
def all_hom{x:T if w256i{T} and elwidth{T}>=32} = hom_to_int{[8]u32 ~~ x} == 0xff
|
||||||
|
|
||||||
def any_top{x:T=[_]E if w256i{T} and width{E}>=32} = topMask{x} != 0
|
def any_top{x:T=[_]E if w256i{T} and width{E}>=32} = top_to_int{x} != 0
|
||||||
def all_top{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<<k)-1
|
def all_top{x:T=[k]E if w256i{T} and width{E}>=32} = top_to_int{x} == (1<<k)-1
|
||||||
|
|||||||
@ -33,17 +33,17 @@ def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{
|
|||||||
def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}}
|
def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}}} = store{p, 0, blend_hom{load{p}, v, m}}
|
||||||
|
|
||||||
# mask stuff
|
# mask stuff
|
||||||
def topMask{x:T if w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
|
def top_to_int{x:T if w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
|
||||||
def topMask{x:T if w256{T, 16}} = {
|
def top_to_int{x:T if w256{T, 16}} = {
|
||||||
msk:u32 = topMask{emit{[32]u8, '_mm256_packs_epi16', x, [16]u16**0}}
|
msk:u32 = top_to_int{emit{[32]u8, '_mm256_packs_epi16', x, [16]u16**0}}
|
||||||
(msk&255) | (msk>>8)
|
(msk&255) | (msk>>8)
|
||||||
}
|
}
|
||||||
|
|
||||||
def any_hom{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
|
def any_hom{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
|
||||||
def all_hom{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
|
def all_hom{x:T if w256i{T}} = hom_to_int{[32]u8 ~~ x} == 0xffff_ffff
|
||||||
def any_top{x:T if w256i{T}} = topMask{x} != 0
|
def any_top{x:T if w256i{T}} = top_to_int{x} != 0
|
||||||
def all_top{x:T=[k]_ if w256i{T}} = topMask{x} == (1<<k)-1
|
def all_top{x:T=[k]_ if w256i{T}} = top_to_int{x} == (1<<k)-1
|
||||||
def homMask{a:T, b:T if w256i{T,16}} = homMask{vec_shuffle{[4]u64, packQ{ty_s{a},ty_s{b}}, 0,2,1,3}}
|
def hom_to_int{a:T, b:T if w256i{T,16}} = hom_to_int{vec_shuffle{[4]u64, packQ{ty_s{a},ty_s{b}}, 0,2,1,3}}
|
||||||
|
|
||||||
def any_top{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
|
def any_top{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
|
||||||
def any_top{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
|
def any_top{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
|
||||||
|
|||||||
@ -31,8 +31,8 @@ def maskStore{p:*V, m:[l](u1), v:V=[l]_ if has512e{V}} = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def topMaskReg{x:V=[k]_} = emit{[k]u1, merge{pref{V},'mov',suf{V},'_mask'}, x}
|
def topMaskReg{x:V=[k]_} = emit{[k]u1, merge{pref{V},'mov',suf{V},'_mask'}, x}
|
||||||
def topMask{x:V=[k]_ if 512==width{V}} = ty_u{k}~~topMaskReg{x}
|
def top_to_int{x:V=[k]_ if 512==width{V}} = ty_u{k}~~topMaskReg{x}
|
||||||
def homMask{x:V=[_]_ if 512==width{V}} = topMask{x}
|
def hom_to_int{x:V=[_]_ if 512==width{V}} = top_to_int{x}
|
||||||
|
|
||||||
def maskToHom{V=[l]_, x:[l](u1)} = emit{V, merge{pref{V},'movm_',suf{V}}, x}
|
def maskToHom{V=[l]_, x:[l](u1)} = emit{V, merge{pref{V},'movm_',suf{V}}, x}
|
||||||
|
|
||||||
|
|||||||
@ -175,8 +175,8 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
|
|||||||
def {
|
def {
|
||||||
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
|
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
|
||||||
all_bit,any_bit,blend_bit,
|
all_bit,any_bit,blend_bit,
|
||||||
all_hom,any_hom,blend_hom,homMask,homMaskStore,homMaskStoreF,
|
all_hom,any_hom,blend_hom,hom_to_int,homMaskStore,homMaskStoreF,
|
||||||
all_top,any_top,blend_top,topMask,topMaskStore,topMaskStoreF,
|
all_top,any_top,blend_top,top_to_int,topMaskStore,topMaskStoreF,
|
||||||
loadBatchBit,loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,
|
loadBatchBit,loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,
|
||||||
packQ,pair,pdep,pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
|
packQ,pair,pdep,pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
|
||||||
unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,multishift,
|
unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,multishift,
|
||||||
@ -186,19 +186,19 @@ def {
|
|||||||
def blend_bit{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m))
|
def blend_bit{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m))
|
||||||
def blend_hom{f:T, t:T, m:M} = blend_bit{f, t, m}
|
def blend_hom{f:T, t:T, m:M} = blend_bit{f, t, m}
|
||||||
|
|
||||||
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
|
def hom_to_int_ext{a:T} = tup{1, hom_to_int{a}} # tup{n,mask}; mask with each bit repeated n times
|
||||||
def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
|
def ctz_ext{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
|
||||||
def homMask{...vs if length{vs}>1} = {
|
def hom_to_int{...vs if length{vs}>1} = {
|
||||||
def n = length{vs}
|
def n = length{vs}
|
||||||
def [k]_ = oneType{vs}
|
def [k]_ = oneType{vs}
|
||||||
def RT = ty_u{max{8,k*n}}
|
def RT = ty_u{max{8,k*n}}
|
||||||
def sl{...a} = promote{RT, homMask{...slice{vs,...a}}}
|
def sl{...a} = promote{RT, hom_to_int{...slice{vs,...a}}}
|
||||||
def h = n/2
|
def h = n/2
|
||||||
def lo = sl{0,h}
|
def lo = sl{0,h}
|
||||||
def hi = sl{h}
|
def hi = sl{h}
|
||||||
(hi << (h*k)) | lo
|
(hi << (h*k)) | lo
|
||||||
}
|
}
|
||||||
def homMask{x if ktup{x}} = homMask{...x}
|
def hom_to_int{x if ktup{x}} = hom_to_int{...x}
|
||||||
|
|
||||||
if_inline (hasarch{'X86_64'}) {
|
if_inline (hasarch{'X86_64'}) {
|
||||||
include 'arch/iintrinsic/basic'
|
include 'arch/iintrinsic/basic'
|
||||||
|
|||||||
@ -143,7 +143,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
|
|||||||
def nb = 256/vl
|
def nb = 256/vl
|
||||||
nu:u8 = 0; def addu{b} = { nu+=cast_i{u8,popc{b}}; b } # Number of uniques
|
nu:u8 = 0; def addu{b} = { nu+=cast_i{u8,popc{b}}; b } # Number of uniques
|
||||||
vb := U~~make{[nb](ty_u{vl}),
|
vb := U~~make{[nb](ty_u{vl}),
|
||||||
@collect (t in *V~~t0 over nb) addu{homMask{t > V**0}}
|
@collect (t in *V~~t0 over nb) addu{hom_to_int{t > V**0}}
|
||||||
}
|
}
|
||||||
dup := promote{u64,nu} < wn
|
dup := promote{u64,nu} < wn
|
||||||
# Unique index to w index conversion
|
# Unique index to w index conversion
|
||||||
@ -290,7 +290,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
|
|||||||
if (isvec{type{rn}}) store{rnp, 0, rn}
|
if (isvec{type{rn}}) store{rnp, 0, rn}
|
||||||
else storeu{rnp, rn}
|
else storeu{rnp, rn}
|
||||||
} else {
|
} else {
|
||||||
def B = ty_u{vl}; out := cast_i{B, homMask{b}}
|
def B = ty_u{vl}; out := cast_i{B, hom_to_int{b}}
|
||||||
store{*B~~rp, cdiv{j,vl}, out>>((-j)%vl)}
|
store{*B~~rp, cdiv{j,vl}, out>>((-j)%vl)}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -73,7 +73,7 @@ def any2bit{VT=[k]T, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = {
|
|||||||
{(__ne) if isint{T}} => __eq
|
{(__ne) if isint{T}} => __eq
|
||||||
{_} => op0
|
{_} => op0
|
||||||
}
|
}
|
||||||
def mask = if (same{op0, op}) homMask else ({x} => ~homMask{x})
|
def mask = if (same{op0, op}) hom_to_int else ({x} => ~hom_to_int{x})
|
||||||
|
|
||||||
@forNZ (ri to cdiv{len,bulk}) {
|
@forNZ (ri to cdiv{len,bulk}) {
|
||||||
b_setBatch{bulk, dst, ri, mask{each{{j}=>op{wV{xi+j}, xV{xi+j}}, iota{unr}}}}
|
b_setBatch{bulk, dst, ri, mask{each{{j}=>op{wV{xi+j}, xV{xi+j}}, iota{unr}}}}
|
||||||
|
|||||||
@ -49,7 +49,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
|
|||||||
def bulk2 = bulk*unr
|
def bulk2 = bulk*unr
|
||||||
xi:ux = 0
|
xi:ux = 0
|
||||||
@forNZ (i to cdiv{l,bulk2}) {
|
@forNZ (i to cdiv{l,bulk2}) {
|
||||||
b_setBatch{bulk2, rp, i, homMask{each{{i} => op{loadBatch{xp, xi+i, XV}}, iota{unr}}}}
|
b_setBatch{bulk2, rp, i, hom_to_int{each{{i} => op{loadBatch{xp, xi+i, XV}}, iota{unr}}}}
|
||||||
xi+= unr
|
xi+= unr
|
||||||
}
|
}
|
||||||
} else if (width{X}<=width{R}) {
|
} else if (width{X}<=width{R}) {
|
||||||
|
|||||||
@ -87,7 +87,7 @@ fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = {
|
|||||||
def bot = 1<<15 - 1
|
def bot = 1<<15 - 1
|
||||||
on:usz = 0
|
on:usz = 0
|
||||||
@for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{any_top{t}}) {
|
@for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{any_top{t}}) {
|
||||||
o := if (hasarch{'X86_64'}) topMask{t} else homMask{t > V**bot}
|
o := if (hasarch{'X86_64'}) top_to_int{t} else hom_to_int{t > V**bot}
|
||||||
if (jv == n/vl) o &= type{o}~~1<<(n%vl) - 1
|
if (jv == n/vl) o &= type{o}~~1<<(n%vl) - 1
|
||||||
while (o > 0) {
|
while (o > 0) {
|
||||||
jv := jv*vl + cast_i{usz, ctz{o}}
|
jv := jv*vl + cast_i{usz, ctz{o}}
|
||||||
@ -157,7 +157,7 @@ def mark_run_ends{x:*T, m:(ux)} = {
|
|||||||
@unroll (j to width{ux} / vec) {
|
@unroll (j to width{ux} / vec) {
|
||||||
def jv = j*vec
|
def jv = j*vec
|
||||||
def lv{k} = load{*V~~(x + k)}
|
def lv{k} = load{*V~~(x + k)}
|
||||||
m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv
|
m |= promote{ux, hom_to_int{lv{jv} != lv{jv+1}}} << jv
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
def inc_marked_runs{x, tab:*T, m, m0} = {
|
def inc_marked_runs{x, tab:*T, m, m0} = {
|
||||||
|
|||||||
@ -41,8 +41,8 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
|||||||
def T = [bulk]X
|
def T = [bulk]X
|
||||||
def sh{c} = c << (width{X}-1)
|
def sh{c} = c << (width{X}-1)
|
||||||
def sh{c if X==u8} = T ~~ (re_el{u16,c}<<7)
|
def sh{c if X==u8} = T ~~ (re_el{u16,c}<<7)
|
||||||
def mask{x:X if hasarch{'X86_64'}} = topMask{x}
|
def mask{x:X if hasarch{'X86_64'}} = top_to_int{x}
|
||||||
def mask{x:X if hasarch{'AARCH64'}} = homMask{andnz{x, ~T**0}}
|
def mask{x:X if hasarch{'AARCH64'}} = hom_to_int{andnz{x, ~T**0}}
|
||||||
|
|
||||||
# TODO compare with doing the comparison in vector registers
|
# TODO compare with doing the comparison in vector registers
|
||||||
badBits:= T ** ~(X~~1)
|
badBits:= T ** ~(X~~1)
|
||||||
|
|||||||
@ -22,7 +22,7 @@ def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
|||||||
|
|
||||||
def anynePositive{x:T, y:T, M if M{0}==0} = anyne{x, y, M}
|
def anynePositive{x:T, y:T, M if M{0}==0} = anyne{x, y, M}
|
||||||
def anynePositive{x:T, y:T, M if M{0}==1 and isvec{T}} = {
|
def anynePositive{x:T, y:T, M if M{0}==1 and isvec{T}} = {
|
||||||
def {n,m} = homMaskX{x==y}
|
def {n,m} = hom_to_int_ext{x==y}
|
||||||
def E = tern{type{m}==u64, u64, u32}
|
def E = tern{type{m}==u64, u64, u32}
|
||||||
(promote{E,~m} << (width{E}-M{'count'}*n)) != 0
|
(promote{E,~m} << (width{E}-M{'count'}*n)) != 0
|
||||||
}
|
}
|
||||||
|
|||||||
@ -139,14 +139,14 @@ def any_top{x:V if nvec{V}} = fold_min{ty_s{x}}<0
|
|||||||
def all_top{x:V if nvec{V}} = fold_max{ty_s{x}}<0
|
def all_top{x:V if nvec{V}} = fold_max{ty_s{x}}<0
|
||||||
|
|
||||||
|
|
||||||
def homMask{x:T=[k]E if nvecu{T} and width{E}>=k} = {
|
def hom_to_int{x:T=[k]E if nvecu{T} and width{E}>=k} = {
|
||||||
truncBits{k, fold_add{x & make{T, 1<<iota{k}}}}
|
truncBits{k, fold_add{x & make{T, 1<<iota{k}}}}
|
||||||
}
|
}
|
||||||
def homMask{x:T=[16]E if width{E}==8} = {
|
def hom_to_int{x:T=[16]E if width{E}==8} = {
|
||||||
t:= [8]u16~~sel{[16]u8, x, make{[16]u8, tr_iota{3,0,1,2}}}
|
t:= [8]u16~~sel{[16]u8, x, make{[16]u8, tr_iota{3,0,1,2}}}
|
||||||
fold_add{t & make{[8]u16, (1<<iota{8})*0x0101}}
|
fold_add{t & make{[8]u16, (1<<iota{8})*0x0101}}
|
||||||
}
|
}
|
||||||
def homMask{a:T,b:T=[16]E if width{E}==8} = {
|
def hom_to_int{a:T,b:T=[16]E if width{E}==8} = {
|
||||||
m:= make{[16]u8, 1<<(iota{16}&7)}
|
m:= make{[16]u8, 1<<(iota{16}&7)}
|
||||||
s:= make{[16]u8, (range{16}>>2) | ((range{16}&3)<<2)}
|
s:= make{[16]u8, (range{16}>>2) | ((range{16}&3)<<2)}
|
||||||
# fold_add{addpw{addpw{addp{ty_u{a}&m, ty_u{b}&m}}}<<make{[4]u32,iota{4}*8}}
|
# fold_add{addpw{addpw{addp{ty_u{a}&m, ty_u{b}&m}}}<<make{[4]u32,iota{4}*8}}
|
||||||
@ -156,21 +156,21 @@ def homMask{a:T,b:T=[16]E if width{E}==8} = {
|
|||||||
# t:= shrm{l, 4, h} & make{[16]u8, (1<<(range{16}>>2)) * 0x11}
|
# t:= shrm{l, 4, h} & make{[16]u8, (1<<(range{16}>>2)) * 0x11}
|
||||||
# fold_add{[4]u32~~t}
|
# fold_add{[4]u32~~t}
|
||||||
}
|
}
|
||||||
def homMask{a:T,b:T,c:T,d:T=[16]E if width{E}==8} = {
|
def hom_to_int{a:T,b:T,c:T,d:T=[16]E if width{E}==8} = {
|
||||||
m:= make{[16]u8, 1<<(iota{16}&7)}
|
m:= make{[16]u8, 1<<(iota{16}&7)}
|
||||||
t1:= addp{ty_u{a}&m, ty_u{b}&m}
|
t1:= addp{ty_u{a}&m, ty_u{b}&m}
|
||||||
t2:= addp{ty_u{c}&m, ty_u{d}&m}
|
t2:= addp{ty_u{c}&m, ty_u{d}&m}
|
||||||
t3:= addp{t1, t2}
|
t3:= addp{t1, t2}
|
||||||
extract{[2]u64~~addp{t3,t3},0}
|
extract{[2]u64~~addp{t3,t3},0}
|
||||||
}
|
}
|
||||||
def homMask{...as={a0:[_]E, _, ..._} if width{E}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}}
|
def hom_to_int{...as={a0:[_]E, _, ..._} if width{E}>=32} = hom_to_int{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}}
|
||||||
def homMask{a:T,b:T=[k]E if k*2<=width{E}} = {
|
def hom_to_int{a:T,b:T=[k]E if k*2<=width{E}} = {
|
||||||
truncBits{k*2, fold_add{shrm{a,width{E}-k,b} & make{T, (1<<iota{k}) | (1<<(iota{k}+k))}}}
|
truncBits{k*2, fold_add{shrm{a,width{E}-k,b} & make{T, (1<<iota{k}) | (1<<(iota{k}+k))}}}
|
||||||
}
|
}
|
||||||
|
|
||||||
def andAllZero{x:T, y:T if nveci{T}} = ~any_bit{x&y}
|
def andAllZero{x:T, y:T if nveci{T}} = ~any_bit{x&y}
|
||||||
|
|
||||||
def homMaskX{a:T=[k]E if E!=u64} = {
|
def hom_to_int_ext{a:T=[k]E if E!=u64} = {
|
||||||
def h = width{E}/2
|
def h = width{E}/2
|
||||||
tup{h, truncBits{k*h, extract{[1]u64~~shrn{el_m{T}~~a, h}, 0}}}
|
tup{h, truncBits{k*h, extract{[1]u64~~shrn{el_m{T}~~a, h}, 0}}}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -128,7 +128,7 @@ fn scan_neq{if hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}}(init:u64, x:*u64, r:*u
|
|||||||
def exor64 = clmul{., sse{1<<64 - 2}, 0}
|
def exor64 = clmul{., sse{1<<64 - 2}, 0}
|
||||||
@for (xv in *V~~x, rv in *V~~r over i to cdiv{nw,vcount{V}}) {
|
@for (xv in *V~~x, rv in *V~~r over i to cdiv{nw,vcount{V}}) {
|
||||||
x8 := xor8{xv}
|
x8 := xor8{xv}
|
||||||
hb := sse{topMask{[64]u8~~x8}}
|
hb := sse{top_to_int{[64]u8~~x8}}
|
||||||
xh := exor64{hb} # Exclusive xor of high bits (xh ^ hb for inclusive)
|
xh := exor64{hb} # Exclusive xor of high bits (xh ^ hb for inclusive)
|
||||||
xc := xh ^ carry
|
xc := xh ^ carry
|
||||||
v := x8 ^ V~~maskToHom{[64]u8, [64]u1~~extract{xc,0}}
|
v := x8 ^ V~~maskToHom{[64]u8, [64]u1~~extract{xc,0}}
|
||||||
|
|||||||
@ -23,7 +23,7 @@ def search{E, x, n:(u64), OP} = {
|
|||||||
def VT = [bulk]E
|
def VT = [bulk]E
|
||||||
def end = makeBranch{
|
def end = makeBranch{
|
||||||
tup{u64, ty_u{VT}},
|
tup{u64, ty_u{VT}},
|
||||||
{i,c} => return{i*bulk + promote{u64, ctzX{homMaskX{c}}}}
|
{i,c} => return{i*bulk + promote{u64, ctz_ext{hom_to_int_ext{c}}}}
|
||||||
}
|
}
|
||||||
@muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
|
@muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
|
||||||
eq:= each{OP, x}
|
eq:= each{OP, x}
|
||||||
@ -95,13 +95,13 @@ def bittab_selector{loadtab} = {
|
|||||||
top := hi4 + VI~~((arch_vec{u32}~~(x&~low))>>3)
|
top := hi4 + VI~~((arch_vec{u32}~~(x&~low))>>3)
|
||||||
byte:= sel{[16]i8, t0, hi4^top} | sel{[16]i8, t1, top}
|
byte:= sel{[16]i8, t0, hi4^top} | sel{[16]i8, t1, top}
|
||||||
mask:= sel{[16]i8, b, x & low}
|
mask:= sel{[16]i8, b, x & low}
|
||||||
homMask{(mask & byte) == mask}
|
hom_to_int{(mask & byte) == mask}
|
||||||
}
|
}
|
||||||
|
|
||||||
def selector{x:([16]i8) if hasarch{'AARCH64'}} = {
|
def selector{x:([16]i8) if hasarch{'AARCH64'}} = {
|
||||||
byte:= [16]u8~~sel{tup{t0,t1}, ty_u{x}>>3}
|
byte:= [16]u8~~sel{tup{t0,t1}, ty_u{x}>>3}
|
||||||
mask:= [16]u8**1 << ty_u{x & low}
|
mask:= [16]u8**1 << ty_u{x & low}
|
||||||
res:= homMask{(mask & byte) == mask}
|
res:= hom_to_int{(mask & byte) == mask}
|
||||||
}
|
}
|
||||||
|
|
||||||
def reload{} = { tup{t0,t1} = loadtab{} }
|
def reload{} = { tup{t0,t1} = loadtab{} }
|
||||||
@ -112,11 +112,11 @@ def readbytes{vtab}{} = {
|
|||||||
def [k]_ = VI; def l = 128/k
|
def [k]_ = VI; def l = 128/k
|
||||||
def side{i} = {
|
def side{i} = {
|
||||||
def U = arch_vec{ty_u{k}}
|
def U = arch_vec{ty_u{k}}
|
||||||
def m = @collect (vtab over _ from i to i+l) homMask{vtab} # TODO multi-value homMask
|
def m = @collect (vtab over _ from i to i+l) hom_to_int{vtab} # TODO multi-value hom_to_int
|
||||||
VI~~make{U, if (vcount{U}>l) merge{m,m} else m}
|
VI~~make{U, if (vcount{U}>l) merge{m,m} else m}
|
||||||
}
|
}
|
||||||
def side{i if hasarch{'AARCH64'}} = {
|
def side{i if hasarch{'AARCH64'}} = {
|
||||||
def m = each{homMask, split{4, @collect (vtab over _ from i to i+l) vtab}}
|
def m = each{hom_to_int, split{4, @collect (vtab over _ from i to i+l) vtab}}
|
||||||
VI~~make{[2]u64, m}
|
VI~~make{[2]u64, m}
|
||||||
}
|
}
|
||||||
each{side, l*iota{2}}
|
each{side, l*iota{2}}
|
||||||
@ -220,8 +220,8 @@ def do_bittab{x0:(*void), n:(u64), tab:(*void), u:(u8), t, mode, r0} = {
|
|||||||
# Filter out values equal to the previous, or first new
|
# Filter out values equal to the previous, or first new
|
||||||
def pind = (iota{k}&15) - 1
|
def pind = (iota{k}&15) - 1
|
||||||
prev:= make{VI, each{max{0, .}, pind}}
|
prev:= make{VI, each{max{0, .}, pind}}
|
||||||
e:= ~homMask{v == VI**TI~~xi}
|
e:= ~hom_to_int{v == VI**TI~~xi}
|
||||||
e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}}
|
e&= base{2,pind<0} | ~hom_to_int{v == sel{[16]i8, v, prev}}
|
||||||
if (rbit) rv&= e | -m # Don't remove first bit
|
if (rbit) rv&= e | -m # Don't remove first bit
|
||||||
m&= e
|
m&= e
|
||||||
while (m != 0) {
|
while (m != 0) {
|
||||||
|
|||||||
@ -279,14 +279,14 @@ exportT{'si_select_tab', join{table{select_fn,
|
|||||||
@maskedLoop{16}(cw0 in w, r in *u16~~r0, M in 'm' over i to wl) {
|
@maskedLoop{16}(cw0 in w, r in *u16~~r0, M in 'm' over i to wl) {
|
||||||
def cw = ty_u{wrapChk{cw0, xlf, M}}
|
def cw = ty_u{wrapChk{cw0, xlf, M}}
|
||||||
def byte = shuf{[16]u8, xrev, cw>>3}
|
def byte = shuf{[16]u8, xrev, cw>>3}
|
||||||
r = homMask{ty_s{byte << (cw & VU**7)} < VI**0}
|
r = hom_to_int{ty_s{byte << (cw & VU**7)} < VI**0}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (wl>32 and xl<=16) {
|
if (wl>32 and xl<=16) {
|
||||||
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
|
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
|
||||||
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
|
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
|
||||||
cw:= wrapChk{cw0, xlf, M}
|
cw:= wrapChk{cw0, xlf, M}
|
||||||
sr = homMask{shuf{[16]i8, xb, cw}}
|
sr = hom_to_int{shuf{[16]i8, xb, cw}}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
x:= shuf{[4]u64, load{*VI ~~ x0}, 0,1,0,1}
|
x:= shuf{[4]u64, load{*VI ~~ x0}, 0,1,0,1}
|
||||||
@ -296,7 +296,7 @@ exportT{'si_select_tab', join{table{select_fn,
|
|||||||
cw:= wrapChk{cw0, xlf, M}
|
cw:= wrapChk{cw0, xlf, M}
|
||||||
byte:= shuf{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
|
byte:= shuf{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
|
||||||
mask:= shuf{[16]i8, b, cw & low}
|
mask:= shuf{[16]i8, b, cw & low}
|
||||||
sr = homMask{(mask & byte) == mask}
|
sr = hom_to_int{(mask & byte) == mask}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -42,19 +42,19 @@ def rcpE{a:([4]f32)} = emit{[4]f32, '_mm_rcp_ps', a}
|
|||||||
# mask stuff
|
# mask stuff
|
||||||
def andAllZero{x:T, y:T if w128i{T}} = all_hom{(x & y) == T**0}
|
def andAllZero{x:T, y:T if w128i{T}} = all_hom{(x & y) == T**0}
|
||||||
|
|
||||||
def topMask{x:T if w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
|
def top_to_int{x:T if w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
|
||||||
def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
|
def top_to_int{x:T if w128{T, 16}} = top_to_int{packs{[8]i16~~x, [8]i16**0}}
|
||||||
def topMask{x:T if w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
|
def top_to_int{x:T if w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
|
||||||
def topMask{x:T if w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
|
def top_to_int{x:T if w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
|
||||||
def homMask{x:T if w128{T}} = topMask{x}
|
def hom_to_int{x:T if w128{T}} = top_to_int{x}
|
||||||
def homMaskX{a:[_]T if width{T}==16} = tup{2, homMask{re_el{u8,a}}}
|
def hom_to_int_ext{a:[_]T if width{T}==16} = tup{2, hom_to_int{re_el{u8,a}}}
|
||||||
def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}}
|
def hom_to_int{a:T, b:T if w128i{T,16}} = hom_to_int{packs{ty_s{a},ty_s{b}}}
|
||||||
|
|
||||||
def any_hom{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0
|
def any_hom{x:T if w128i{T}} = hom_to_int{[16]u8 ~~ x} != 0
|
||||||
def all_hom{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
|
def all_hom{x:T if w128i{T}} = hom_to_int{[16]u8 ~~ x} == 0xffff
|
||||||
|
|
||||||
def any_top{x:T if w128i{T}} = topMask{x} != 0
|
def any_top{x:T if w128i{T}} = top_to_int{x} != 0
|
||||||
def all_top{x:T=[k]_ if w128i{T}} = topMask{x} == (1<<k)-1
|
def all_top{x:T=[k]_ if w128i{T}} = top_to_int{x} == (1<<k)-1
|
||||||
def any_top{x:T if w128i{T, 16}} = any_hom{[8]i16~~x < [8]i16**0}
|
def any_top{x:T if w128i{T, 16}} = any_hom{[8]i16~~x < [8]i16**0}
|
||||||
def all_top{x:T if w128i{T, 16}} = all_hom{[8]i16~~x < [8]i16**0}
|
def all_top{x:T if w128i{T, 16}} = all_hom{[8]i16~~x < [8]i16**0}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user