Heuristic filtering when AVX2 Mark Firsts sees multiple new values
This commit is contained in:
parent
9c7c6aabc5
commit
ba43dae908
@ -10,6 +10,8 @@ if (hasarch{'AVX2'}) {
|
||||
}
|
||||
include './mask'
|
||||
|
||||
def ctzi{x} = promote{u64, ctz{x}} # Count trailing zeros, as index
|
||||
|
||||
def findFirst{C, M, F, ...v1} = {
|
||||
def exit = makelabel{}
|
||||
def args = undef{M{...each{{c}=>tupsel{0,c}, v1}}}
|
||||
@ -30,7 +32,7 @@ def search{E, x, n:u64, OP} = {
|
||||
def VT = [bulk]E
|
||||
def end = makeBranch{
|
||||
tup{u64, ty_u{VT}},
|
||||
{i,c} => return{i*bulk + promote{u64, ctz{homMask{c}}}}
|
||||
{i,c} => return{i*bulk + ctzi{homMask{c}}}
|
||||
}
|
||||
|
||||
muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => {
|
||||
@ -122,13 +124,13 @@ fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = {
|
||||
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
|
||||
xv:= *VI~~x0
|
||||
while (i < n) {
|
||||
m:= u32~~cast_i{i32, t} ^ bitsel{load{xv, i/32}}
|
||||
m:= u32~~promote{i32, t} ^ bitsel{load{xv, i/32}}
|
||||
i0:= i
|
||||
i+= 32
|
||||
if (i > n) m&= (~u32~~0)>>((-n)%32)
|
||||
if (m != 0) { # Need to add entries
|
||||
do {
|
||||
settab{load{x, i0 + cast_i{u64,ctz{m}}}}
|
||||
settab{load{x, i0 + ctzi{m}}}
|
||||
m&= m-1 # Clear last bit
|
||||
} while (m != 0)
|
||||
if (u == 0) return{u} # Table is all one value
|
||||
@ -144,7 +146,7 @@ fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi
|
||||
|
||||
# Initialize
|
||||
rev:u1 = nx < nw/4 # Reverse lookup
|
||||
z:= VI**(-cast_i{i8,rev})
|
||||
z:= VI**(-promote{i8,rev})
|
||||
@unroll (vtab over 8) vtab = z
|
||||
u:u8 = 0 # Sum of table, either 0 or 256
|
||||
if (rev) u = fill_bittab(x0, nx, tab, u, 0)
|
||||
@ -174,30 +176,44 @@ fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = {
|
||||
store{btab, x, 0}
|
||||
v
|
||||
}
|
||||
def settab{T, x} = T~~promote{ty_s{T}, settab{x}}
|
||||
def settab1{x} = { store{btab, x, 0}; ++u }
|
||||
|
||||
# Do first 64 values with a scalar loop
|
||||
x:= *u8~~x0
|
||||
i:u64 = 64; if (n<=64) i=n
|
||||
i:u64 = 32; if (n<=48) i=n
|
||||
rw:u64 = 0
|
||||
@for (x over j to i) rw|= u64~~cast_i{i64,settab{x}} & ((u64~~1)<<j)
|
||||
@for (x over j to i) rw|= settab{u64, x} & ((u64~~1)<<j)
|
||||
store{*u64~~r0, 0, rw}
|
||||
|
||||
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
|
||||
xv:= *VI~~x0; r:= *u32~~r0
|
||||
while (i < n) {
|
||||
i0:= i; iw:= i/32
|
||||
m:= bitsel{load{xv, iw}} # Mask of possibly-new values
|
||||
v:= load{xv, iw}
|
||||
m:= bitsel{v} # Mask of possibly-new values
|
||||
i+= 32
|
||||
if (i > n) m&= (~u32~~0)>>((-n)%32)
|
||||
if (m == 0) { # No new values
|
||||
store{r, iw, m}
|
||||
} else { # Add values to the table and filter m
|
||||
rv:= m
|
||||
do {
|
||||
new:= settab{load{x, i0 + cast_i{u64,ctz{m}}}}
|
||||
m1:= m-1; m&= m1 # Clear last bit
|
||||
rv&= m1 | u32~~cast_i{i32,new} # Clear if not new
|
||||
} while (m != 0)
|
||||
xi:= load{x, i0 + ctzi{m}}
|
||||
settab1{xi}
|
||||
if ((m&(m-1)) != 0) { # More bits than one
|
||||
# Filter out values equal to the previous, or first new
|
||||
def pind = (iota{32}&15) - 1
|
||||
prev:= make{VI, each{bind{max,0}, pind}}
|
||||
e:= ~homMask{v == VI**TI~~xi}
|
||||
e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}}
|
||||
rv&= e | -m # Don't remove first bit
|
||||
m&= e
|
||||
while (m != 0) {
|
||||
new:= settab{u32, load{x, i0 + ctzi{m}}}
|
||||
m1:= m-1; m&= m1 # Clear low bit
|
||||
rv&= m1 | new # Clear if not new
|
||||
}
|
||||
}
|
||||
store{r, iw, rv}
|
||||
if (u == 0) { # All bytes seen
|
||||
@for (r over _ from iw+1 to cdiv{n,32}) r = 0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user