Heuristic filtering when AVX2 Mark Firsts sees multiple new values

This commit is contained in:
Marshall Lochbaum 2023-04-26 20:16:45 -04:00
parent 9c7c6aabc5
commit ba43dae908

View File

@ -10,6 +10,8 @@ if (hasarch{'AVX2'}) {
}
include './mask'
def ctzi{x} = promote{u64, ctz{x}} # Count trailing zeros, as index
def findFirst{C, M, F, ...v1} = {
def exit = makelabel{}
def args = undef{M{...each{{c}=>tupsel{0,c}, v1}}}
@ -30,7 +32,7 @@ def search{E, x, n:u64, OP} = {
def VT = [bulk]E
def end = makeBranch{
tup{u64, ty_u{VT}},
{i,c} => return{i*bulk + promote{u64, ctz{homMask{c}}}}
{i,c} => return{i*bulk + ctzi{homMask{c}}}
}
muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => {
@ -122,13 +124,13 @@ fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = {
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
xv:= *VI~~x0
while (i < n) {
m:= u32~~cast_i{i32, t} ^ bitsel{load{xv, i/32}}
m:= u32~~promote{i32, t} ^ bitsel{load{xv, i/32}}
i0:= i
i+= 32
if (i > n) m&= (~u32~~0)>>((-n)%32)
if (m != 0) { # Need to add entries
do {
settab{load{x, i0 + cast_i{u64,ctz{m}}}}
settab{load{x, i0 + ctzi{m}}}
m&= m-1 # Clear last bit
} while (m != 0)
if (u == 0) return{u} # Table is all one value
@ -144,7 +146,7 @@ fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi
# Initialize
rev:u1 = nx < nw/4 # Reverse lookup
z:= VI**(-cast_i{i8,rev})
z:= VI**(-promote{i8,rev})
@unroll (vtab over 8) vtab = z
u:u8 = 0 # Sum of table, either 0 or 256
if (rev) u = fill_bittab(x0, nx, tab, u, 0)
@ -174,30 +176,44 @@ fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = {
store{btab, x, 0}
v
}
def settab{T, x} = T~~promote{ty_s{T}, settab{x}}
def settab1{x} = { store{btab, x, 0}; ++u }
# Do first 64 values with a scalar loop
x:= *u8~~x0
i:u64 = 64; if (n<=64) i=n
i:u64 = 32; if (n<=48) i=n
rw:u64 = 0
@for (x over j to i) rw|= u64~~cast_i{i64,settab{x}} & ((u64~~1)<<j)
@for (x over j to i) rw|= settab{u64, x} & ((u64~~1)<<j)
store{*u64~~r0, 0, rw}
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
xv:= *VI~~x0; r:= *u32~~r0
while (i < n) {
i0:= i; iw:= i/32
m:= bitsel{load{xv, iw}} # Mask of possibly-new values
v:= load{xv, iw}
m:= bitsel{v} # Mask of possibly-new values
i+= 32
if (i > n) m&= (~u32~~0)>>((-n)%32)
if (m == 0) { # No new values
store{r, iw, m}
} else { # Add values to the table and filter m
rv:= m
do {
new:= settab{load{x, i0 + cast_i{u64,ctz{m}}}}
m1:= m-1; m&= m1 # Clear last bit
rv&= m1 | u32~~cast_i{i32,new} # Clear if not new
} while (m != 0)
xi:= load{x, i0 + ctzi{m}}
settab1{xi}
if ((m&(m-1)) != 0) { # More bits than one
# Filter out values equal to the previous, or first new
def pind = (iota{32}&15) - 1
prev:= make{VI, each{bind{max,0}, pind}}
e:= ~homMask{v == VI**TI~~xi}
e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}}
rv&= e | -m # Don't remove first bit
m&= e
while (m != 0) {
new:= settab{u32, load{x, i0 + ctzi{m}}}
m1:= m-1; m&= m1 # Clear low bit
rv&= m1 | new # Clear if not new
}
}
store{r, iw, rv}
if (u == 0) { # All bytes seen
@for (r over _ from iw+1 to cdiv{n,32}) r = 0