Unify AVX2 bit-table code between Member-of and Mark Firsts
This commit is contained in:
parent
ba43dae908
commit
9658104f3e
@ -111,35 +111,89 @@ def readbytes{vtab}{} = {
|
|||||||
# Stop early if the sum u reaches 0, indicating all bytes in the table
|
# Stop early if the sum u reaches 0, indicating all bytes in the table
|
||||||
# are equal: by the time it's checked at least one has been set to t,
|
# are equal: by the time it's checked at least one has been set to t,
|
||||||
# so they're all t
|
# so they're all t
|
||||||
fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = {
|
# If r0 is a pointer, set it to the unique mask of x0
|
||||||
|
def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, r0} = {
|
||||||
btab:= *i8~~tab
|
btab:= *i8~~tab
|
||||||
def settab{x} = {
|
def settab_sub{x, v} = {
|
||||||
u+= u8~~(t - load{btab, x}) # u tracks the total of btab
|
u+= u8~~(t - v) # u tracks the total of btab
|
||||||
store{btab, x, t}
|
store{btab, x, t}
|
||||||
|
v
|
||||||
}
|
}
|
||||||
|
def settab1{x} = settab_sub{x, -1 - t} # Known new
|
||||||
|
def settab{x} = settab_sub{x, load{btab, x}} # General case
|
||||||
|
def settab{T, x} = T~~promote{ty_s{T}, settab{x}}
|
||||||
|
|
||||||
|
def rbit = ~match{r0, 0}
|
||||||
|
def storebit{i, v:T} = if (rbit) store{*T~~r0, i, v}
|
||||||
|
|
||||||
|
# Do first few values with a scalar loop
|
||||||
|
# Avoids the cost of ever loading the table into vectors for n<=48
|
||||||
x:= *u8~~x0
|
x:= *u8~~x0
|
||||||
i:u64 = 32; if (n<=48) i=n
|
i:u64 = 32; if (n<=48) i=n
|
||||||
@for (x over i) settab{x}
|
{rw,rv} := undef{tup{u64,u32}} # Bit results, used if rbit
|
||||||
if (u == 0) return{u} # Won't ever trigger (m != 0)!
|
if (rbit) rw = 0
|
||||||
|
@for (x over j to i) {
|
||||||
|
new:= settab{u64, x}
|
||||||
|
if (rbit) rw|= new & ((u64~~1)<<j)
|
||||||
|
}
|
||||||
|
storebit{0, rw}
|
||||||
|
if (not rbit and u == 0) return{u} # Won't ever trigger (m != 0)!
|
||||||
|
|
||||||
|
def done = makelabel{}
|
||||||
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
|
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
|
||||||
xv:= *VI~~x0
|
xv:= *VI~~x0
|
||||||
while (i < n) {
|
while (i < n) {
|
||||||
m:= u32~~promote{i32, t} ^ bitsel{load{xv, i/32}}
|
i0:= i; iw:= i/32
|
||||||
i0:= i
|
v:= load{xv, iw}
|
||||||
|
m:= bitsel{v} # Mask of possibly-new values
|
||||||
|
if (not match{t,0}) m^= u32~~promote{i32, t}
|
||||||
i+= 32
|
i+= 32
|
||||||
if (i > n) m&= (~u32~~0)>>((-n)%32)
|
if (i > n) m&= (~u32~~0)>>((-n)%32)
|
||||||
if (m != 0) { # Need to add entries
|
# Any new values?
|
||||||
do {
|
if (m == 0) {
|
||||||
settab{load{x, i0 + ctzi{m}}}
|
storebit{iw, m}
|
||||||
m&= m-1 # Clear last bit
|
} else {
|
||||||
} while (m != 0)
|
# Add values to the table and filter m
|
||||||
if (u == 0) return{u} # Table is all one value
|
if (rbit) rv = m
|
||||||
|
xi:= load{x, i0 + ctzi{m}}
|
||||||
|
settab1{xi}
|
||||||
|
if ((m&(m-1)) != 0) { # More bits than one
|
||||||
|
# Filter out values equal to the previous, or first new
|
||||||
|
def pind = (iota{32}&15) - 1
|
||||||
|
prev:= make{VI, each{bind{max,0}, pind}}
|
||||||
|
e:= ~homMask{v == VI**TI~~xi}
|
||||||
|
e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}}
|
||||||
|
if (rbit) rv&= e | -m # Don't remove first bit
|
||||||
|
m&= e
|
||||||
|
while (m != 0) {
|
||||||
|
new:= settab{u32, load{x, i0 + ctzi{m}}}
|
||||||
|
m1:= m-1; m&= m1 # Clear low bit
|
||||||
|
if (rbit) rv&= m1 | new # Clear if not new
|
||||||
|
}
|
||||||
|
}
|
||||||
|
storebit{iw, rv}
|
||||||
|
if (u == 0) { # All bytes seen
|
||||||
|
if (rbit) @for (r in *u32~~r0 over _ from iw+1 to cdiv{n,32}) r = 0
|
||||||
|
goto{done}
|
||||||
|
}
|
||||||
reload_tab{}
|
reload_tab{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
setlabel{done}
|
||||||
u
|
u
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = {
|
||||||
|
init:= VI**(-1)
|
||||||
|
@unroll (t in *VI~~tab over 8) t = init
|
||||||
|
u:u8 = 0
|
||||||
|
do_bittab{x0, n, tab, u, 0, r0}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = {
|
||||||
|
do_bittab{x0, n, tab, u, t, 0}
|
||||||
|
}
|
||||||
|
|
||||||
fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = {
|
fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = {
|
||||||
assert{nw > 0}
|
assert{nw > 0}
|
||||||
vtab:= *VI~~tab
|
vtab:= *VI~~tab
|
||||||
@ -163,67 +217,6 @@ fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi
|
|||||||
@for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x}
|
@for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = {
|
|
||||||
vtab:= *VI~~tab
|
|
||||||
init:= VI**(-1)
|
|
||||||
@unroll (vtab over 8) vtab = init
|
|
||||||
|
|
||||||
u:u8 = 0 # Sum of table, either 0 or 256
|
|
||||||
btab:= *i8~~tab
|
|
||||||
def settab{x} = {
|
|
||||||
v:= load{btab, x}
|
|
||||||
u-= u8~~v # u tracks the total of btab
|
|
||||||
store{btab, x, 0}
|
|
||||||
v
|
|
||||||
}
|
|
||||||
def settab{T, x} = T~~promote{ty_s{T}, settab{x}}
|
|
||||||
def settab1{x} = { store{btab, x, 0}; ++u }
|
|
||||||
|
|
||||||
# Do first 64 values with a scalar loop
|
|
||||||
x:= *u8~~x0
|
|
||||||
i:u64 = 32; if (n<=48) i=n
|
|
||||||
rw:u64 = 0
|
|
||||||
@for (x over j to i) rw|= settab{u64, x} & ((u64~~1)<<j)
|
|
||||||
store{*u64~~r0, 0, rw}
|
|
||||||
|
|
||||||
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
|
|
||||||
xv:= *VI~~x0; r:= *u32~~r0
|
|
||||||
while (i < n) {
|
|
||||||
i0:= i; iw:= i/32
|
|
||||||
v:= load{xv, iw}
|
|
||||||
m:= bitsel{v} # Mask of possibly-new values
|
|
||||||
i+= 32
|
|
||||||
if (i > n) m&= (~u32~~0)>>((-n)%32)
|
|
||||||
if (m == 0) { # No new values
|
|
||||||
store{r, iw, m}
|
|
||||||
} else { # Add values to the table and filter m
|
|
||||||
rv:= m
|
|
||||||
xi:= load{x, i0 + ctzi{m}}
|
|
||||||
settab1{xi}
|
|
||||||
if ((m&(m-1)) != 0) { # More bits than one
|
|
||||||
# Filter out values equal to the previous, or first new
|
|
||||||
def pind = (iota{32}&15) - 1
|
|
||||||
prev:= make{VI, each{bind{max,0}, pind}}
|
|
||||||
e:= ~homMask{v == VI**TI~~xi}
|
|
||||||
e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}}
|
|
||||||
rv&= e | -m # Don't remove first bit
|
|
||||||
m&= e
|
|
||||||
while (m != 0) {
|
|
||||||
new:= settab{u32, load{x, i0 + ctzi{m}}}
|
|
||||||
m1:= m-1; m&= m1 # Clear low bit
|
|
||||||
rv&= m1 | new # Clear if not new
|
|
||||||
}
|
|
||||||
}
|
|
||||||
store{r, iw, rv}
|
|
||||||
if (u == 0) { # All bytes seen
|
|
||||||
@for (r over _ from iw+1 to cdiv{n,32}) r = 0
|
|
||||||
return{}
|
|
||||||
}
|
|
||||||
reload_tab{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export{'avx2_mark_firsts_u8', avx2_mark_firsts_u8}
|
export{'avx2_mark_firsts_u8', avx2_mark_firsts_u8}
|
||||||
export{'avx2_member_u8', avx2_member_u8}
|
export{'avx2_member_u8', avx2_member_u8}
|
||||||
} # hasarch{'AVX2'}
|
} # hasarch{'AVX2'}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user