diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index c38159b7..1077c150 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -111,35 +111,89 @@ def readbytes{vtab}{} = { # Stop early if the sum u reaches 0, indicating all bytes in the table # are equal: by the time it's checked at least one has been set to t, # so they're all t -fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { +# If r0 is a pointer, set it to the unique mask of x0 +def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, r0} = { btab:= *i8~~tab - def settab{x} = { - u+= u8~~(t - load{btab, x}) # u tracks the total of btab + def settab_sub{x, v} = { + u+= u8~~(t - v) # u tracks the total of btab store{btab, x, t} + v } + def settab1{x} = settab_sub{x, -1 - t} # Known new + def settab{x} = settab_sub{x, load{btab, x}} # General case + def settab{T, x} = T~~promote{ty_s{T}, settab{x}} + + def rbit = ~match{r0, 0} + def storebit{i, v:T} = if (rbit) store{*T~~r0, i, v} + + # Do first few values with a scalar loop + # Avoids the cost of ever loading the table into vectors for n<=48 x:= *u8~~x0 i:u64 = 32; if (n<=48) i=n - @for (x over i) settab{x} - if (u == 0) return{u} # Won't ever trigger (m != 0)! + {rw,rv} := undef{tup{u64,u32}} # Bit results, used if rbit + if (rbit) rw = 0 + @for (x over j to i) { + new:= settab{u64, x} + if (rbit) rw|= new & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) - if (m != 0) { # Need to add entries - do { - settab{load{x, i0 + ctzi{m}}} - m&= m-1 # Clear last bit - } while (m != 0) - if (u == 0) return{u} # Table is all one value + # Any new values? + if (m == 0) { + storebit{iw, m} + } else { + # Add values to the table and filter m + if (rbit) rv = m + xi:= load{x, i0 + ctzi{m}} + settab1{xi} + if ((m&(m-1)) != 0) { # More bits than one + # Filter out values equal to the previous, or first new + def pind = (iota{32}&15) - 1 + prev:= make{VI, each{bind{max,0}, pind}} + e:= ~homMask{v == VI**TI~~xi} + e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} + if (rbit) rv&= e | -m # Don't remove first bit + m&= e + while (m != 0) { + new:= settab{u32, load{x, i0 + ctzi{m}}} + m1:= m-1; m&= m1 # Clear low bit + if (rbit) rv&= m1 | new # Clear if not new + } + } + storebit{iw, rv} + if (u == 0) { # All bytes seen + if (rbit) @for (r in *u32~~r0 over _ from iw+1 to cdiv{n,32}) r = 0 + goto{done} + } reload_tab{} } } + setlabel{done} u } +fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { + init:= VI**(-1) + @unroll (t in *VI~~tab over 8) t = init + u:u8 = 0 + do_bittab{x0, n, tab, u, 0, r0} +} + +fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { + do_bittab{x0, n, tab, u, t, 0} +} + fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { assert{nw > 0} vtab:= *VI~~tab @@ -163,67 +217,6 @@ fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi @for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x} } -fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { - vtab:= *VI~~tab - init:= VI**(-1) - @unroll (vtab over 8) vtab = init - - u:u8 = 0 # Sum of table, either 0 or 256 - btab:= *i8~~tab - def settab{x} = { - v:= load{btab, x} - u-= u8~~v # u tracks the total of btab - store{btab, x, 0} - v - } - def settab{T, x} = T~~promote{ty_s{T}, settab{x}} - def settab1{x} = { store{btab, x, 0}; ++u } - - # Do first 64 values with a scalar loop - x:= *u8~~x0 - i:u64 = 32; if (n<=48) i=n - rw:u64 = 0 - @for (x over j to i) rw|= settab{u64, x} & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) - if (m == 0) { # No new values - store{r, iw, m} - } else { # Add values to the table and filter m - rv:= m - xi:= load{x, i0 + ctzi{m}} - settab1{xi} - if ((m&(m-1)) != 0) { # More bits than one - # Filter out values equal to the previous, or first new - def pind = (iota{32}&15) - 1 - prev:= make{VI, each{bind{max,0}, pind}} - e:= ~homMask{v == VI**TI~~xi} - e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} - rv&= e | -m # Don't remove first bit - m&= e - while (m != 0) { - new:= settab{u32, load{x, i0 + ctzi{m}}} - m1:= m-1; m&= m1 # Clear low bit - rv&= m1 | new # Clear if not new - } - } - store{r, iw, rv} - if (u == 0) { # All bytes seen - @for (r over _ from iw+1 to cdiv{n,32}) r = 0 - return{} - } - reload_tab{} - } - } -} - export{'avx2_mark_firsts_u8', avx2_mark_firsts_u8} export{'avx2_member_u8', avx2_member_u8} } # hasarch{'AVX2'}