diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 7bc14226..c38159b7 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -10,6 +10,8 @@ if (hasarch{'AVX2'}) { } include './mask' +def ctzi{x} = promote{u64, ctz{x}} # Count trailing zeros, as index + def findFirst{C, M, F, ...v1} = { def exit = makelabel{} def args = undef{M{...each{{c}=>tupsel{0,c}, v1}}} @@ -30,7 +32,7 @@ def search{E, x, n:u64, OP} = { def VT = [bulk]E def end = makeBranch{ tup{u64, ty_u{VT}}, - {i,c} => return{i*bulk + promote{u64, ctz{homMask{c}}}} + {i,c} => return{i*bulk + ctzi{homMask{c}}} } muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => { @@ -122,13 +124,13 @@ fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}} xv:= *VI~~x0 while (i < n) { - m:= u32~~cast_i{i32, t} ^ bitsel{load{xv, i/32}} + m:= u32~~promote{i32, t} ^ bitsel{load{xv, i/32}} i0:= i i+= 32 if (i > n) m&= (~u32~~0)>>((-n)%32) if (m != 0) { # Need to add entries do { - settab{load{x, i0 + cast_i{u64,ctz{m}}}} + settab{load{x, i0 + ctzi{m}}} m&= m-1 # Clear last bit } while (m != 0) if (u == 0) return{u} # Table is all one value @@ -144,7 +146,7 @@ fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi # Initialize rev:u1 = nx < nw/4 # Reverse lookup - z:= VI**(-cast_i{i8,rev}) + z:= VI**(-promote{i8,rev}) @unroll (vtab over 8) vtab = z u:u8 = 0 # Sum of table, either 0 or 256 if (rev) u = fill_bittab(x0, nx, tab, u, 0) @@ -174,30 +176,44 @@ fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { store{btab, x, 0} v } + def settab{T, x} = T~~promote{ty_s{T}, settab{x}} + def settab1{x} = { store{btab, x, 0}; ++u } # Do first 64 values with a scalar loop x:= *u8~~x0 - i:u64 = 64; if (n<=64) i=n + i:u64 = 32; if (n<=48) i=n rw:u64 = 0 - @for (x over j to i) rw|= u64~~cast_i{i64,settab{x}} & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) if (m == 0) { # No new values store{r, iw, m} } else { # Add values to the table and filter m rv:= m - do { - new:= settab{load{x, i0 + cast_i{u64,ctz{m}}}} - m1:= m-1; m&= m1 # Clear last bit - rv&= m1 | u32~~cast_i{i32,new} # Clear if not new - } while (m != 0) + xi:= load{x, i0 + ctzi{m}} + settab1{xi} + if ((m&(m-1)) != 0) { # More bits than one + # Filter out values equal to the previous, or first new + def pind = (iota{32}&15) - 1 + prev:= make{VI, each{bind{max,0}, pind}} + e:= ~homMask{v == VI**TI~~xi} + e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} + rv&= e | -m # Don't remove first bit + m&= e + while (m != 0) { + new:= settab{u32, load{x, i0 + ctzi{m}}} + m1:= m-1; m&= m1 # Clear low bit + rv&= m1 | new # Clear if not new + } + } store{r, iw, rv} if (u == 0) { # All bytes seen @for (r over _ from iw+1 to cdiv{n,32}) r = 0