diff --git a/build/src/build.bqn b/build/src/build.bqn index 4e5c0d29..40233bcd 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -609,7 +609,7 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", - "xa."‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", + "xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", diff --git a/src/builtins/search.c b/src/builtins/search.c index 619691da..2e903ece 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -299,11 +299,11 @@ B memberOf_c2(B t, B w, B x) { if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { B r; - #if SINGELI_AVX2 + #if SINGELI if (we==el_i8 && xe==el_i8) { TALLOC(u8, tab, 256); u64* rp; r = m_bitarrc(&rp, w); - avx2_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab); + simd_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab); TFREE(tab); return r; } diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index ba09e73d..6394b67e 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -213,8 +213,8 @@ static NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i 0) { + k:= rem; if (k>64) k=64 + rv:u64 = 0 + @for (x over j to k) rv|= u64~~promote{i64,load{t,x}} & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) - # Any new values? - if (m == 0) { - storebit{iw, m} - } else { - # Add values to the table and filter m - if (rbit) rv = m - xi:= load{x, i0 + ctzi{m}} - settab1{xi} - if ((m&(m-1)) != 0) { # More bits than one - # Filter out values equal to the previous, or first new - def pind = (iota{32}&15) - 1 - prev:= make{VI, each{bind{max,0}, pind}} - e:= ~homMask{v == VI**TI~~xi} - e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} - if (rbit) rv&= e | -m # Don't remove first bit - m&= e - while (m != 0) { - new:= settab{u32, load{x, i0 + ctzi{m}}} - m1:= m-1; m&= m1 # Clear low bit - if (rbit) rv&= m1 | new # Clear if not new - } + if (not hasarch{'AVX2'}) { + rem:= n + @for (i to cdiv{n,64}) { + k:= rem; if (k>64) k=64 + rw:u64 = 0 + @for (x over j to k) { + new:= settab{u64, x} + if (rbit) rw|= new & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) + # Any new values? + if (m == 0) { + storebit{iw, m} + } else { + # Add values to the table and filter m + if (rbit) rv = m + xi:= load{x, i0 + ctzi{m}} + settab1{xi} + if ((m&(m-1)) != 0) { # More bits than one + # Filter out values equal to the previous, or first new + def pind = (iota{32}&15) - 1 + prev:= make{VI, each{bind{max,0}, pind}} + e:= ~homMask{v == VI**TI~~xi} + e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} + if (rbit) rv&= e | -m # Don't remove first bit + m&= e + while (m != 0) { + new:= settab{u32, load{x, i0 + ctzi{m}}} + m1:= m-1; m&= m1 # Clear low bit + if (rbit) rv&= m1 | new # Clear if not new + } + } + storebit{iw, rv} + if (u == 0) { # All bytes seen + if (rbit) @for (r in *u32~~r0 over _ from iw+1 to cdiv{n,32}) r = 0 + goto{done} + } + reload_tab{} + } + } + setlabel{done} } - setlabel{done} u } -fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { - init:= VI**(-1) - @unroll (t in *VI~~tab over 8) t = init +fn simd_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { + bittab_init{tab, -1} u:u8 = 0 do_bittab{x0, n, tab, u, 0, 'mask', r0} } -fn avx2_deduplicate_u8(x0:*void, n:u64, r0:*void, tab:*void) : u64 = { +fn simd_deduplicate_u8(x0:*void, n:u64, r0:*void, tab:*void) : u64 = { assert{n != 0} - init:= VI**(-1) - @unroll (t in *VI~~tab over 8) t = init + bittab_init{tab, -1} u:u8 = 0 do_bittab{x0, n, tab, u, 0, 'unique', r0} 1 + promote{u64, u-1} # 0 to 256 @@ -209,30 +247,26 @@ fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { do_bittab{x0, n, tab, u, t, 'none', 0} } -fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { +fn simd_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { assert{nw > 0} - vtab:= *VI~~tab # Initialize rev:u1 = nx < nw/4 # Reverse lookup - z:= VI**(-promote{i8,rev}) - @unroll (vtab over 8) vtab = z + bittab_init{tab, -promote{i8,rev}} u:u8 = 0 # Sum of table, either 0 or 256 if (rev) u = fill_bittab(x0, nx, tab, u, 0) # Fill table u = fill_bittab(w0, nw, tab, u, -1) - if (u == 0) { # All found! - @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} - return{} - } # Read result - def {bitsel, _} = bittab_selector{readbytes{vtab}} - @for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x} + if (u == 0) { # All found! + @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} + } else { + bittab_lookup{x0, nx, r0, tab} + } } -export{'avx2_mark_firsts_u8', avx2_mark_firsts_u8} -export{'avx2_deduplicate_u8', avx2_deduplicate_u8} -export{'avx2_member_u8', avx2_member_u8} -} # hasarch{'AVX2'} +export{'simd_mark_firsts_u8', simd_mark_firsts_u8} +export{'simd_deduplicate_u8', simd_deduplicate_u8} +export{'simd_member_u8', simd_member_u8}