diff --git a/build/src/build.bqn b/build/src/build.bqn index 4e5c0d29..40233bcd 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -609,7 +609,7 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", - "xa."‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", + "xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", diff --git a/src/builtins/search.c b/src/builtins/search.c index 94ff754a..0b2d8345 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -23,6 +23,11 @@ #include "../utils/hash.h" #include "../utils/talloc.h" +#if SINGELI_SIMD + #define SINGELI_FILE search + #include "../utils/includeSingeli.h" +#endif + #define C2i(F, W, X) C2(F, m_i32(W), X) extern B eq_c2(B,B,B); extern B ne_c2(B,B,B); @@ -131,11 +136,6 @@ static B reduceI32Width(B r, usz count) { return count<=I8_MAX? taga(cpyI8Arr(r)) : count<=I16_MAX? taga(cpyI16Arr(r)) : r; } -#if SINGELI_SIMD - #define SINGELI_FILE search - #include "../utils/includeSingeli.h" -#endif - static NOINLINE usz indexOfOne(B l, B e) { void* lp = tyany_ptr(l); usz wia = IA(l); @@ -201,7 +201,10 @@ B indexOf_c2(B t, B w, B x) { } else { u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); - if (wia == 0) { B r=taga(arr_shCopy(allZeroes(xia), x)); decG(w); decG(x); return r; } + if (wia==0 || xia==0) { + B r=taga(arr_shCopy(allZeroes(xia), x)); + decG(w); decG(x); return r; + } if (elNum(we) && elNum(xe)) { tyEls: if (we==el_bit) { @@ -223,6 +226,18 @@ B indexOf_c2(B t, B w, B x) { } if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { + #if SINGELI + if (wia>256 && we==el_i8 && xe==el_i8) { + TALLOC(u8, tab, 256*(1+sizeof(usz))); usz* ind = (usz*)(tab+256); + void* fp = tyany_ptr(x); + simd_index_tab_u8(tyany_ptr(w), wia, fp, xia, tab, ind); + decG(w); + i32* rp; B r = m_i32arrc(&rp, x); + for (usz i=0; i16) { + if (xia<=(xe==el_i8?1:xe==el_i16?4:16) && wia>16) { SGetU(x); r = WEQ(GetU(x,0)); for (usz i=1; i20 && we<=el_i16 && xe<=el_i16) { - B r; + #if SINGELI + if (we==el_i8 && xe==el_i8) { + TALLOC(u8, tab, 256); + u64* rp; r = m_bitarrc(&rp, w); + simd_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab); + TFREE(tab); decG(w); + goto dec_x; + } + #endif TABLE(x, w, i8, 0, 1) return taga(cpyBitArr(r)); } @@ -321,7 +347,7 @@ B count_c2(B t, B w, B x) { x = t.n; } - if (!isArr(x) || IA(x)<=1) return C2(indexOf, w, x); + if (!isArr(x) || IA(x)<=1 || IA(w)==0) return C2(indexOf, w, x); u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); i32* rp; B r = m_i32arrc(&rp, x); diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 45bac03a..d8b92a56 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -213,6 +213,9 @@ static NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; itupsel{0,c}, v1}}} @@ -30,7 +34,7 @@ def search{E, x, n:u64, OP} = { def VT = [bulk]E def end = makeBranch{ tup{u64, ty_u{VT}}, - {i,c} => return{i*bulk + promote{u64, ctz{homMask{c}}}} + {i,c} => return{i*bulk + ctzi{homMask{c}}} } muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => { @@ -74,4 +78,217 @@ export{'simd_search_u16', searchOne{u64, u16}} export{'simd_search_u32', searchOne{u64, u32}} export{'simd_search_f64', searchOne{f64, f64}} export{'simd_search_normalizable', searchNormalizable{}} -export{'simd_copy_ordered', copyOrdered{}} \ No newline at end of file +export{'simd_copy_ordered', copyOrdered{}} + + +# In-register bit table +def arch_vec{T} = [arch_defvw/width{T}]T +def TI = i8 # Table values +def VI = arch_vec{TI} +def simd_bittab = hasarch{'SSSE3'} + +def bittab_init{tab, z} = { + @for (t in *TI~~tab over 256) t = z +} +def bittab_init{tab, z & simd_bittab} = { + init:= VI**z + @unroll (t in *VI~~tab over 256/vcount{VI}) t = init +} + +def bittab_selector{loadtab} = { + def nv = vcount{VI} + {t0, t1}:= loadtab{} + low:= VI**7 + hi4:= VI**(-(1<<4)) + b := VI~~make{[nv]u8, 1 << (iota{nv} & 7)} + def selector{x} = { + top := hi4 + VI~~((arch_vec{u32}~~(x&~low))>>3) + byte:= sel{[16]i8, t0, hi4^top} | sel{[16]i8, t1, top} + mask:= sel{[16]i8, b, x & low} + homMask{(mask & byte) == mask} + } + def reload{} = { tup{t0,t1} = loadtab{} } + tup{selector, reload} +} + +def readbytes{vtab}{} = { + def k = vcount{VI}; def l = 128/vcount{VI} + def side{i} = { + def U = arch_vec{ty_u{k}} + def m = @collect (vtab over _ from i to i+l) homMask{vtab} + VI~~make{U, if (vcount{U}>l) merge{m,m} else m} + } + each{side, l*iota{2}} +} + +# Look up bits from table +def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void} = { + x:= *u8~~x0 + t:= *TI~~tab + r:= *u64~~r0 + rem:= n + while (rem > 0) { + k:= rem; if (k>64) k=64 + rv:u64 = 0 + @for (x over j to k) rv|= u64~~promote{i64,load{t,x}} & ((u64~~1)<64) k=64 + rw:u64 = 0 + @for (x over j to k) { + new:= settab{u64, x, i*64+j} # Index usually unused + if (rbit) rw|= new & ((u64~~1)< n) m&= (~uk~~0)>>((-n)%k) + # Any new values? + if (m == 0) { + storebit{iw, m} + } else { + # Add values to the table and filter m + if (rbit) rv = m + im:= i0 + ctzi{m} + xi:= load{x, im} + settab1{xi, im} + if ((m&(m-1)) != 0) { # More bits than one + # Filter out values equal to the previous, or first new + def pind = (iota{k}&15) - 1 + prev:= make{VI, each{bind{max,0}, pind}} + e:= ~homMask{v == VI**TI~~xi} + e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} + if (rbit) rv&= e | -m # Don't remove first bit + m&= e + while (m != 0) { + im:= i0 + ctzi{m} + new:= settab{uk, load{x, im}, im} + m1:= m-1; m&= m1 # Clear low bit + if (rbit) rv&= m1 | new # Clear if not new + } + } + storebit{iw, rv} + if (u == 0) { # All bytes seen + if (rbit) @for (r in *uk~~r0 over _ from iw+1 to cdiv{n,k}) r = 0 + goto{done} + } + reload_tab{} + } + } + setlabel{done} + } + u +} + +fn simd_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { + bittab_init{tab, -1} + u:u8 = 0 + do_bittab{x0, n, tab, u, 0, 'mask', r0} +} + +fn simd_deduplicate_u8(x0:*void, n:u64, r0:*void, tab:*void) : u64 = { + assert{n != 0} + bittab_init{tab, -1} + u:u8 = 0 + do_bittab{x0, n, tab, u, 0, 'unique', r0} + 1 + promote{u64, u-1} # 0 to 256 +} + +fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { + do_bittab{x0, n, tab, u, t, 'none', 0} +} + +fn simd_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { + assert{nw > 0} + + rev:u1 = nx < nw/4 # Reverse lookup + bittab_init{tab, -promote{i8,rev}} + u:u8 = 0 # Sum of table, either 0 or 256 + if (rev) u = fill_bittab(x0, nx, tab, u, 0) + + u = fill_bittab(w0, nw, tab, u, -1) + + if (u == 0) { # All found! + @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} + } else { + bittab_lookup{x0, nx, r0, tab} + } +} + +fn simd_index_tab_u8{I}(w0:*void, nw:u64, x0:*void, nx:u64, tab:*void, i0:*void) : void = { + rev:u1 = nx < nw/4 + bittab_init{tab, -promote{i8,rev}} + ind:= *I~~i0 + @for (ind over 256) ind = cast_i{I, nw} + u:u8 = 0 + if (rev) u = fill_bittab(x0, nx, tab, u, 0) + do_bittab{w0, nw, tab, u, -1, 'index', ind} +} + +export{'simd_mark_firsts_u8', simd_mark_firsts_u8} +export{'simd_deduplicate_u8', simd_deduplicate_u8} +export{'simd_member_u8', simd_member_u8} +export{'simd_index_tab_u8', simd_index_tab_u8{usz}}