From 0232f915828c9545e7676728ba7fb3d8980fbaa1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 24 Apr 2023 12:37:13 -0400 Subject: [PATCH] AVX2 bit table Member-of --- src/builtins/search.c | 19 ++++++--- src/singeli/src/search.singeli | 75 +++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 94ff754a..0775eb9c 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -23,6 +23,11 @@ #include "../utils/hash.h" #include "../utils/talloc.h" +#if SINGELI_SIMD + #define SINGELI_FILE search + #include "../utils/includeSingeli.h" +#endif + #define C2i(F, W, X) C2(F, m_i32(W), X) extern B eq_c2(B,B,B); extern B ne_c2(B,B,B); @@ -131,11 +136,6 @@ static B reduceI32Width(B r, usz count) { return count<=I8_MAX? taga(cpyI8Arr(r)) : count<=I16_MAX? taga(cpyI16Arr(r)) : r; } -#if SINGELI_SIMD - #define SINGELI_FILE search - #include "../utils/includeSingeli.h" -#endif - static NOINLINE usz indexOfOne(B l, B e) { void* lp = tyany_ptr(l); usz wia = IA(l); @@ -293,6 +293,15 @@ B memberOf_c2(B t, B w, B x) { if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { B r; + #if SINGELI_AVX2 + if (we==el_i8 && xe==el_i8) { + TALLOC(u8, tab, 256); + u64* rp; r = m_bitarrc(&rp, w); + avx2_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab); + TFREE(tab); + return r; + } + #endif TABLE(x, w, i8, 0, 1) return taga(cpyBitArr(r)); } diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 03c9c683..48063fdc 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -74,4 +74,77 @@ export{'simd_search_u16', searchOne{u64, u16}} export{'simd_search_u32', searchOne{u64, u32}} export{'simd_search_f64', searchOne{f64, f64}} export{'simd_search_normalizable', searchNormalizable{}} -export{'simd_copy_ordered', copyOrdered{}} \ No newline at end of file +export{'simd_copy_ordered', copyOrdered{}} + + +# In-register bit table +if (hasarch{'AVX2'}) { +def TI = i8 +def VI = [32]TI + +def bittab_selector{loadtab} = { + {t0, t1}:= loadtab{} + low:= VI**7 + hi4:= VI**(-(1<<4)) + b := VI~~make{[32]u8, 1 << (iota{32} & 7)} + def selector{x} = { + top := hi4 + VI~~(([8]u32~~(x&~low))>>3) + byte:= sel{[16]i8, t0, hi4^top} | sel{[16]i8, t1, top} + mask:= sel{[16]i8, b, x & low} + homMask{(mask & byte) == mask} + } + def reload{} = { tup{t0,t1} = loadtab{} } + tup{selector, reload} +} + +fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { + assert{nw > 0} + vtab:= *VI~~tab; btab:= *i8~~tab + z:= VI**0 + @unroll (vtab over 8) vtab = z + def readbytes{} = { + def side{i} = { + def m = @collect (vtab over _ from i to i+4) homMask{vtab} + VI~~make{[8]u32, merge{m,m}} + } + each{side, 4*iota{2}} + } + + # Fill table + u:u8 = 0 # Unseen unique bytes (goes to 255 on first step) + w:= *u8~~w0 + iw:u64 = 32; if (nw<=48) iw=nw + @for (w over iw) { + u-= 1 + u8~~load{btab, w} # Subtract 1 if new + store{btab, w, -1} + } + def {bitsel, reload_tab} = bittab_selector{readbytes} + wv:= *VI~~w0 + while (iw < nw) { + ii:= iw+32 + m:= bitsel{load{wv, iw/32}} + if (ii > nw) { ii=nw; m|=(~u32~~0)<<(nw%32) } + m = ~m + if (m != 0) { # Need to add entries + do { + wj:= load{w, iw+emit{u64, 'CTZ', m}} + u-= 1 + u8~~load{btab, wj} # Subtract 1 if new + store{btab, wj, -1} + m&= m-1 # Clear last bit + } while (m != 0) + + if (u == 0) { # All found! + @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} + return{} + } + reload_tab{} + } + iw = ii + } + + # Read result + @for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x} +} + +export{'avx2_member_u8', avx2_member_u8} +} # hasarch{'AVX2'}