AVX2 bit table Member-of
This commit is contained in:
parent
03a28e4e34
commit
0232f91582
@ -23,6 +23,11 @@
|
||||
#include "../utils/hash.h"
|
||||
#include "../utils/talloc.h"
|
||||
|
||||
#if SINGELI_SIMD
|
||||
#define SINGELI_FILE search
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
#define C2i(F, W, X) C2(F, m_i32(W), X)
|
||||
extern B eq_c2(B,B,B);
|
||||
extern B ne_c2(B,B,B);
|
||||
@ -131,11 +136,6 @@ static B reduceI32Width(B r, usz count) {
|
||||
return count<=I8_MAX? taga(cpyI8Arr(r)) : count<=I16_MAX? taga(cpyI16Arr(r)) : r;
|
||||
}
|
||||
|
||||
#if SINGELI_SIMD
|
||||
#define SINGELI_FILE search
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
static NOINLINE usz indexOfOne(B l, B e) {
|
||||
void* lp = tyany_ptr(l);
|
||||
usz wia = IA(l);
|
||||
@ -293,6 +293,15 @@ B memberOf_c2(B t, B w, B x) {
|
||||
|
||||
if (xia+wia>20 && we<=el_i16 && xe<=el_i16) {
|
||||
B r;
|
||||
#if SINGELI_AVX2
|
||||
if (we==el_i8 && xe==el_i8) {
|
||||
TALLOC(u8, tab, 256);
|
||||
u64* rp; r = m_bitarrc(&rp, w);
|
||||
avx2_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab);
|
||||
TFREE(tab);
|
||||
return r;
|
||||
}
|
||||
#endif
|
||||
TABLE(x, w, i8, 0, 1)
|
||||
return taga(cpyBitArr(r));
|
||||
}
|
||||
|
||||
@ -74,4 +74,77 @@ export{'simd_search_u16', searchOne{u64, u16}}
|
||||
export{'simd_search_u32', searchOne{u64, u32}}
|
||||
export{'simd_search_f64', searchOne{f64, f64}}
|
||||
export{'simd_search_normalizable', searchNormalizable{}}
|
||||
export{'simd_copy_ordered', copyOrdered{}}
|
||||
export{'simd_copy_ordered', copyOrdered{}}
|
||||
|
||||
|
||||
# In-register bit table
|
||||
if (hasarch{'AVX2'}) {
|
||||
def TI = i8
|
||||
def VI = [32]TI
|
||||
|
||||
def bittab_selector{loadtab} = {
|
||||
{t0, t1}:= loadtab{}
|
||||
low:= VI**7
|
||||
hi4:= VI**(-(1<<4))
|
||||
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
|
||||
def selector{x} = {
|
||||
top := hi4 + VI~~(([8]u32~~(x&~low))>>3)
|
||||
byte:= sel{[16]i8, t0, hi4^top} | sel{[16]i8, t1, top}
|
||||
mask:= sel{[16]i8, b, x & low}
|
||||
homMask{(mask & byte) == mask}
|
||||
}
|
||||
def reload{} = { tup{t0,t1} = loadtab{} }
|
||||
tup{selector, reload}
|
||||
}
|
||||
|
||||
fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = {
|
||||
assert{nw > 0}
|
||||
vtab:= *VI~~tab; btab:= *i8~~tab
|
||||
z:= VI**0
|
||||
@unroll (vtab over 8) vtab = z
|
||||
def readbytes{} = {
|
||||
def side{i} = {
|
||||
def m = @collect (vtab over _ from i to i+4) homMask{vtab}
|
||||
VI~~make{[8]u32, merge{m,m}}
|
||||
}
|
||||
each{side, 4*iota{2}}
|
||||
}
|
||||
|
||||
# Fill table
|
||||
u:u8 = 0 # Unseen unique bytes (goes to 255 on first step)
|
||||
w:= *u8~~w0
|
||||
iw:u64 = 32; if (nw<=48) iw=nw
|
||||
@for (w over iw) {
|
||||
u-= 1 + u8~~load{btab, w} # Subtract 1 if new
|
||||
store{btab, w, -1}
|
||||
}
|
||||
def {bitsel, reload_tab} = bittab_selector{readbytes}
|
||||
wv:= *VI~~w0
|
||||
while (iw < nw) {
|
||||
ii:= iw+32
|
||||
m:= bitsel{load{wv, iw/32}}
|
||||
if (ii > nw) { ii=nw; m|=(~u32~~0)<<(nw%32) }
|
||||
m = ~m
|
||||
if (m != 0) { # Need to add entries
|
||||
do {
|
||||
wj:= load{w, iw+emit{u64, 'CTZ', m}}
|
||||
u-= 1 + u8~~load{btab, wj} # Subtract 1 if new
|
||||
store{btab, wj, -1}
|
||||
m&= m-1 # Clear last bit
|
||||
} while (m != 0)
|
||||
|
||||
if (u == 0) { # All found!
|
||||
@for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64}
|
||||
return{}
|
||||
}
|
||||
reload_tab{}
|
||||
}
|
||||
iw = ii
|
||||
}
|
||||
|
||||
# Read result
|
||||
@for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x}
|
||||
}
|
||||
|
||||
export{'avx2_member_u8', avx2_member_u8}
|
||||
} # hasarch{'AVX2'}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user