AVX2 bit table Member-of
This commit is contained in:
parent
03a28e4e34
commit
0232f91582
@ -23,6 +23,11 @@
|
|||||||
#include "../utils/hash.h"
|
#include "../utils/hash.h"
|
||||||
#include "../utils/talloc.h"
|
#include "../utils/talloc.h"
|
||||||
|
|
||||||
|
#if SINGELI_SIMD
|
||||||
|
#define SINGELI_FILE search
|
||||||
|
#include "../utils/includeSingeli.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
#define C2i(F, W, X) C2(F, m_i32(W), X)
|
#define C2i(F, W, X) C2(F, m_i32(W), X)
|
||||||
extern B eq_c2(B,B,B);
|
extern B eq_c2(B,B,B);
|
||||||
extern B ne_c2(B,B,B);
|
extern B ne_c2(B,B,B);
|
||||||
@ -131,11 +136,6 @@ static B reduceI32Width(B r, usz count) {
|
|||||||
return count<=I8_MAX? taga(cpyI8Arr(r)) : count<=I16_MAX? taga(cpyI16Arr(r)) : r;
|
return count<=I8_MAX? taga(cpyI8Arr(r)) : count<=I16_MAX? taga(cpyI16Arr(r)) : r;
|
||||||
}
|
}
|
||||||
|
|
||||||
#if SINGELI_SIMD
|
|
||||||
#define SINGELI_FILE search
|
|
||||||
#include "../utils/includeSingeli.h"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static NOINLINE usz indexOfOne(B l, B e) {
|
static NOINLINE usz indexOfOne(B l, B e) {
|
||||||
void* lp = tyany_ptr(l);
|
void* lp = tyany_ptr(l);
|
||||||
usz wia = IA(l);
|
usz wia = IA(l);
|
||||||
@ -293,6 +293,15 @@ B memberOf_c2(B t, B w, B x) {
|
|||||||
|
|
||||||
if (xia+wia>20 && we<=el_i16 && xe<=el_i16) {
|
if (xia+wia>20 && we<=el_i16 && xe<=el_i16) {
|
||||||
B r;
|
B r;
|
||||||
|
#if SINGELI_AVX2
|
||||||
|
if (we==el_i8 && xe==el_i8) {
|
||||||
|
TALLOC(u8, tab, 256);
|
||||||
|
u64* rp; r = m_bitarrc(&rp, w);
|
||||||
|
avx2_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab);
|
||||||
|
TFREE(tab);
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
TABLE(x, w, i8, 0, 1)
|
TABLE(x, w, i8, 0, 1)
|
||||||
return taga(cpyBitArr(r));
|
return taga(cpyBitArr(r));
|
||||||
}
|
}
|
||||||
|
|||||||
@ -74,4 +74,77 @@ export{'simd_search_u16', searchOne{u64, u16}}
|
|||||||
export{'simd_search_u32', searchOne{u64, u32}}
|
export{'simd_search_u32', searchOne{u64, u32}}
|
||||||
export{'simd_search_f64', searchOne{f64, f64}}
|
export{'simd_search_f64', searchOne{f64, f64}}
|
||||||
export{'simd_search_normalizable', searchNormalizable{}}
|
export{'simd_search_normalizable', searchNormalizable{}}
|
||||||
export{'simd_copy_ordered', copyOrdered{}}
|
export{'simd_copy_ordered', copyOrdered{}}
|
||||||
|
|
||||||
|
|
||||||
|
# In-register bit table
|
||||||
|
if (hasarch{'AVX2'}) {
|
||||||
|
def TI = i8
|
||||||
|
def VI = [32]TI
|
||||||
|
|
||||||
|
def bittab_selector{loadtab} = {
|
||||||
|
{t0, t1}:= loadtab{}
|
||||||
|
low:= VI**7
|
||||||
|
hi4:= VI**(-(1<<4))
|
||||||
|
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
|
||||||
|
def selector{x} = {
|
||||||
|
top := hi4 + VI~~(([8]u32~~(x&~low))>>3)
|
||||||
|
byte:= sel{[16]i8, t0, hi4^top} | sel{[16]i8, t1, top}
|
||||||
|
mask:= sel{[16]i8, b, x & low}
|
||||||
|
homMask{(mask & byte) == mask}
|
||||||
|
}
|
||||||
|
def reload{} = { tup{t0,t1} = loadtab{} }
|
||||||
|
tup{selector, reload}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = {
|
||||||
|
assert{nw > 0}
|
||||||
|
vtab:= *VI~~tab; btab:= *i8~~tab
|
||||||
|
z:= VI**0
|
||||||
|
@unroll (vtab over 8) vtab = z
|
||||||
|
def readbytes{} = {
|
||||||
|
def side{i} = {
|
||||||
|
def m = @collect (vtab over _ from i to i+4) homMask{vtab}
|
||||||
|
VI~~make{[8]u32, merge{m,m}}
|
||||||
|
}
|
||||||
|
each{side, 4*iota{2}}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fill table
|
||||||
|
u:u8 = 0 # Unseen unique bytes (goes to 255 on first step)
|
||||||
|
w:= *u8~~w0
|
||||||
|
iw:u64 = 32; if (nw<=48) iw=nw
|
||||||
|
@for (w over iw) {
|
||||||
|
u-= 1 + u8~~load{btab, w} # Subtract 1 if new
|
||||||
|
store{btab, w, -1}
|
||||||
|
}
|
||||||
|
def {bitsel, reload_tab} = bittab_selector{readbytes}
|
||||||
|
wv:= *VI~~w0
|
||||||
|
while (iw < nw) {
|
||||||
|
ii:= iw+32
|
||||||
|
m:= bitsel{load{wv, iw/32}}
|
||||||
|
if (ii > nw) { ii=nw; m|=(~u32~~0)<<(nw%32) }
|
||||||
|
m = ~m
|
||||||
|
if (m != 0) { # Need to add entries
|
||||||
|
do {
|
||||||
|
wj:= load{w, iw+emit{u64, 'CTZ', m}}
|
||||||
|
u-= 1 + u8~~load{btab, wj} # Subtract 1 if new
|
||||||
|
store{btab, wj, -1}
|
||||||
|
m&= m-1 # Clear last bit
|
||||||
|
} while (m != 0)
|
||||||
|
|
||||||
|
if (u == 0) { # All found!
|
||||||
|
@for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64}
|
||||||
|
return{}
|
||||||
|
}
|
||||||
|
reload_tab{}
|
||||||
|
}
|
||||||
|
iw = ii
|
||||||
|
}
|
||||||
|
|
||||||
|
# Read result
|
||||||
|
@for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x}
|
||||||
|
}
|
||||||
|
|
||||||
|
export{'avx2_member_u8', avx2_member_u8}
|
||||||
|
} # hasarch{'AVX2'}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user