Singeli 1-byte table implementations for generic architecture

This commit is contained in:
Marshall Lochbaum 2023-04-27 21:52:23 -04:00
parent 6c81fb3149
commit 844b537593
4 changed files with 115 additions and 81 deletions

View File

@ -609,7 +609,7 @@ cachedBin‿linkerCache ← {
"xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp",
"xa.""src/builtins/squeeze.c""squeeze", "xa.""src/utils/mut.c""copy",
"xa.""src/utils/bits.c""bits", "xag""src/builtins/transpose.c""transpose",
"xa.""src/builtins/search.c""search", "xa.""src/builtins/fold.c""fold",
"xag""src/builtins/search.c""search", "xa.""src/builtins/fold.c""fold",
"2..""src/builtins/select.c""select", "2..""src/builtins/scan.c""scan",
"2..""src/builtins/slash.c""constrep", "2..""src/builtins/scan.c""neq",

View File

@ -299,11 +299,11 @@ B memberOf_c2(B t, B w, B x) {
if (xia+wia>20 && we<=el_i16 && xe<=el_i16) {
B r;
#if SINGELI_AVX2
#if SINGELI
if (we==el_i8 && xe==el_i8) {
TALLOC(u8, tab, 256);
u64* rp; r = m_bitarrc(&rp, w);
avx2_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab);
simd_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab);
TFREE(tab);
return r;
}

View File

@ -213,8 +213,8 @@ static NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i<l; i++) p[
/*AUXEXTEND*/val -= dif; memset32(val, 0, dif); , \
/*AUXMOVE*/u32 v = val[j]; val[j] = 0; val[k] = v;)
extern void (*const avx2_mark_firsts_u8)(void*,uint64_t,void*,void*);
extern u64 (*const avx2_deduplicate_u8)(void*,uint64_t,void*,void*);
extern void (*const simd_mark_firsts_u8)(void*,uint64_t,void*,void*);
extern u64 (*const simd_deduplicate_u8)(void*,uint64_t,void*,void*);
B memberOf_c1(B t, B x) {
if (isAtm(x) || RNK(x)==0) thrM("∊: Argument cannot have rank 0");
@ -255,10 +255,10 @@ B memberOf_c1(B t, B x) {
decG(x); TFREE(tab); \
return taga(cpyBitArr(r))
if (lw==3) { if (n<8) { BRUTE(8); } else {
#if SINGELI_AVX2
#if SINGELI
TALLOC(u8, tab, 256);
u64* rp; B r = m_bitarrv(&rp, n);
avx2_mark_firsts_u8(xv, n, rp, tab);
simd_mark_firsts_u8(xv, n, rp, tab);
TFREE(tab);
return r;
#else
@ -555,10 +555,10 @@ B find_c1(B t, B x) {
rp[0] = 2 ^ -x0;
return r;
}
#if SINGELI_AVX2
#if SINGELI
if (elWidth(xe)==1 && RNK(x)==1 && !FL_HAS(x, fl_asc|fl_dsc)) {
TALLOC(u8, tab, 513); u8* res = tab+256;
usz ria = avx2_deduplicate_u8(tyany_ptr(x), n, res, tab);
usz ria = simd_deduplicate_u8(tyany_ptr(x), n, res, tab);
B r; i8* rp = m_tyarrv(&r, 1, ria, el2t(xe));
memcpy(rp, res, ria);
TFREE(tab); decG(x);

View File

@ -80,10 +80,17 @@ export{'simd_copy_ordered', copyOrdered{}}
# In-register bit table
if (hasarch{'AVX2'}) {
def TI = i8
def TI = i8 # Table values
def VI = [32]TI
def bittab_init{tab, z} = {
@for (t in *TI~~tab over 256) t = z
}
def bittab_init{tab, z & hasarch{'AVX2'}} = {
init:= VI**z
@unroll (t in *VI~~tab over 256/vcount{VI}) t = init
}
def bittab_selector{loadtab} = {
{t0, t1}:= loadtab{}
low:= VI**7
@ -107,6 +114,25 @@ def readbytes{vtab}{} = {
each{side, 4*iota{2}}
}
# Look up bits from table
def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void} = {
x:= *u8~~x0
t:= *TI~~tab
r:= *u64~~r0
rem:= n
while (rem > 0) {
k:= rem; if (k>64) k=64
rv:u64 = 0
@for (x over j to k) rv|= u64~~promote{i64,load{t,x}} & ((u64~~1)<<j)
store{r, 0, rv}
x+=k; rem-=k; ++r
}
}
def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void & hasarch{'AVX2'}} = {
def {bitsel, _} = bittab_selector{readbytes{*VI~~tab}}
@for (x in *VI~~x0, r in *u32~~r0 over cdiv{n,32}) r = bitsel{x}
}
# Fill table with t (0 or -1) at all bytes in x0
# Stop early if the sum u reaches 0, indicating all bytes in the table
# are equal: by the time it's checked at least one has been set to t,
@ -132,74 +158,86 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = {
def settab{x} = settab_sub{x, load{btab, x}} # General case
def settab{T, x} = T~~promote{ty_s{T}, settab{x}}
# Do first few values with a scalar loop
# Avoids the cost of ever loading the table into vectors for n<=48
x:= *u8~~x0
i:u64 = 32; if (n<=48) i=n
{rw,rv} := undef{tup{u64,u32}} # Bit results, used if rbit
if (rbit) rw = 0
@for (x over j to i) {
new:= settab{u64, x}
if (rbit) rw|= new & ((u64~~1)<<j)
}
storebit{0, rw}
if (mode == 'none' and u == 0) return{u} # Won't ever trigger (m != 0)!
def done = makelabel{}
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
xv:= *VI~~x0
while (i < n) {
i0:= i; iw:= i/32
v:= load{xv, iw}
m:= bitsel{v} # Mask of possibly-new values
if (not match{t,0}) m^= u32~~promote{i32, t}
i+= 32
if (i > n) m&= (~u32~~0)>>((-n)%32)
# Any new values?
if (m == 0) {
storebit{iw, m}
} else {
# Add values to the table and filter m
if (rbit) rv = m
xi:= load{x, i0 + ctzi{m}}
settab1{xi}
if ((m&(m-1)) != 0) { # More bits than one
# Filter out values equal to the previous, or first new
def pind = (iota{32}&15) - 1
prev:= make{VI, each{bind{max,0}, pind}}
e:= ~homMask{v == VI**TI~~xi}
e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}}
if (rbit) rv&= e | -m # Don't remove first bit
m&= e
while (m != 0) {
new:= settab{u32, load{x, i0 + ctzi{m}}}
m1:= m-1; m&= m1 # Clear low bit
if (rbit) rv&= m1 | new # Clear if not new
}
if (not hasarch{'AVX2'}) {
rem:= n
@for (i to cdiv{n,64}) {
k:= rem; if (k>64) k=64
rw:u64 = 0
@for (x over j to k) {
new:= settab{u64, x}
if (rbit) rw|= new & ((u64~~1)<<j)
}
storebit{iw, rv}
if (u == 0) { # All bytes seen
if (rbit) @for (r in *u32~~r0 over _ from iw+1 to cdiv{n,32}) r = 0
goto{done}
}
reload_tab{}
storebit{i, rw}
x+=k; rem-=k
}
} else {
# Do first few values with a scalar loop
# Avoids the cost of ever loading the table into vectors for n<=48
i:u64 = 32; if (n<=48) i=n
{rw,rv} := undef{tup{u64,u32}} # Bit results, used if rbit
if (rbit) rw = 0
@for (x over j to i) {
new:= settab{u64, x}
if (rbit) rw|= new & ((u64~~1)<<j)
}
storebit{0, rw}
if (mode == 'none' and u == 0) return{u} # Won't ever trigger (m != 0)!
def done = makelabel{}
def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}}
xv:= *VI~~x0
while (i < n) {
i0:= i; iw:= i/32
v:= load{xv, iw}
m:= bitsel{v} # Mask of possibly-new values
if (not match{t,0}) m^= u32~~promote{i32, t}
i+= 32
if (i > n) m&= (~u32~~0)>>((-n)%32)
# Any new values?
if (m == 0) {
storebit{iw, m}
} else {
# Add values to the table and filter m
if (rbit) rv = m
xi:= load{x, i0 + ctzi{m}}
settab1{xi}
if ((m&(m-1)) != 0) { # More bits than one
# Filter out values equal to the previous, or first new
def pind = (iota{32}&15) - 1
prev:= make{VI, each{bind{max,0}, pind}}
e:= ~homMask{v == VI**TI~~xi}
e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}}
if (rbit) rv&= e | -m # Don't remove first bit
m&= e
while (m != 0) {
new:= settab{u32, load{x, i0 + ctzi{m}}}
m1:= m-1; m&= m1 # Clear low bit
if (rbit) rv&= m1 | new # Clear if not new
}
}
storebit{iw, rv}
if (u == 0) { # All bytes seen
if (rbit) @for (r in *u32~~r0 over _ from iw+1 to cdiv{n,32}) r = 0
goto{done}
}
reload_tab{}
}
}
setlabel{done}
}
setlabel{done}
u
}
fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = {
init:= VI**(-1)
@unroll (t in *VI~~tab over 8) t = init
fn simd_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = {
bittab_init{tab, -1}
u:u8 = 0
do_bittab{x0, n, tab, u, 0, 'mask', r0}
}
fn avx2_deduplicate_u8(x0:*void, n:u64, r0:*void, tab:*void) : u64 = {
fn simd_deduplicate_u8(x0:*void, n:u64, r0:*void, tab:*void) : u64 = {
assert{n != 0}
init:= VI**(-1)
@unroll (t in *VI~~tab over 8) t = init
bittab_init{tab, -1}
u:u8 = 0
do_bittab{x0, n, tab, u, 0, 'unique', r0}
1 + promote{u64, u-1} # 0 to 256
@ -209,30 +247,26 @@ fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = {
do_bittab{x0, n, tab, u, t, 'none', 0}
}
fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = {
fn simd_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = {
assert{nw > 0}
vtab:= *VI~~tab
# Initialize
rev:u1 = nx < nw/4 # Reverse lookup
z:= VI**(-promote{i8,rev})
@unroll (vtab over 8) vtab = z
bittab_init{tab, -promote{i8,rev}}
u:u8 = 0 # Sum of table, either 0 or 256
if (rev) u = fill_bittab(x0, nx, tab, u, 0)
# Fill table
u = fill_bittab(w0, nw, tab, u, -1)
if (u == 0) { # All found!
@for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64}
return{}
}
# Read result
def {bitsel, _} = bittab_selector{readbytes{vtab}}
@for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x}
if (u == 0) { # All found!
@for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64}
} else {
bittab_lookup{x0, nx, r0, tab}
}
}
export{'avx2_mark_firsts_u8', avx2_mark_firsts_u8}
export{'avx2_deduplicate_u8', avx2_deduplicate_u8}
export{'avx2_member_u8', avx2_member_u8}
} # hasarch{'AVX2'}
export{'simd_mark_firsts_u8', simd_mark_firsts_u8}
export{'simd_deduplicate_u8', simd_deduplicate_u8}
export{'simd_member_u8', simd_member_u8}