Dedicated 32-bit Singeli where, taking index offset

This commit is contained in:
Marshall Lochbaum 2023-07-16 21:32:48 -04:00
parent 3bd8d1de68
commit 8b297ae2dc
2 changed files with 20 additions and 11 deletions

View File

@ -275,9 +275,8 @@ static B where(B x, usz xia, u64 s) {
bs = bit_sum(xp,b);
}
#if SINGELI
if (bs >= b/si_thresh_1slash16+b/16) {
si_1slash16(xp, buf, b, bs);
for (usz j=0; j<bs; j++) rq[j] = i+buf[j];
if (bs >= b/si_thresh_1slash32) {
si_1slash32(xp, i, rq, b, bs);
}
#else
if (bs >= b/2) {
@ -379,17 +378,17 @@ B grade_bool(B x, usz xia, bool up) {
i32* rp0; r = m_i32arrv(&rp0, xia);
i32* rp1 = rp0 + l0;
if (!up) { i32* t=rp1; rp1=rp0; rp0=t; }
usz b = 256; TALLOC(u8, buf, b);
usz b = 256;
u64 xp0[4]; // 4 ≡ b/64
u64* xp1 = xp;
for (usz i=0; i<xia; i+=b) {
for (usz j=0; j<BIT_N(b); j++) xp0[j] = ~xp1[j];
usz b2 = b>xia-i? xia-i : b;
usz s0=bit_sum(xp0,b2); si_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j<s0; j++) *rp0++ = i+buf[j];
usz s1=b2-s0; si_1slash8(xp1, (i8*)buf, b2, s1); for (usz j=0; j<s1; j++) *rp1++ = i+buf[j];
if (b2<b) { u64 q=b2%64; usz e=b2/64; u64 m=((u64)1<<q)-1; xp0[e]&=m; xp1[e]&=m; }
usz s0=bit_sum(xp0,b2); si_1slash32(xp0, i, rp0, b2, s0); rp0+=s0;
usz s1=b2-s0; si_1slash32(xp1, i, rp1, b2, s1); rp1+=s1;
xp1+= b2/64;
}
TFREE(buf);
}
#else
if (xia <= 128) { BRANCHLESS_GRADE(i8) }

View File

@ -29,7 +29,7 @@ if (hasarch{'AVX512F'}) {
include './mask'
include 'util/tup'
def arg{c,T} = if (c) *T else tup{}
def arg{c,T} = if (c) *T else if (T==i32) T else tup{}
# Modifies the input variable r
# Assumes iter{} will increment r, by at most write_len
@ -73,7 +73,7 @@ def thresh2{T} = 2
fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1
@for (i to l) {
store{r, 0, if (c) load{x,i} else i}
store{r, 0, if (c) load{x,i} else if (T==i32) cast_i{T,i}+x else i}
r+= bitp_get{w,i}
}
}
@ -85,6 +85,7 @@ def getter{c, V, x} = {
} else {
def k = vcount{V}
i := make{V, iota{k}}
if (eltype{V}==i32) i += V**x
ii := V**k
{} => { v:=i; i+=ii; v }
}
@ -220,15 +221,23 @@ def thresh2{T==i64 & hasarch{'AVX2'}} = 8
fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def tw = width{T}
def V = [8]u32
def X = getter{c, V, x}
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}}
def from_ind = if (c) {
i:u64 = 0
{j} => { v:=load{*V~~x, i}; ++i; sel{V, v, j} }
} else if (T==i32) {
def VT = [8]T
i := VT**x
ii := VT**8
{j} => { v:=i+VT~~j; i+=ii; v }
}
def tab = if (tw==32) itab else i64tab
def step{w} = {
pc := popc{w}
ind := load{tab, w}; def I = type{ind}
s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander}
if (tw==64) s |= make{V, iota{8}%2}
store{*V~~r, 0, sel{V, X{}, s}}
store{*V~~r, 0, from_ind{s}}
r+= pc
}
@for_special_buffered{r,8} (w in *u8~~wp to sum) {
@ -272,6 +281,7 @@ fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64,
export{'si_1slash8' , slash{0, i8 }}
export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh2{i16}}
export{'si_1slash32', slash{0, i32}}; export{'si_thresh_1slash32', u64~~thresh2{i32}}
export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }}
export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}}
export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}}