Dedicated 32-bit Singeli where, taking index offset
This commit is contained in:
parent
3bd8d1de68
commit
8b297ae2dc
@ -275,9 +275,8 @@ static B where(B x, usz xia, u64 s) {
|
||||
bs = bit_sum(xp,b);
|
||||
}
|
||||
#if SINGELI
|
||||
if (bs >= b/si_thresh_1slash16+b/16) {
|
||||
si_1slash16(xp, buf, b, bs);
|
||||
for (usz j=0; j<bs; j++) rq[j] = i+buf[j];
|
||||
if (bs >= b/si_thresh_1slash32) {
|
||||
si_1slash32(xp, i, rq, b, bs);
|
||||
}
|
||||
#else
|
||||
if (bs >= b/2) {
|
||||
@ -379,17 +378,17 @@ B grade_bool(B x, usz xia, bool up) {
|
||||
i32* rp0; r = m_i32arrv(&rp0, xia);
|
||||
i32* rp1 = rp0 + l0;
|
||||
if (!up) { i32* t=rp1; rp1=rp0; rp0=t; }
|
||||
usz b = 256; TALLOC(u8, buf, b);
|
||||
usz b = 256;
|
||||
u64 xp0[4]; // 4 ≡ b/64
|
||||
u64* xp1 = xp;
|
||||
for (usz i=0; i<xia; i+=b) {
|
||||
for (usz j=0; j<BIT_N(b); j++) xp0[j] = ~xp1[j];
|
||||
usz b2 = b>xia-i? xia-i : b;
|
||||
usz s0=bit_sum(xp0,b2); si_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j<s0; j++) *rp0++ = i+buf[j];
|
||||
usz s1=b2-s0; si_1slash8(xp1, (i8*)buf, b2, s1); for (usz j=0; j<s1; j++) *rp1++ = i+buf[j];
|
||||
if (b2<b) { u64 q=b2%64; usz e=b2/64; u64 m=((u64)1<<q)-1; xp0[e]&=m; xp1[e]&=m; }
|
||||
usz s0=bit_sum(xp0,b2); si_1slash32(xp0, i, rp0, b2, s0); rp0+=s0;
|
||||
usz s1=b2-s0; si_1slash32(xp1, i, rp1, b2, s1); rp1+=s1;
|
||||
xp1+= b2/64;
|
||||
}
|
||||
TFREE(buf);
|
||||
}
|
||||
#else
|
||||
if (xia <= 128) { BRANCHLESS_GRADE(i8) }
|
||||
|
||||
@ -29,7 +29,7 @@ if (hasarch{'AVX512F'}) {
|
||||
include './mask'
|
||||
include 'util/tup'
|
||||
|
||||
def arg{c,T} = if (c) *T else tup{}
|
||||
def arg{c,T} = if (c) *T else if (T==i32) T else tup{}
|
||||
|
||||
# Modifies the input variable r
|
||||
# Assumes iter{} will increment r, by at most write_len
|
||||
@ -73,7 +73,7 @@ def thresh2{T} = 2
|
||||
fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1
|
||||
@for (i to l) {
|
||||
store{r, 0, if (c) load{x,i} else i}
|
||||
store{r, 0, if (c) load{x,i} else if (T==i32) cast_i{T,i}+x else i}
|
||||
r+= bitp_get{w,i}
|
||||
}
|
||||
}
|
||||
@ -85,6 +85,7 @@ def getter{c, V, x} = {
|
||||
} else {
|
||||
def k = vcount{V}
|
||||
i := make{V, iota{k}}
|
||||
if (eltype{V}==i32) i += V**x
|
||||
ii := V**k
|
||||
{} => { v:=i; i+=ii; v }
|
||||
}
|
||||
@ -220,15 +221,23 @@ def thresh2{T==i64 & hasarch{'AVX2'}} = 8
|
||||
fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def tw = width{T}
|
||||
def V = [8]u32
|
||||
def X = getter{c, V, x}
|
||||
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}}
|
||||
def from_ind = if (c) {
|
||||
i:u64 = 0
|
||||
{j} => { v:=load{*V~~x, i}; ++i; sel{V, v, j} }
|
||||
} else if (T==i32) {
|
||||
def VT = [8]T
|
||||
i := VT**x
|
||||
ii := VT**8
|
||||
{j} => { v:=i+VT~~j; i+=ii; v }
|
||||
}
|
||||
def tab = if (tw==32) itab else i64tab
|
||||
def step{w} = {
|
||||
pc := popc{w}
|
||||
ind := load{tab, w}; def I = type{ind}
|
||||
s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander}
|
||||
if (tw==64) s |= make{V, iota{8}%2}
|
||||
store{*V~~r, 0, sel{V, X{}, s}}
|
||||
store{*V~~r, 0, from_ind{s}}
|
||||
r+= pc
|
||||
}
|
||||
@for_special_buffered{r,8} (w in *u8~~wp to sum) {
|
||||
@ -272,6 +281,7 @@ fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64,
|
||||
|
||||
export{'si_1slash8' , slash{0, i8 }}
|
||||
export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh2{i16}}
|
||||
export{'si_1slash32', slash{0, i32}}; export{'si_thresh_1slash32', u64~~thresh2{i32}}
|
||||
export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }}
|
||||
export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}}
|
||||
export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user