Dedicated 32-bit Singeli where, taking index offset
This commit is contained in:
parent
3bd8d1de68
commit
8b297ae2dc
@ -275,9 +275,8 @@ static B where(B x, usz xia, u64 s) {
|
|||||||
bs = bit_sum(xp,b);
|
bs = bit_sum(xp,b);
|
||||||
}
|
}
|
||||||
#if SINGELI
|
#if SINGELI
|
||||||
if (bs >= b/si_thresh_1slash16+b/16) {
|
if (bs >= b/si_thresh_1slash32) {
|
||||||
si_1slash16(xp, buf, b, bs);
|
si_1slash32(xp, i, rq, b, bs);
|
||||||
for (usz j=0; j<bs; j++) rq[j] = i+buf[j];
|
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
if (bs >= b/2) {
|
if (bs >= b/2) {
|
||||||
@ -379,17 +378,17 @@ B grade_bool(B x, usz xia, bool up) {
|
|||||||
i32* rp0; r = m_i32arrv(&rp0, xia);
|
i32* rp0; r = m_i32arrv(&rp0, xia);
|
||||||
i32* rp1 = rp0 + l0;
|
i32* rp1 = rp0 + l0;
|
||||||
if (!up) { i32* t=rp1; rp1=rp0; rp0=t; }
|
if (!up) { i32* t=rp1; rp1=rp0; rp0=t; }
|
||||||
usz b = 256; TALLOC(u8, buf, b);
|
usz b = 256;
|
||||||
u64 xp0[4]; // 4 ≡ b/64
|
u64 xp0[4]; // 4 ≡ b/64
|
||||||
u64* xp1 = xp;
|
u64* xp1 = xp;
|
||||||
for (usz i=0; i<xia; i+=b) {
|
for (usz i=0; i<xia; i+=b) {
|
||||||
for (usz j=0; j<BIT_N(b); j++) xp0[j] = ~xp1[j];
|
for (usz j=0; j<BIT_N(b); j++) xp0[j] = ~xp1[j];
|
||||||
usz b2 = b>xia-i? xia-i : b;
|
usz b2 = b>xia-i? xia-i : b;
|
||||||
usz s0=bit_sum(xp0,b2); si_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j<s0; j++) *rp0++ = i+buf[j];
|
if (b2<b) { u64 q=b2%64; usz e=b2/64; u64 m=((u64)1<<q)-1; xp0[e]&=m; xp1[e]&=m; }
|
||||||
usz s1=b2-s0; si_1slash8(xp1, (i8*)buf, b2, s1); for (usz j=0; j<s1; j++) *rp1++ = i+buf[j];
|
usz s0=bit_sum(xp0,b2); si_1slash32(xp0, i, rp0, b2, s0); rp0+=s0;
|
||||||
|
usz s1=b2-s0; si_1slash32(xp1, i, rp1, b2, s1); rp1+=s1;
|
||||||
xp1+= b2/64;
|
xp1+= b2/64;
|
||||||
}
|
}
|
||||||
TFREE(buf);
|
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
if (xia <= 128) { BRANCHLESS_GRADE(i8) }
|
if (xia <= 128) { BRANCHLESS_GRADE(i8) }
|
||||||
|
|||||||
@ -29,7 +29,7 @@ if (hasarch{'AVX512F'}) {
|
|||||||
include './mask'
|
include './mask'
|
||||||
include 'util/tup'
|
include 'util/tup'
|
||||||
|
|
||||||
def arg{c,T} = if (c) *T else tup{}
|
def arg{c,T} = if (c) *T else if (T==i32) T else tup{}
|
||||||
|
|
||||||
# Modifies the input variable r
|
# Modifies the input variable r
|
||||||
# Assumes iter{} will increment r, by at most write_len
|
# Assumes iter{} will increment r, by at most write_len
|
||||||
@ -73,7 +73,7 @@ def thresh2{T} = 2
|
|||||||
fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||||
def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1
|
def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1
|
||||||
@for (i to l) {
|
@for (i to l) {
|
||||||
store{r, 0, if (c) load{x,i} else i}
|
store{r, 0, if (c) load{x,i} else if (T==i32) cast_i{T,i}+x else i}
|
||||||
r+= bitp_get{w,i}
|
r+= bitp_get{w,i}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -85,6 +85,7 @@ def getter{c, V, x} = {
|
|||||||
} else {
|
} else {
|
||||||
def k = vcount{V}
|
def k = vcount{V}
|
||||||
i := make{V, iota{k}}
|
i := make{V, iota{k}}
|
||||||
|
if (eltype{V}==i32) i += V**x
|
||||||
ii := V**k
|
ii := V**k
|
||||||
{} => { v:=i; i+=ii; v }
|
{} => { v:=i; i+=ii; v }
|
||||||
}
|
}
|
||||||
@ -220,15 +221,23 @@ def thresh2{T==i64 & hasarch{'AVX2'}} = 8
|
|||||||
fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||||
def tw = width{T}
|
def tw = width{T}
|
||||||
def V = [8]u32
|
def V = [8]u32
|
||||||
def X = getter{c, V, x}
|
|
||||||
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}}
|
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}}
|
||||||
|
def from_ind = if (c) {
|
||||||
|
i:u64 = 0
|
||||||
|
{j} => { v:=load{*V~~x, i}; ++i; sel{V, v, j} }
|
||||||
|
} else if (T==i32) {
|
||||||
|
def VT = [8]T
|
||||||
|
i := VT**x
|
||||||
|
ii := VT**8
|
||||||
|
{j} => { v:=i+VT~~j; i+=ii; v }
|
||||||
|
}
|
||||||
def tab = if (tw==32) itab else i64tab
|
def tab = if (tw==32) itab else i64tab
|
||||||
def step{w} = {
|
def step{w} = {
|
||||||
pc := popc{w}
|
pc := popc{w}
|
||||||
ind := load{tab, w}; def I = type{ind}
|
ind := load{tab, w}; def I = type{ind}
|
||||||
s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander}
|
s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander}
|
||||||
if (tw==64) s |= make{V, iota{8}%2}
|
if (tw==64) s |= make{V, iota{8}%2}
|
||||||
store{*V~~r, 0, sel{V, X{}, s}}
|
store{*V~~r, 0, from_ind{s}}
|
||||||
r+= pc
|
r+= pc
|
||||||
}
|
}
|
||||||
@for_special_buffered{r,8} (w in *u8~~wp to sum) {
|
@for_special_buffered{r,8} (w in *u8~~wp to sum) {
|
||||||
@ -272,6 +281,7 @@ fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64,
|
|||||||
|
|
||||||
export{'si_1slash8' , slash{0, i8 }}
|
export{'si_1slash8' , slash{0, i8 }}
|
||||||
export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh2{i16}}
|
export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh2{i16}}
|
||||||
|
export{'si_1slash32', slash{0, i32}}; export{'si_thresh_1slash32', u64~~thresh2{i32}}
|
||||||
export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }}
|
export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }}
|
||||||
export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}}
|
export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}}
|
||||||
export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}}
|
export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user