Generic Singeli where implementation
This commit is contained in:
parent
c484f8ee8d
commit
4415869496
@ -162,8 +162,8 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) {
|
||||
|
||||
static void where_block_u16(u64* src, u16* dst, usz len, usz sum) {
|
||||
assert(len <= bsp_max);
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len, sum);
|
||||
#if SINGELI
|
||||
if (sum >= len/si_thresh_1slash16) si_1slash16(src, (i16*)dst, len, sum);
|
||||
#else
|
||||
if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0);
|
||||
#endif
|
||||
@ -235,17 +235,17 @@ static B where(B x, usz xia, u64 s) {
|
||||
u64* xp = bitarr_ptr(x);
|
||||
usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<<q) - 1;
|
||||
if (xia <= 128) {
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
#if SINGELI
|
||||
i8* rp = m_tyarrv(&r, 1, s, t_i8arr);
|
||||
bmipopc_1slash8(xp, rp, xia, s);
|
||||
si_1slash8(xp, rp, xia, s);
|
||||
#else
|
||||
i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s,0,);
|
||||
#endif
|
||||
} else if (xia <= 32768) {
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (s >= xia/8) {
|
||||
#if SINGELI
|
||||
if (s >= xia/si_thresh_1slash16) {
|
||||
i16* rp = m_tyarrv(&r, 2, s, t_i16arr);
|
||||
bmipopc_1slash16(xp, rp, xia, s);
|
||||
si_1slash16(xp, rp, xia, s);
|
||||
}
|
||||
#else
|
||||
if (s >= xia/4+xia/8) {
|
||||
@ -274,9 +274,9 @@ static B where(B x, usz xia, u64 s) {
|
||||
} else {
|
||||
bs = bit_sum(xp,b);
|
||||
}
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (bs >= b/8+b/16) {
|
||||
bmipopc_1slash16(xp, buf, b, bs);
|
||||
#if SINGELI
|
||||
if (bs >= b/si_thresh_1slash16+b/16) {
|
||||
si_1slash16(xp, buf, b, bs);
|
||||
for (usz j=0; j<bs; j++) rq[j] = i+buf[j];
|
||||
}
|
||||
#else
|
||||
@ -360,19 +360,19 @@ B grade_bool(B x, usz xia, bool up) {
|
||||
u64* xp = bitarr_ptr(x);
|
||||
u64 sum = bit_sum(xp, xia);
|
||||
u64 l0 = up? xia-sum : sum; // Length of first set of indices
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
#if SINGELI
|
||||
if (xia < 16) { BRANCHLESS_GRADE(i8) }
|
||||
else if (xia <= 1<<15) {
|
||||
B notx = bit_negate(incG(x));
|
||||
u64* xp0 = bitarr_ptr(notx);
|
||||
u64* xp1 = xp;
|
||||
if (!up) { u64* t=xp1; xp1=xp0; xp0=t; }
|
||||
#define BMI_GRADE(W) \
|
||||
#define SI_GRADE(W) \
|
||||
i##W* rp = m_tyarrv(&r, W/8, xia, t_i##W##arr); \
|
||||
bmipopc_1slash##W(xp0, rp , xia, l0 ); \
|
||||
bmipopc_1slash##W(xp1, rp+l0, xia, xia-l0);
|
||||
if (xia <= 128) { BMI_GRADE(8) } else { BMI_GRADE(16) }
|
||||
#undef BMI_GRADE
|
||||
si_1slash##W(xp0, rp , xia, l0 ); \
|
||||
si_1slash##W(xp1, rp+l0, xia, xia-l0);
|
||||
if (xia <= 128) { SI_GRADE(8) } else { SI_GRADE(16) }
|
||||
#undef SI_GRADE
|
||||
decG(notx);
|
||||
} else if (xia <= 1ull<<31) {
|
||||
i32* rp0; r = m_i32arrv(&rp0, xia);
|
||||
@ -384,8 +384,8 @@ B grade_bool(B x, usz xia, bool up) {
|
||||
for (usz i=0; i<xia; i+=b) {
|
||||
for (usz j=0; j<BIT_N(b); j++) xp0[j] = ~xp1[j];
|
||||
usz b2 = b>xia-i? xia-i : b;
|
||||
usz s0=bit_sum(xp0,b2); bmipopc_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j<s0; j++) *rp0++ = i+buf[j];
|
||||
usz s1=b2-s0; bmipopc_1slash8(xp1, (i8*)buf, b2, s1); for (usz j=0; j<s1; j++) *rp1++ = i+buf[j];
|
||||
usz s0=bit_sum(xp0,b2); si_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j<s0; j++) *rp0++ = i+buf[j];
|
||||
usz s1=b2-s0; si_1slash8(xp1, (i8*)buf, b2, s1); for (usz j=0; j<s1; j++) *rp1++ = i+buf[j];
|
||||
xp1+= b2/64;
|
||||
}
|
||||
TFREE(buf);
|
||||
|
||||
@ -57,11 +57,12 @@ def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
|
||||
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
|
||||
|
||||
# Assumes w is trimmed, so the last 1 appears at index l-1
|
||||
def thresh1{T} = 2
|
||||
def thresh2{T} = 2
|
||||
fn slash{c==1, T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = {
|
||||
fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1
|
||||
@for (x over i to l) {
|
||||
store{r, 0, x}
|
||||
@for (i to l) {
|
||||
store{r, 0, if (c) load{x,i} else i}
|
||||
r+= bitp_get{w,i}
|
||||
}
|
||||
}
|
||||
@ -104,6 +105,7 @@ def tab{n,l} = if (n==0) tup{0} else {
|
||||
c16lut:*u64 = tab{4,16}
|
||||
def vgLoad{p:T, i & T == *u64} = emit{eltype{T}, 'vg_loadLUT64', p, i}
|
||||
|
||||
def thresh1{T==i16 & hasarch{'BMI2'}} = 8
|
||||
def thresh2{T==i8 & hasarch{'BMI2'}} = 32
|
||||
def thresh2{T==i16 & hasarch{'BMI2'}} = 16
|
||||
fn slash{c, T & hasarch{'BMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
@ -233,10 +235,8 @@ fn slash{c==1, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u
|
||||
}
|
||||
}
|
||||
|
||||
if (hasarch{'BMI2'}) {
|
||||
export{'bmipopc_1slash8', slash{0, i8}}
|
||||
export{'bmipopc_1slash16', slash{0, i16}}
|
||||
}
|
||||
export{'si_1slash8' , slash{0, i8 }}
|
||||
export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh1{i16}}
|
||||
export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }}
|
||||
export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}}
|
||||
export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user