Move pdep-based k/bool code to Singeli

This commit is contained in:
Marshall Lochbaum 2024-08-05 09:57:05 -04:00
parent de5bebb239
commit 0602927b17
2 changed files with 45 additions and 32 deletions

View File

@ -82,6 +82,8 @@
extern void (*const si_scan_max_i32)(int32_t* v0,int32_t* v1,uint64_t v2);
#define SINGELI_FILE slash
#include "../utils/includeSingeli.h"
extern uint64_t* const si_spaced_masks;
#define get_spaced_mask(i) si_spaced_masks[i-1]
#define SINGELI_FILE replicate
#include "../utils/includeSingeli.h"
#endif
@ -765,38 +767,9 @@ B slash_c2(B t, B w, B x) {
if (xl == 0) {
u64* xp = bitarr_ptr(x);
u64* rp; r = m_bitarrv(&rp, s);
#if FAST_PDEP
if (wv <= 52) {
#if SINGELI
u64 m = si_spaced_masks[wv-1];
#else
u64 m = (u64)-1 / (((u64)1<<wv)-1);
#endif
u64 xw = 0;
usz d = POPC(m); // == 64/wv
if (m & 1) { // Power of two
for (usz i=-1, j=0; j<BIT_N(s); j++) {
xw >>= d;
if ((j&(wv-1))==0) xw = xp[++i];
u64 rw = _pdep_u64(xw, m);
rp[j] = (rw<<wv)-rw;
}
} else {
usz q = CTZ(m); // == 64%wv
m = m<<(wv-q) | 1;
u64 mt = (u64)1<<(d+1); // Bit d+1 may be needed, isn't pdep-ed
usz tsh = d*wv-(d+1);
for (usz xi=0, o=0, j=0; j<BIT_N(s); j++) {
xw = loadu_u64((u64*)((u8*)xp+xi/8)) >> (xi%8);
u64 ex = (xw&mt)<<tsh;
u64 rw = _pdep_u64(xw, m);
rp[j] = ((rw-ex)<<(wv-o))-(rw>>o|(xw&1));
o += q;
bool oo = o>=wv; xi+=d+oo; o-=wv&-oo;
}
}
goto atmW_maybesh;
}
#if SINGELI
if (si_constrep_bool(wv, xp, rp, s)) ; // Handles small wv
else
#endif
if (wv <= 256) { BOOL_REP_XOR_SCAN(wv) }
else { BOOL_REP_OVER(wv, xlen) }

View File

@ -1,5 +1,7 @@
include './base'
if_inline (hasarch{'BMI2'}) include './bmi2'
include './mask'
include './spaced'
def ind_types = tup{i8, i16, i32}
def dat_types = tup{...ind_types, u64}
@ -284,3 +286,41 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
}
exportT{'si_constrep', each{rep_const, dat_types}}
# Constant replicate on boolean
fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = 0
fn rep_const_bool{if hasarch{'BMI2'}}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = {
if (wv > 52) return{0}
m:u64 = spaced_mask_of{wv}
xw:u64 = 0
d := cast_i{usz, popc{m}} # == 64/wv
nw := cdiv{rlen, 64}
if (m&1 != 0) { # Power of two
i := -usz~~1
@for (r over j to nw) {
xw >>= d
if ((j&(wv-1))==0) { ++i; xw = load{x, i} }
rw := pdep{xw, m}
r = (rw<<wv) - rw
}
} else {
q := cast_i{usz, ctz{m}} # == 64%wv
m = m<<(wv-q) | 1
mt := u64~~1 << (d+1) # Bit d+1 may be needed, isn't pdep-ed
tsh := d*wv-(d+1)
xb := *u8~~x
xi:usz=0; o:usz=0
@for (r over j to nw) {
xw = loadu{*u64~~(xb + xi/8)} >> (xi%8)
ex := (xw & mt) << tsh
rw := pdep{xw, m}
r = ((rw-ex)<<(wv-o)) - (rw>>o|(xw&1))
o += q
oo := o>=wv; xi+=d+promote{usz,oo}; o-=wv&-oo
}
}
1
}
export{'si_constrep_bool', rep_const_bool{}}