diff --git a/src/builtins/slash.c b/src/builtins/slash.c index a92f7878..6bb45fe4 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -109,6 +109,95 @@ for (u64 v=X[i]; v; v&=v-1) R[j++] = i*64 + CTZ(v); \ } while (0) +static B where(B x, usz xia, u64 s) { + B r; + u64* xp = bitarr_ptr(x); + usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<= xia/16) { + i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16); + bmipopc_1slash16(xp, rp, xia); + } + #else + if (s >= xia/2) { + i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 2); + for (usz i=0; i<(xia+7)/8; i++) { + u8 v = ((u8*)xp)[i]; + for (usz k=0; k<8; k++) { *rp=8*i+k; rp+=v&1; v>>=1; } + } + } + #endif + else { + i16* rp; r=m_i16arrv(&rp,s); WHERE_SPARSE(xp,rp,s); + } + } else { + assert(xia <= (usz)I32_MAX+1); + #if SINGELI && defined(__BMI2__) + i32* rp; r = m_i32arrv(&rp, s); + #else + i32* rp = m_tyarrvO(&r, 4, s, t_i32arr, 4); + #endif + usz b = 1<<11; // Maximum allowed for branchless sparse method + TALLOC(i16, buf, b); + i32* rq=rp; usz i=0; + for (; ixia-i) { + b = xia-i; + bs = s-(rq-rp); + } else { + bs = bit_sum(xp,b); + } + #if SINGELI && defined(__BMI2__) + if (bs >= b/8+b/16) { + bmipopc_1slash16(xp, buf, b); + for (usz j=0; j= b/2) { + for (usz ii=0; ii<(b+7)/8; ii++) { + u8 v = ((u8*)xp)[ii]; + i32* rs=rq; + for (usz k=0; k<8; k++) { *rs=i+8*ii+k; rs+=v&1; v>>=1; } + } + } + #endif + else if (bs >= b/256) { // Branchless sparse + for (usz j=0; j>=24; + p=(u32)u&(top-1); rq[j]+=(3*top)|p; j+=POPC(p); u>>=24; + p=(u32)u ; rq[j]+=(3*top)|p; j+=POPC(p); + } + u64 t=((u64)i<<21)-2*top; + for (usz j=0; j>24) + CTZ((u32)t); + t &= t-1; + } + } else { // Branched very sparse + for (usz ii=i/64, j=0; j 32768) { - usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<xia-i) { - b = xia-i; - bs = s-(rq-rp); - } else { - bs = bit_sum(xp,b); - } - #if SINGELI && defined(__BMI2__) - if (bs >= b/8+b/16) { - bmipopc_1slash16(xp, buf, b); - for (usz j=0; j= b/2) { - for (usz ii=0; ii<(b+7)/8; ii++) { - u8 v = ((u8*)xp)[ii]; - i32* rs=rq; - for (usz k=0; k<8; k++) { *rs=i+8*ii+k; rs+=v&1; v>>=1; } - } - #endif - } else if (bs >= b/256) { // Branchless sparse - for (usz j=0; j>=24; - p=(u32)u&(top-1); rq[j]+=(3*top)|p; j+=POPC(p); u>>=24; - p=(u32)u ; rq[j]+=(3*top)|p; j+=POPC(p); - } - u64 t=((u64)i<<21)-2*top; - for (usz j=0; j>24) + CTZ((u32)t); - t &= t-1; - } - } else { // Branched very sparse - for (usz ii=i/64, j=0; j128 && s < xia/16) { - #else - if (xia<=128 || s < xia/2) { - #endif - usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<>=1; } - } - } - #endif + r = where(x, xia, s); } else { i32* rp; r = m_i32arrv(&rp, s); if (xe==el_i8) {