diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 6bb45fe4..e5b527bd 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -104,11 +104,58 @@ #endif #endif -#define WHERE_SPARSE(X,R,S) do { \ - for (usz i=0, j=0; j>=24; + p=u&bsp_mask; buf[j]+=(3*bsp_top)|p; j+=POPC(p); u>>=24; + p=u ; buf[j]+=(3*bsp_top)|p; j+=POPC(p); + } + return j; +} +static void bsp_block_u32(u64* src, u32* dst, usz len, usz sum, usz off) { + for (usz j=0; j>24) + CTZ((u32)t); + t &= t-1; + } +} +static void bsp_u16(u64* src, u16* dst, usz len, usz sum) { + usz b = bsp_max; + usz bufsize = b len-i) b = len-i; + usz bs = bsp_fill(src+i/64, buf, b); + u64 t=((u64)i<<21)-2*bsp_top; + for (usz j=0; j>24) + CTZ((u32)t); + t &= t-1; + } + buf[bs] = 0; + dst+= bs; + } + TFREE(buf); +} + static B where(B x, usz xia, u64 s) { B r; u64* xp = bitarr_ptr(x); @@ -118,16 +165,16 @@ static B where(B x, usz xia, u64 s) { i8* rp = m_tyarrvO(&r, 1, s, t_i8arr, 8); bmipopc_1slash8(xp, rp, xia); #else - i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s); + i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s,0,); #endif } else if (xia <= 32768) { #if SINGELI && defined(__BMI2__) - if (s >= xia/16) { + if (s >= xia/8) { i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16); bmipopc_1slash16(xp, rp, xia); } #else - if (s >= xia/2) { + if (s >= xia/4+xia/8) { i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 2); for (usz i=0; i<(xia+7)/8; i++) { u8 v = ((u8*)xp)[i]; @@ -136,7 +183,12 @@ static B where(B x, usz xia, u64 s) { } #endif else { - i16* rp; r=m_i16arrv(&rp,s); WHERE_SPARSE(xp,rp,s); + i16* rp; r=m_i16arrv(&rp,s); + if (s >= xia/128) { + bsp_u16(xp, (u16*)rp, xia, s); + } else { + WHERE_SPARSE(xp, rp, s, 0, RARE); + } } } else { assert(xia <= (usz)I32_MAX+1); @@ -145,10 +197,9 @@ static B where(B x, usz xia, u64 s) { #else i32* rp = m_tyarrvO(&r, 4, s, t_i32arr, 4); #endif - usz b = 1<<11; // Maximum allowed for branchless sparse method - TALLOC(i16, buf, b); - i32* rq=rp; usz i=0; - for (; ixia-i) { b = xia-i; @@ -170,25 +221,10 @@ static B where(B x, usz xia, u64 s) { } } #endif - else if (bs >= b/256) { // Branchless sparse - for (usz j=0; j>=24; - p=(u32)u&(top-1); rq[j]+=(3*top)|p; j+=POPC(p); u>>=24; - p=(u32)u ; rq[j]+=(3*top)|p; j+=POPC(p); - } - u64 t=((u64)i<<21)-2*top; - for (usz j=0; j>24) + CTZ((u32)t); - t &= t-1; - } - } else { // Branched very sparse - for (usz ii=i/64, j=0; j= b/256) { + bsp_block_u32(xp, (u32*)rq, b, bs, i); + } else { + WHERE_SPARSE(xp-i/64, rq, bs, i/64, RARE); } rq+= bs; xp+= b/64;