From fd1178bc0ec09ea7a389e6998001af45cc84bb0d Mon Sep 17 00:00:00 2001 From: dzaima Date: Fri, 10 Feb 2023 23:03:46 +0200 Subject: [PATCH] use BMI2 for widening to <32-bit cells --- src/utils/bits.c | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/utils/bits.c b/src/utils/bits.c index 83bbb663..1ec48905 100644 --- a/src/utils/bits.c +++ b/src/utils/bits.c @@ -8,6 +8,10 @@ #define bitselFns simd_bitsel #endif +#if defined(__BMI2__) && !SLOW_PDEP + #define FAST_PDEP 1 +#endif + NOINLINE Arr* allZeroes(usz ia) { u64* rp; Arr* r = m_bitarrp(&rp, ia); for (usz i = 0; i < BIT_N(ia); i++) rp[i] = 0; return r; } NOINLINE Arr* allOnes (usz ia) { u64* rp; Arr* r = m_bitarrp(&rp, ia); for (usz i = 0; i < BIT_N(ia); i++) rp[i] = ~0ULL; return r; } @@ -137,13 +141,33 @@ static NOINLINE B zeroPadToCellBits0(B x, usz lr, usz cam, usz pcsz, usz ncsz) { // TODO widen 8/16-bit cells to 16/32 via cpyC(16|32)Arr if (ncsz<=64 && (ncsz&(ncsz-1)) == 0) { - u64 msk = (1ull<>6; // previous full u64 count in cell @@ -172,10 +196,6 @@ NOINLINE B widenBitArr(B x, ur axis) { return zeroPadToCellBits0(x, axis, shProd(SH(x), 0, axis), pcsz, ncsz); } -#if defined(__BMI2__) && !SLOW_PDEP - #define FAST_PDEP 1 -#endif - B narrowWidenedBitArr(B x, ur axis, ur cr, usz* csh) { // for now assumes the bits to be dropped are zero, origCellBits is a multiple of 8, and that there's at most 63 padding bits if (TI(x,elType)!=el_bit) return taga(cpyBitArr(x)); @@ -213,12 +233,12 @@ B narrowWidenedBitArr(B x, ur axis, ur cr, usz* csh) { // for now assumes the bi u64 msk0 = tmsk * (c8? 0x0101010101010101 : 0x0001000100010001); ux am = c8? cam/8 : cam/4; u32 count = POPC(msk0); - // printf("base %04lx %016lx count=%d am=%zu\n", tmsk, msk0, count, am); + // printf("narrow base %04lx %016lx count=%d am=%zu\n", tmsk, msk0, count, am); for (ux i=0; i