From 855541954f1f8b3d071b04b5697a46a8fe952cf5 Mon Sep 17 00:00:00 2001 From: dzaima Date: Fri, 10 Feb 2023 15:35:34 +0200 Subject: [PATCH] =?UTF-8?q?initial=20bit=20widen&narrow=20for=20=E2=8A=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/select.c | 39 ++++++++++-- src/builtins/sfns.c | 2 +- src/core/stuff.c | 2 +- src/core/stuff.h | 19 +++--- src/h.h | 6 ++ src/utils/bits.c | 144 +++++++++++++++++++++++++++++++++++++++++- src/vm.c | 2 +- 7 files changed, 192 insertions(+), 22 deletions(-) diff --git a/src/builtins/select.c b/src/builtins/select.c index e260be25..f9dfa8f6 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -7,6 +7,9 @@ // Atom or enclosed atom 𝕨 and high-rank 𝕩: slice // Empty 𝕨: no selection // Float or generic 𝕨: attempt to squeeze, go generic cell size path if stays float +// High-rank 𝕩 & boolean 𝕨: either widens 𝕨 to i8, or goes generic cell path +// SHOULD go a bit select path for small cells +// SHOULD reshape for 1=≠𝕩 // Boolean 𝕩 (cell size = 1 bit): // 𝕨 larger than 𝕩: convert 𝕩 to i8, select, convert back // Otherwise: select/shift bytes, reversed for fast writing @@ -43,7 +46,6 @@ #include "../utils/includeSingeli.h" #endif -typedef size_t ux; // TODO move to h.h typedef void (*CFn)(void* r, ux rs, void* x, ux xs, ux data); typedef struct { CFn fn; @@ -58,6 +60,7 @@ static void cf_3(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*) static void cf_4(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 4); } static CFn cfs_0_4[] = {cf_0, cf_1, cf_2, cf_3, cf_4}; static void cf_8(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 8); } +static void cf_16(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 16); } static void cf_5_7 (void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 4); memcpy(r+d, x+d, 4); } static void cf_9_16 (void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 8); memcpy(r+d, x+d, 8); } static void cf_17_24(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 16); memcpy(r+d, x+d, 8); } @@ -74,6 +77,7 @@ NOINLINE CFRes cf_get(usz count, usz cszBits) { if (bytes<5) return (CFRes){.mul=cszBytes, .fn = cfs_0_4[bytes]}; if (bytes<8) return (CFRes){.mul=cszBytes, .fn = cf_5_7, .data=bytes-4}; if (bytes==8) return (CFRes){.mul=cszBytes, .fn = cf_8}; + if (bytes==16) return (CFRes){.mul=cszBytes, .fn = cf_16}; if (bytes<=16) return (CFRes){.mul=cszBytes, .fn = cf_9_16, .data=bytes-8}; if (bytes<=24) return (CFRes){.mul=cszBytes, .fn = cf_17_24, .data=bytes-8}; if (bytes<=32) return (CFRes){.mul=cszBytes, .fn = cf_25_32, .data=bytes-8}; @@ -173,9 +177,9 @@ B select_c2(B t, B w, B x) { #if SINGELI_X86_64 #define CPUSEL(W, NEXT) /*assumes 3≤xl≤6*/ \ if (!avx2_select_tab[4*(we-el_i8)+xl-3](wp, xp, rp, wia, xn)) thrM("⊏: Indexing out-of-bounds"); - #define BOOL_USE_SIMD (xia<=128) + bool bool_use_simd = we==el_i8 && xl==0 && xia<=128; #define BOOL_SPECIAL(W) \ - if (sizeof(W)==1 && BOOL_USE_SIMD) { \ + if (sizeof(W)==1 && bool_use_simd) { \ if (!avx2_select_bool128(wp, xp, rp, wia, xn)) thrM("⊏: Indexing out-of-bounds"); \ goto setsh; \ } @@ -202,12 +206,26 @@ B select_c2(B t, B w, B x) { } \ if (wt) TFREE(wt); \ } - #define BOOL_USE_SIMD 0 + bool bool_use_simd = 0; #define BOOL_SPECIAL(W) #endif - if (!(BOOL_USE_SIMD && xl==0 && we==el_i8) && xe==el_bit && wia>=256 && we!=el_bit && ((csz&7)!=0) && (xl==0? wia/4>=xia : wia>=xia/4 && csz<40)) { - return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x))))); + if (!bool_use_simd && xe==el_bit && (csz&7)!=0 && (xl==0? wia>=256 : wia>=4) && csz<128) { + // test widen/narrow on bitarr input + // ShArr* sh = RNK(x)==1? NULL : ptr_inc(shObj(x)); + // B t = select_c2(m_f64(0), w, widenBitArr(x, 1)); + // B r = narrowWidenedBitArr(t, wr, xr-1, sh==NULL? &xn : sh->a+1); + // if (sh!=NULL) ptr_dec(sh); + // return r; + if (csz==1) { + if (wia/4>=xia) return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x))))); + } else if (csz>64? wia/2>=xn : wia>=xn/2) { + ShArr* sh = ptr_inc(shObj(x)); + B t = select_c2(m_f64(0), w, widenBitArr(x, 1)); + B r = narrowWidenedBitArr(t, wr, xr-1, sh->a+1); + ptr_dec(sh); + return r; + } } @@ -242,7 +260,14 @@ B select_c2(B t, B w, B x) { retry: switch (we) { default: UD; case el_bit: { - if (xr!=1) goto generic_l; + if (xr!=1) { + if (xe!=el_B && (csz<refc<=0 || (x->refc>>28) == 'a' || x->type==t_empty) { PRINT_ID(x); - fprintf(stderr, "bad refcount for type %d: %d\nattempting to print: ", x->type, x->refc); fflush(stderr); + fprintf(stderr, "bad refcount for type %d @ %p: %d\nattempting to print: ", x->type, x, x->refc); fflush(stderr); fprintI(stderr, tag(x,OBJ_TAG)); fputc('\n', stderr); fflush(stderr); err(""); } diff --git a/src/core/stuff.h b/src/core/stuff.h index 408c6ee4..b944c540 100644 --- a/src/core/stuff.h +++ b/src/core/stuff.h @@ -108,9 +108,9 @@ static Arr* arr_shCopy(Arr* n, B o) { // copy shape & rank from o to n assert(IA(o)==n->ia); return arr_shCopyUnchecked(n, o); } -static void shcpy(usz* dst, usz* src, size_t len) { +static void shcpy(usz* dst, usz* src, ux len) { // memcpy(dst, src, len*sizeof(usz)); - PLAINLOOP for (size_t i = 0; i < len; i++) dst[i] = src[i]; + PLAINLOOP for (ux i = 0; i < len; i++) dst[i] = src[i]; } static usz shProd(usz* sh, usz s, usz e) { @@ -140,6 +140,9 @@ B bit_sel(B b, B e0, B e1); // consumes b; b must be bitarr; b⊏e0‿e1 Arr* allZeroes(usz ia); Arr* allOnes(usz ia); B bit_negate(B x); // consumes +B widenBitArr(B x, ur axis); // consumes x, assumes bitarr; returns some array with cell size padded to the nearest of 8,16,32,64 if ≤64 bits, or a multiple of 64 bits otherwise +B narrowWidenedBitArr(B x, ur axis, ur cr, usz* csh); // consumes x.val; undoes widenBitArr, overriding shape past axis to cr↑csh + Arr* cpyWithShape(B x); // consumes; returns array with refcount 1 with the same shape as x; to allocate a new shape in its place, the previous one needs to be freed, rank set to 1, and then shape & rank set to the new ones Arr* emptyArr(B x, ur xr); // doesn't consume; returns an empty array with the fill of x; if xr>1, shape is unset @@ -165,15 +168,9 @@ static u8 selfElType(B x) { // guaranteed to fit fill if (isC32(x)) return LIKELY(q_c8(x))? el_c8 : q_c16(x)? el_c16 : el_c32; return el_B; } -static bool elChr(u8 x) { - return x>=el_c8 && x<=el_c32; -} -static bool elNum(u8 x) { - return x<=el_f64; -} -static bool elInt(u8 x) { - return x<=el_i32; -} +static bool elChr(u8 x) { return x>=el_c8 && x<=el_c32; } +static bool elNum(u8 x) { return x<=el_f64; } +static bool elInt(u8 x) { return x<=el_i32; } // string stuff diff --git a/src/h.h b/src/h.h index 7095ba16..305109f5 100644 --- a/src/h.h +++ b/src/h.h @@ -114,6 +114,12 @@ typedef uint32_t u32; typedef int64_t i64; typedef uint64_t u64; typedef double f64; +typedef size_t ux; +#if defined(__SIZEOF_INT128__) + #define HAS_U128 1 + typedef unsigned __int128 u128; + typedef signed __int128 i128; +#endif #define I8_MIN -128 #define I8_MAX 127 #define I16_MIN -32768 diff --git a/src/utils/bits.c b/src/utils/bits.c index 19955289..80e9af76 100644 --- a/src/utils/bits.c +++ b/src/utils/bits.c @@ -1,4 +1,5 @@ #include "../core.h" +#include "mut.h" #if SINGELI @@ -78,4 +79,145 @@ NOINLINE B bit_sel(B b, B e0, B e1) { dec_ret: decG(b); return r; -} \ No newline at end of file +} + + +static inline u64 rbuu64(u64* p, ux off) { // read bit-unaligned u64; aka 64↑off↓p + ux p0 = off>>6; + ux m0 = off&63; + u64 v0 = p[p0]; + u64 v1 = p[p0+1]; + #if HAS_U128 + u128 v = v0 | ((u128)v1)<<64; + return v>>m0; + #else + return m0==0? v0 : v0>>m0 | v1<<(64-m0); + #endif +} + +typedef struct { + u64* ptr; + u64 tmp; + ux off; +} ABState; +static ABState ab_new(u64* p) { return (ABState){.ptr=p, .tmp=0, .off=0}; } +static void ab_done(ABState s) { if (s.off) *s.ptr = s.tmp; } +FORCE_INLINE void ab_add(ABState* state, u64 val, ux count) { // assumes bits past count are 0 + assert(count==64 || (val>>count)==0); + ux off0 = state->off; + ux off1 = off0 + count; + state->off = off1&63; + assert((state->tmp>>off0) == 0); + state->tmp|= val<=64) { + *state->ptr++ = state->tmp; + state->tmp = off0==0? 0 : val>>(64-off0); + } +} + + +FORCE_INLINE u64 ruu64(void* p) { u64 r; memcpy(&r, p, 8); return r; } // read byte-unaligned u64 +FORCE_INLINE void wuu64(void* p, u64 v) { memcpy(p, &v, 8); } // write byte-unaligned u64 + +static NOINLINE B zeroPadToCellBits0(B x, usz lr, usz cam, usz pcsz, usz ncsz) { // consumes; for now assumes ncsz is either a multiple of 64, or one of 8,16,32 + assert((ncsz&7) == 0 && RNK(x)>=1 && pcsz>6; // previous full u64 count in cell + u64 msk = (1ull<<(pcsz&63))-1; + for (ux i = 0; i < cam; i++) { + for (ux j = 0; j < pfu64; j++) rp[j] = rbuu64(xp, i*pcsz + j*64); + rp[pfu64] = rbuu64(xp, i*pcsz + pfu64*64) & msk; + rp+= ncsz>>6; + } + } + decG(x); + return taga(r); +} +NOINLINE B widenBitArr(B x, ur axis) { + assert(isArr(x) && TI(x,elType)==el_bit && axis>=1 && RNK(x)>=axis); + usz pcsz = shProd(SH(x), axis, RNK(x)); + usz ncsz; + if (pcsz<=8) ncsz = 8; + else if (pcsz<=16) ncsz = 16; + else if (pcsz<=32) ncsz = 32; + // else if (pcsz<=64) ncsz = 64; + // else ncsz = (pcsz+7)&~(usz)7; + else ncsz = (pcsz+63)&~(usz)63; + if (ncsz==pcsz) return x; + + return zeroPadToCellBits0(x, axis, shProd(SH(x), 0, axis), pcsz, ncsz); +} +B narrowWidenedBitArr(B x, ur axis, ur cr, usz* csh) { // for now assumes the bits to be dropped are zero, origCellBits is a multiple of 8, and that there's at most 63 padding bits + if (TI(x,elType)!=el_bit) return taga(cpyBitArr(x)); + + usz xcsz = shProd(SH(x), axis, RNK(x)); + usz ocsz = shProd(csh, 0, cr); + // printf("narrowWidenedBitArr ia=%d axis=%d cr=%d ocsz=%d xcsz=%d\n", IA(x), axis, cr, ocsz, xcsz); + assert((xcsz&7) == 0 && ocsza, SH(x), axis); + shcpy(rsh->a+axis, csh, cr); + arr_shReplace(r, axis+cr, rsh); + return taga(r); + } + + + usz cam = shProd(SH(x), 0, axis); + u64* rp; + Arr* r = m_bitarrp(&rp, cam*ocsz); + usz* rsh = arr_shAlloc(r, axis+cr); + shcpy(rsh, SH(x), axis); + shcpy(rsh+axis, csh, cr); + + u8* xp = tyany_ptr(x); + // FILL_TO(rp, el_bit, 0, m_f64(1), PIA(r)); + ABState ab = ab_new(rp); + if (xcsz<=64 && (xcsz&(xcsz-1)) == 0) { + switch(xcsz) { default: UD; + case 8: for (ux i=0; i>6; // full u64 count per cell in x + u64 msk = (1ull<<(ocsz&63))-1; + for (ux i = 0; i < cam; i++) { + for (ux j = 0; j < rfu64; j++) ab_add(&ab, ruu64(j + (u64*)xp), 64); + ab_add(&ab, ruu64(rfu64 + (u64*)xp)&msk, ocsz&63); + rp+= ocsz>>6; + xp+= xcsz>>3; + } + } + ab_done(ab); + decG(x); + return taga(r); +} diff --git a/src/vm.c b/src/vm.c index ccc023ad..83964a87 100644 --- a/src/vm.c +++ b/src/vm.c @@ -1008,7 +1008,7 @@ B md1Bl_c2(Md1D* d, B w, B x) { Md1Block* b=(Md1Block*)d->m1; ptr_inc(d); return B md2Bl_c1(Md2D* d, B x) { Md2Block* b=(Md2Block*)d->m2; ptr_inc(d); return execBlock(b->bl, b->bl->bodies[0], b->sc, 6, (B[]){tag(d,FUN_TAG), x, bi_N, tag(ptr_inc(d->m2),MD2_TAG), inc(d->f), inc(d->g)}); } B md2Bl_c2(Md2D* d, B w, B x) { Md2Block* b=(Md2Block*)d->m2; ptr_inc(d); return execBlock(b->bl, b->bl->dyBody, b->sc, 6, (B[]){tag(d,FUN_TAG), x, w , tag(ptr_inc(d->m2),MD2_TAG), inc(d->f), inc(d->g)}); } -NORETURN NOINLINE static void noInv(Body* bo, Scope* psc, i8 type, i8 inv) { +static NOINLINE NORETURN void noInv(Body* bo, Scope* psc, i8 type, i8 inv) { pushEnv(m_scope(bo, psc, 0, 0, (B[]){}), bo->bc); thrF("No %U undo header found for this%U block", inv==0? "monadic" : inv==1? "dyadic F˜⁼" : "dyadic F⁼", type==0? "" : type==1? " 1-modifier" : " 2-modifier"); }