initial bit widen&narrow for ⊏
This commit is contained in:
parent
23d99d7a24
commit
855541954f
@ -7,6 +7,9 @@
|
||||
// Atom or enclosed atom 𝕨 and high-rank 𝕩: slice
|
||||
// Empty 𝕨: no selection
|
||||
// Float or generic 𝕨: attempt to squeeze, go generic cell size path if stays float
|
||||
// High-rank 𝕩 & boolean 𝕨: either widens 𝕨 to i8, or goes generic cell path
|
||||
// SHOULD go a bit select path for small cells
|
||||
// SHOULD reshape for 1=≠𝕩
|
||||
// Boolean 𝕩 (cell size = 1 bit):
|
||||
// 𝕨 larger than 𝕩: convert 𝕩 to i8, select, convert back
|
||||
// Otherwise: select/shift bytes, reversed for fast writing
|
||||
@ -43,7 +46,6 @@
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
typedef size_t ux; // TODO move to h.h
|
||||
typedef void (*CFn)(void* r, ux rs, void* x, ux xs, ux data);
|
||||
typedef struct {
|
||||
CFn fn;
|
||||
@ -58,6 +60,7 @@ static void cf_3(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)
|
||||
static void cf_4(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 4); }
|
||||
static CFn cfs_0_4[] = {cf_0, cf_1, cf_2, cf_3, cf_4};
|
||||
static void cf_8(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 8); }
|
||||
static void cf_16(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 16); }
|
||||
static void cf_5_7 (void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 4); memcpy(r+d, x+d, 4); }
|
||||
static void cf_9_16 (void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 8); memcpy(r+d, x+d, 8); }
|
||||
static void cf_17_24(void* r, ux rs, void* x, ux xs, ux d) { r=rs+(u8*)r; x=xs+(u8*)x; memcpy(r, x, 16); memcpy(r+d, x+d, 8); }
|
||||
@ -74,6 +77,7 @@ NOINLINE CFRes cf_get(usz count, usz cszBits) {
|
||||
if (bytes<5) return (CFRes){.mul=cszBytes, .fn = cfs_0_4[bytes]};
|
||||
if (bytes<8) return (CFRes){.mul=cszBytes, .fn = cf_5_7, .data=bytes-4};
|
||||
if (bytes==8) return (CFRes){.mul=cszBytes, .fn = cf_8};
|
||||
if (bytes==16) return (CFRes){.mul=cszBytes, .fn = cf_16};
|
||||
if (bytes<=16) return (CFRes){.mul=cszBytes, .fn = cf_9_16, .data=bytes-8};
|
||||
if (bytes<=24) return (CFRes){.mul=cszBytes, .fn = cf_17_24, .data=bytes-8};
|
||||
if (bytes<=32) return (CFRes){.mul=cszBytes, .fn = cf_25_32, .data=bytes-8};
|
||||
@ -173,9 +177,9 @@ B select_c2(B t, B w, B x) {
|
||||
#if SINGELI_X86_64
|
||||
#define CPUSEL(W, NEXT) /*assumes 3≤xl≤6*/ \
|
||||
if (!avx2_select_tab[4*(we-el_i8)+xl-3](wp, xp, rp, wia, xn)) thrM("⊏: Indexing out-of-bounds");
|
||||
#define BOOL_USE_SIMD (xia<=128)
|
||||
bool bool_use_simd = we==el_i8 && xl==0 && xia<=128;
|
||||
#define BOOL_SPECIAL(W) \
|
||||
if (sizeof(W)==1 && BOOL_USE_SIMD) { \
|
||||
if (sizeof(W)==1 && bool_use_simd) { \
|
||||
if (!avx2_select_bool128(wp, xp, rp, wia, xn)) thrM("⊏: Indexing out-of-bounds"); \
|
||||
goto setsh; \
|
||||
}
|
||||
@ -202,12 +206,26 @@ B select_c2(B t, B w, B x) {
|
||||
} \
|
||||
if (wt) TFREE(wt); \
|
||||
}
|
||||
#define BOOL_USE_SIMD 0
|
||||
bool bool_use_simd = 0;
|
||||
#define BOOL_SPECIAL(W)
|
||||
#endif
|
||||
|
||||
if (!(BOOL_USE_SIMD && xl==0 && we==el_i8) && xe==el_bit && wia>=256 && we!=el_bit && ((csz&7)!=0) && (xl==0? wia/4>=xia : wia>=xia/4 && csz<40)) {
|
||||
return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x)))));
|
||||
if (!bool_use_simd && xe==el_bit && (csz&7)!=0 && (xl==0? wia>=256 : wia>=4) && csz<128) {
|
||||
// test widen/narrow on bitarr input
|
||||
// ShArr* sh = RNK(x)==1? NULL : ptr_inc(shObj(x));
|
||||
// B t = select_c2(m_f64(0), w, widenBitArr(x, 1));
|
||||
// B r = narrowWidenedBitArr(t, wr, xr-1, sh==NULL? &xn : sh->a+1);
|
||||
// if (sh!=NULL) ptr_dec(sh);
|
||||
// return r;
|
||||
if (csz==1) {
|
||||
if (wia/4>=xia) return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x)))));
|
||||
} else if (csz>64? wia/2>=xn : wia>=xn/2) {
|
||||
ShArr* sh = ptr_inc(shObj(x));
|
||||
B t = select_c2(m_f64(0), w, widenBitArr(x, 1));
|
||||
B r = narrowWidenedBitArr(t, wr, xr-1, sh->a+1);
|
||||
ptr_dec(sh);
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -242,7 +260,14 @@ B select_c2(B t, B w, B x) {
|
||||
retry:
|
||||
switch (we) { default: UD;
|
||||
case el_bit: {
|
||||
if (xr!=1) goto generic_l;
|
||||
if (xr!=1) {
|
||||
if (xe!=el_B && (csz<<elWidthLogBits(xe)) < 128) {
|
||||
dec(xf);
|
||||
return select_c2(m_f64(0), taga(cpyI8Arr(w)), x);
|
||||
} else {
|
||||
goto generic_l;
|
||||
}
|
||||
}
|
||||
SGetU(x)
|
||||
B x0 = GetU(x, 0);
|
||||
B x1;
|
||||
|
||||
@ -1413,7 +1413,7 @@ B pick_ucw(B t, B o, B w, B x) {
|
||||
return qWithFill(mut_fcd(r, x), xf);
|
||||
}
|
||||
|
||||
static B takedrop_ucw(i64 wi, B o, u64 am, B x, size_t xr) {
|
||||
static B takedrop_ucw(i64 wi, B o, u64 am, B x, ux xr) {
|
||||
usz xia = IA(x);
|
||||
usz csz = arr_csz(x);
|
||||
usz tk = csz*am; // taken element count
|
||||
|
||||
@ -712,7 +712,7 @@ void g_pst(void) { vm_pstLive(); fflush(stdout); fflush(stderr); }
|
||||
NOINLINE Value* VALIDATEP(Value* x) {
|
||||
if (x->refc<=0 || (x->refc>>28) == 'a' || x->type==t_empty) {
|
||||
PRINT_ID(x);
|
||||
fprintf(stderr, "bad refcount for type %d: %d\nattempting to print: ", x->type, x->refc); fflush(stderr);
|
||||
fprintf(stderr, "bad refcount for type %d @ %p: %d\nattempting to print: ", x->type, x, x->refc); fflush(stderr);
|
||||
fprintI(stderr, tag(x,OBJ_TAG)); fputc('\n', stderr); fflush(stderr);
|
||||
err("");
|
||||
}
|
||||
|
||||
@ -108,9 +108,9 @@ static Arr* arr_shCopy(Arr* n, B o) { // copy shape & rank from o to n
|
||||
assert(IA(o)==n->ia);
|
||||
return arr_shCopyUnchecked(n, o);
|
||||
}
|
||||
static void shcpy(usz* dst, usz* src, size_t len) {
|
||||
static void shcpy(usz* dst, usz* src, ux len) {
|
||||
// memcpy(dst, src, len*sizeof(usz));
|
||||
PLAINLOOP for (size_t i = 0; i < len; i++) dst[i] = src[i];
|
||||
PLAINLOOP for (ux i = 0; i < len; i++) dst[i] = src[i];
|
||||
}
|
||||
|
||||
static usz shProd(usz* sh, usz s, usz e) {
|
||||
@ -140,6 +140,9 @@ B bit_sel(B b, B e0, B e1); // consumes b; b must be bitarr; b⊏e0‿e1
|
||||
Arr* allZeroes(usz ia);
|
||||
Arr* allOnes(usz ia);
|
||||
B bit_negate(B x); // consumes
|
||||
B widenBitArr(B x, ur axis); // consumes x, assumes bitarr; returns some array with cell size padded to the nearest of 8,16,32,64 if ≤64 bits, or a multiple of 64 bits otherwise
|
||||
B narrowWidenedBitArr(B x, ur axis, ur cr, usz* csh); // consumes x.val; undoes widenBitArr, overriding shape past axis to cr↑csh
|
||||
|
||||
Arr* cpyWithShape(B x); // consumes; returns array with refcount 1 with the same shape as x; to allocate a new shape in its place, the previous one needs to be freed, rank set to 1, and then shape & rank set to the new ones
|
||||
Arr* emptyArr(B x, ur xr); // doesn't consume; returns an empty array with the fill of x; if xr>1, shape is unset
|
||||
|
||||
@ -165,15 +168,9 @@ static u8 selfElType(B x) { // guaranteed to fit fill
|
||||
if (isC32(x)) return LIKELY(q_c8(x))? el_c8 : q_c16(x)? el_c16 : el_c32;
|
||||
return el_B;
|
||||
}
|
||||
static bool elChr(u8 x) {
|
||||
return x>=el_c8 && x<=el_c32;
|
||||
}
|
||||
static bool elNum(u8 x) {
|
||||
return x<=el_f64;
|
||||
}
|
||||
static bool elInt(u8 x) {
|
||||
return x<=el_i32;
|
||||
}
|
||||
static bool elChr(u8 x) { return x>=el_c8 && x<=el_c32; }
|
||||
static bool elNum(u8 x) { return x<=el_f64; }
|
||||
static bool elInt(u8 x) { return x<=el_i32; }
|
||||
|
||||
// string stuff
|
||||
|
||||
|
||||
6
src/h.h
6
src/h.h
@ -114,6 +114,12 @@ typedef uint32_t u32;
|
||||
typedef int64_t i64;
|
||||
typedef uint64_t u64;
|
||||
typedef double f64;
|
||||
typedef size_t ux;
|
||||
#if defined(__SIZEOF_INT128__)
|
||||
#define HAS_U128 1
|
||||
typedef unsigned __int128 u128;
|
||||
typedef signed __int128 i128;
|
||||
#endif
|
||||
#define I8_MIN -128
|
||||
#define I8_MAX 127
|
||||
#define I16_MIN -32768
|
||||
|
||||
144
src/utils/bits.c
144
src/utils/bits.c
@ -1,4 +1,5 @@
|
||||
#include "../core.h"
|
||||
#include "mut.h"
|
||||
|
||||
|
||||
#if SINGELI
|
||||
@ -78,4 +79,145 @@ NOINLINE B bit_sel(B b, B e0, B e1) {
|
||||
|
||||
dec_ret:
|
||||
decG(b); return r;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static inline u64 rbuu64(u64* p, ux off) { // read bit-unaligned u64; aka 64↑off↓p
|
||||
ux p0 = off>>6;
|
||||
ux m0 = off&63;
|
||||
u64 v0 = p[p0];
|
||||
u64 v1 = p[p0+1];
|
||||
#if HAS_U128
|
||||
u128 v = v0 | ((u128)v1)<<64;
|
||||
return v>>m0;
|
||||
#else
|
||||
return m0==0? v0 : v0>>m0 | v1<<(64-m0);
|
||||
#endif
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
u64* ptr;
|
||||
u64 tmp;
|
||||
ux off;
|
||||
} ABState;
|
||||
static ABState ab_new(u64* p) { return (ABState){.ptr=p, .tmp=0, .off=0}; }
|
||||
static void ab_done(ABState s) { if (s.off) *s.ptr = s.tmp; }
|
||||
FORCE_INLINE void ab_add(ABState* state, u64 val, ux count) { // assumes bits past count are 0
|
||||
assert(count==64 || (val>>count)==0);
|
||||
ux off0 = state->off;
|
||||
ux off1 = off0 + count;
|
||||
state->off = off1&63;
|
||||
assert((state->tmp>>off0) == 0);
|
||||
state->tmp|= val<<off0;
|
||||
if (off1>=64) {
|
||||
*state->ptr++ = state->tmp;
|
||||
state->tmp = off0==0? 0 : val>>(64-off0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE u64 ruu64(void* p) { u64 r; memcpy(&r, p, 8); return r; } // read byte-unaligned u64
|
||||
FORCE_INLINE void wuu64(void* p, u64 v) { memcpy(p, &v, 8); } // write byte-unaligned u64
|
||||
|
||||
static NOINLINE B zeroPadToCellBits0(B x, usz lr, usz cam, usz pcsz, usz ncsz) { // consumes; for now assumes ncsz is either a multiple of 64, or one of 8,16,32
|
||||
assert((ncsz&7) == 0 && RNK(x)>=1 && pcsz<ncsz);
|
||||
// printf("zeroPadToCellBits0 ia=%d cam=%d pcsz=%d ncsz=%d\n", IA(x), cam, pcsz, ncsz);
|
||||
if (pcsz==1) {
|
||||
if (ncsz== 8) return taga(cpyI8Arr(x));
|
||||
if (ncsz==16) return taga(cpyI16Arr(x));
|
||||
if (ncsz==32) return taga(cpyI32Arr(x));
|
||||
}
|
||||
|
||||
u64* rp;
|
||||
Arr* r = m_bitarrp(&rp, cam*ncsz);
|
||||
usz* rsh = arr_shAlloc(r, lr+1);
|
||||
shcpy(rsh, SH(x), lr);
|
||||
rsh[lr] = ncsz;
|
||||
u64* xp = tyany_ptr(x);
|
||||
|
||||
// TODO widen 8/16-bit cells to 16/32 via cpyC(16|32)Arr
|
||||
if (ncsz<=64 && (ncsz&(ncsz-1)) == 0) {
|
||||
u64 msk = (1ull<<pcsz)-1;
|
||||
switch(ncsz) { default: UD;
|
||||
case 8: for (ux i=0; i<cam; i++) ((u8* )rp)[i] = rbuu64(xp, i*pcsz)&msk; break;
|
||||
case 16: for (ux i=0; i<cam; i++) ((u16*)rp)[i] = rbuu64(xp, i*pcsz)&msk; break;
|
||||
case 32: for (ux i=0; i<cam; i++) ((u32*)rp)[i] = rbuu64(xp, i*pcsz)&msk; break;
|
||||
case 64: for (ux i=0; i<cam; i++) ((u64*)rp)[i] = rbuu64(xp, i*pcsz)&msk; break;
|
||||
}
|
||||
} else {
|
||||
assert((ncsz&63) == 0 && ncsz-pcsz < 64 && (pcsz&63) != 0);
|
||||
ux pfu64 = pcsz>>6; // previous full u64 count in cell
|
||||
u64 msk = (1ull<<(pcsz&63))-1;
|
||||
for (ux i = 0; i < cam; i++) {
|
||||
for (ux j = 0; j < pfu64; j++) rp[j] = rbuu64(xp, i*pcsz + j*64);
|
||||
rp[pfu64] = rbuu64(xp, i*pcsz + pfu64*64) & msk;
|
||||
rp+= ncsz>>6;
|
||||
}
|
||||
}
|
||||
decG(x);
|
||||
return taga(r);
|
||||
}
|
||||
NOINLINE B widenBitArr(B x, ur axis) {
|
||||
assert(isArr(x) && TI(x,elType)==el_bit && axis>=1 && RNK(x)>=axis);
|
||||
usz pcsz = shProd(SH(x), axis, RNK(x));
|
||||
usz ncsz;
|
||||
if (pcsz<=8) ncsz = 8;
|
||||
else if (pcsz<=16) ncsz = 16;
|
||||
else if (pcsz<=32) ncsz = 32;
|
||||
// else if (pcsz<=64) ncsz = 64;
|
||||
// else ncsz = (pcsz+7)&~(usz)7;
|
||||
else ncsz = (pcsz+63)&~(usz)63;
|
||||
if (ncsz==pcsz) return x;
|
||||
|
||||
return zeroPadToCellBits0(x, axis, shProd(SH(x), 0, axis), pcsz, ncsz);
|
||||
}
|
||||
B narrowWidenedBitArr(B x, ur axis, ur cr, usz* csh) { // for now assumes the bits to be dropped are zero, origCellBits is a multiple of 8, and that there's at most 63 padding bits
|
||||
if (TI(x,elType)!=el_bit) return taga(cpyBitArr(x));
|
||||
|
||||
usz xcsz = shProd(SH(x), axis, RNK(x));
|
||||
usz ocsz = shProd(csh, 0, cr);
|
||||
// printf("narrowWidenedBitArr ia=%d axis=%d cr=%d ocsz=%d xcsz=%d\n", IA(x), axis, cr, ocsz, xcsz);
|
||||
assert((xcsz&7) == 0 && ocsz<xcsz);
|
||||
if (xcsz==ocsz) {
|
||||
if (RNK(x)-axis == cr && eqShPart(SH(x)+axis, csh, cr)) return x;
|
||||
Arr* r = cpyWithShape(x);
|
||||
ShArr* rsh = m_shArr(axis+cr);
|
||||
shcpy(rsh->a, SH(x), axis);
|
||||
shcpy(rsh->a+axis, csh, cr);
|
||||
arr_shReplace(r, axis+cr, rsh);
|
||||
return taga(r);
|
||||
}
|
||||
|
||||
|
||||
usz cam = shProd(SH(x), 0, axis);
|
||||
u64* rp;
|
||||
Arr* r = m_bitarrp(&rp, cam*ocsz);
|
||||
usz* rsh = arr_shAlloc(r, axis+cr);
|
||||
shcpy(rsh, SH(x), axis);
|
||||
shcpy(rsh+axis, csh, cr);
|
||||
|
||||
u8* xp = tyany_ptr(x);
|
||||
// FILL_TO(rp, el_bit, 0, m_f64(1), PIA(r));
|
||||
ABState ab = ab_new(rp);
|
||||
if (xcsz<=64 && (xcsz&(xcsz-1)) == 0) {
|
||||
switch(xcsz) { default: UD;
|
||||
case 8: for (ux i=0; i<cam; i++) ab_add(&ab, ((u8* )xp)[i], ocsz); break;
|
||||
case 16: for (ux i=0; i<cam; i++) ab_add(&ab, ((u16*)xp)[i], ocsz); break;
|
||||
case 32: for (ux i=0; i<cam; i++) ab_add(&ab, ((u32*)xp)[i], ocsz); break;
|
||||
case 64: for (ux i=0; i<cam; i++) ab_add(&ab, ((u64*)xp)[i], ocsz); break;
|
||||
}
|
||||
} else {
|
||||
assert(xcsz-ocsz<64);
|
||||
ux rfu64 = ocsz>>6; // full u64 count per cell in x
|
||||
u64 msk = (1ull<<(ocsz&63))-1;
|
||||
for (ux i = 0; i < cam; i++) {
|
||||
for (ux j = 0; j < rfu64; j++) ab_add(&ab, ruu64(j + (u64*)xp), 64);
|
||||
ab_add(&ab, ruu64(rfu64 + (u64*)xp)&msk, ocsz&63);
|
||||
rp+= ocsz>>6;
|
||||
xp+= xcsz>>3;
|
||||
}
|
||||
}
|
||||
ab_done(ab);
|
||||
decG(x);
|
||||
return taga(r);
|
||||
}
|
||||
|
||||
2
src/vm.c
2
src/vm.c
@ -1008,7 +1008,7 @@ B md1Bl_c2(Md1D* d, B w, B x) { Md1Block* b=(Md1Block*)d->m1; ptr_inc(d); return
|
||||
B md2Bl_c1(Md2D* d, B x) { Md2Block* b=(Md2Block*)d->m2; ptr_inc(d); return execBlock(b->bl, b->bl->bodies[0], b->sc, 6, (B[]){tag(d,FUN_TAG), x, bi_N, tag(ptr_inc(d->m2),MD2_TAG), inc(d->f), inc(d->g)}); }
|
||||
B md2Bl_c2(Md2D* d, B w, B x) { Md2Block* b=(Md2Block*)d->m2; ptr_inc(d); return execBlock(b->bl, b->bl->dyBody, b->sc, 6, (B[]){tag(d,FUN_TAG), x, w , tag(ptr_inc(d->m2),MD2_TAG), inc(d->f), inc(d->g)}); }
|
||||
|
||||
NORETURN NOINLINE static void noInv(Body* bo, Scope* psc, i8 type, i8 inv) {
|
||||
static NOINLINE NORETURN void noInv(Body* bo, Scope* psc, i8 type, i8 inv) {
|
||||
pushEnv(m_scope(bo, psc, 0, 0, (B[]){}), bo->bc);
|
||||
thrF("No %U undo header found for this%U block", inv==0? "monadic" : inv==1? "dyadic F˜⁼" : "dyadic F⁼", type==0? "" : type==1? " 1-modifier" : " 2-modifier");
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user