more explicit inline requiring

This commit is contained in:
dzaima 2023-04-06 13:03:14 +03:00
parent 8b05cd243f
commit 0aca470931
11 changed files with 44 additions and 35 deletions

View File

@ -163,7 +163,7 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) {
GROUP_CHUNKED(MEM_CPY)
} else {
allocBitGroups(rp, ria, z, xr, xsh, len, width);
GROUP_CHUNKED(bit_cpy)
GROUP_CHUNKED(bit_cpyN)
}
#undef GROUP_CHUNKED
decG(ind);
@ -202,8 +202,8 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) {
if (xr==1) GROUP_SORT(MEM_CPY, m_tyarrv(rp+j, width, l, xt))
else GROUP_SORT(MEM_CPY, rp[j] = m_shChangeLen(xt, xr, xsh, l, width, csz))
} else {
if (xr==1) GROUP_SORT(bit_cpy, rp[j] = taga(arr_shVec(m_bitarr_nop(l))))
else GROUP_SORT(bit_cpy, rp[j] = taga(arr_shChangeLen(m_bitarr_nop(l*width), xr, xsh, l)))
if (xr==1) GROUP_SORT(bit_cpyN, rp[j] = taga(arr_shVec(m_bitarr_nop(l))))
else GROUP_SORT(bit_cpyN, rp[j] = taga(arr_shChangeLen(m_bitarr_nop(l*width), xr, xsh, l)))
}
#undef GROUP_SORT
} else if (notB && xk <= 3) { // Cells of 𝕩 are CPU-sized

View File

@ -286,7 +286,7 @@ B internalTemp_c1(B t, B x) {
#endif
#ifdef TEST_BITCPY
SGetU(x)
bit_cpy(bitarr_ptr(GetU(x,0)), o2s(GetU(x,1)), bitarr_ptr(GetU(x,2)), o2s(GetU(x,3)), o2s(GetU(x,4)));
bit_cpyN(bitarr_ptr(GetU(x,0)), o2s(GetU(x,1)), bitarr_ptr(GetU(x,2)), o2s(GetU(x,3)), o2s(GetU(x,4)));
#endif
return x;
}

View File

@ -259,11 +259,11 @@ B shape_c2(B t, B w, B x) {
}
for (; b%8; b*=2) {
if (b>nw*32) {
if (b<nia) bit_cpy(rq, b, rq, 0, nia-b);
if (b<nia) bit_cpyN(rq, b, rq, 0, nia-b);
b = 64*nw; // Ensure bi>=bf since bf is rounded up
break;
}
bit_cpy(rq, b, rq, 0, b);
bit_cpyN(rq, b, rq, 0, b);
}
} else {
memcpy(rp, xp, b/8);

View File

@ -217,7 +217,7 @@ static B compress_grouped(u64* wp, B x, usz wia, usz wsum, u8 xt) {
usz width = csz;
u64* xp = tyany_ptr(x);
u64* rp; r = m_bitarrv(&rp,wsum*width); a(r)->ia = wsum;
COMPRESS_GROUP(bit_cpy)
COMPRESS_GROUP(bit_cpyN)
}
return r;
}

View File

@ -104,7 +104,7 @@ static Arr* transpose_noshape(B* px, usz ia, usz w, usz h) {
((u32*)r0)[i] = _pext_u64(v, 0x5555555555555555);
((u32*)r1)[i] = _pext_u64(v, 0xAAAAAAAAAAAAAAAA);
}
bit_cpy(r0, h, r1, 0, h);
bit_cpyN(r0, h, r1, 0, h);
TFREE(r1);
} else
#endif

View File

@ -32,22 +32,22 @@ static void* tyany_ptr(B x) { assert(IS_ARR(TY(x)) || IS_SLICE(TY(x)));
return ((TyArr*)r)->a; \
}
// width in bytes; overalloc is a byte count
static void* m_tyarrp (Arr** rp, usz w, usz ia, u8 type ) M_TYARR(0, , , r, )
static void* m_tyarrpO(Arr** rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, , r, )
static void* m_tyarrv (B* rp, usz w, usz ia, u8 type ) M_TYARR(0, , arr_shVec((Arr*)r);, taga(r), )
static void* m_tyarrvO(B* rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, arr_shVec((Arr*)r);, taga(r), )
static void* m_tyarrc (B* rp, usz w, B x, u8 type ) M_TYARR(0, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
static void* m_tyarrcO(B* rp, usz w, B x, u8 type, usz over) M_TYARR(0,+over, arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
SHOULD_INLINE void* m_tyarrp (Arr** rp, usz w, usz ia, u8 type ) M_TYARR(0, , , r, )
SHOULD_INLINE void* m_tyarrpO(Arr** rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, , r, )
SHOULD_INLINE void* m_tyarrv (B* rp, usz w, usz ia, u8 type ) M_TYARR(0, , arr_shVec((Arr*)r);, taga(r), )
SHOULD_INLINE void* m_tyarrvO(B* rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, arr_shVec((Arr*)r);, taga(r), )
SHOULD_INLINE void* m_tyarrc (B* rp, usz w, B x, u8 type ) M_TYARR(0, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
SHOULD_INLINE void* m_tyarrcO(B* rp, usz w, B x, u8 type, usz over) M_TYARR(0,+over, arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
// width in log2(bytes)
static void* m_tyarrlp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(1, , , r, )
static void* m_tyarrlv(B* rp, usz w, usz ia, u8 type) M_TYARR(1, , arr_shVec((Arr*)r);, taga(r), )
static void* m_tyarrlc(B* rp, usz w, B x, u8 type) M_TYARR(1, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
SHOULD_INLINE void* m_tyarrlp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(1, , , r, )
SHOULD_INLINE void* m_tyarrlv(B* rp, usz w, usz ia, u8 type) M_TYARR(1, , arr_shVec((Arr*)r);, taga(r), )
SHOULD_INLINE void* m_tyarrlc(B* rp, usz w, B x, u8 type) M_TYARR(1, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
// width in log2(bits)
static void* m_tyarrlbp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(2, , , r, )
static void* m_tyarrlbv(B* rp, usz w, usz ia, u8 type) M_TYARR(2, , arr_shVec((Arr*)r);, taga(r), )
static void* m_tyarrlbc(B* rp, usz w, B x, u8 type) M_TYARR(2, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
SHOULD_INLINE void* m_tyarrlbp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(2, , , r, )
SHOULD_INLINE void* m_tyarrlbv(B* rp, usz w, usz ia, u8 type) M_TYARR(2, , arr_shVec((Arr*)r);, taga(r), )
SHOULD_INLINE void* m_tyarrlbc(B* rp, usz w, B x, u8 type) M_TYARR(2, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
extern u8 elType2type[];
#define el2t(X) elType2type[X] // TODO maybe reorganize array types such that this can just be addition?
@ -62,7 +62,7 @@ extern u8 arrTypeBitsLog[];
#define arrNewType(X) el2t(TIi(X,elType))
// Log of width in bits: max of 7, and also return 7 if not power of 2
static u8 cellWidthLog(B x) {
SHOULD_INLINE u8 cellWidthLog(B x) {
assert(isArr(x) && RNK(x)>=1);
u8 lw = arrTypeBitsLog(TY(x));
if (LIKELY(RNK(x)==1)) return lw;

View File

@ -28,7 +28,7 @@ NOINLINE void harr_pfree(B x, usz am); // am - item after last written
#define HARR_ADDA(N, V) ({ B v_ = (V); N##_v.a[N##_i] = v_; *N##_ia = ++N##_i; v_; })
#define HARR_O(N) N##_v
#define HARR_I(N) N##_i
static HArr_p m_harr_impl(usz ia) {
SHOULD_INLINE HArr_p m_harr_impl(usz ia) {
CHECK_IA(ia, sizeof(B));
HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harrPartial, ia);
r->ia = 0;
@ -43,27 +43,27 @@ static HArr_p m_harr_impl(usz ia) {
#define HARR_FCD(N, X) ({ assert(N##_v.c->ia == N##_len); harr_fcd_impl(N##_v, X); })
#define HARR_FA(N, R) ({ assert(N##_v.c->ia == N##_len); harr_fa_impl(N##_v, R); })
#define HARR_ABANDON(N) harr_abandon_impl(N##_v.c)
static B harr_fv_impl(HArr_p p) { VTY(p.b, t_harrPartial);
SHOULD_INLINE B harr_fv_impl(HArr_p p) { VTY(p.b, t_harrPartial);
p.c->type = t_harr;
p.c->sh = &p.c->ia;
SRNK(p.b, 1);
gsPop();
return p.b;
}
static B harr_fc_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial);
SHOULD_INLINE B harr_fc_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial);
p.c->type = t_harr;
arr_shCopy((Arr*)p.c, x);
gsPop();
return p.b;
}
static B harr_fcd_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial);
SHOULD_INLINE B harr_fcd_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial);
p.c->type = t_harr;
arr_shCopy((Arr*)p.c, x);
decG(x);
gsPop();
return p.b;
}
static usz* harr_fa_impl(HArr_p p, ur r) { VTY(p.b, t_harrPartial);
SHOULD_INLINE usz* harr_fa_impl(HArr_p p, ur r) { VTY(p.b, t_harrPartial);
p.c->type = t_harr;
gsPop();
return arr_shAlloc((Arr*)p.c, r);
@ -75,20 +75,20 @@ void harr_abandon_impl(HArr* p);
#define m_harr0v(N) ({ usz n_ = (N); HArr_p r_ = m_harrUv(n_); for(usz i=0;i<n_;i++)r_.a[i]=m_f64(0); NOGC_E; r_; })
#define m_harr0c(X) ({ B x_ = (X); usz n_ = IA(x_); HArr_p r_ = m_harrUc(x_); for(usz i=0;i<n_;i++)r_.a[i]=m_f64(0); NOGC_E; r_; })
#define m_harr0p(N) ({ usz n_ = (N); HArr_p r_ = m_harrUp(n_); for(usz i=0;i<n_;i++)r_.a[i]=m_f64(0); NOGC_E; r_; })
static HArr_p m_harrUv(usz ia) {
SHOULD_INLINE HArr_p m_harrUv(usz ia) {
CHECK_IA(ia, sizeof(B));
HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harr, ia); if(ia) NOGC_S;
arr_shVec((Arr*)r);
return harrP_parts(r);
}
static HArr_p m_harrUc(B x) { assert(isArr(x));
SHOULD_INLINE HArr_p m_harrUc(B x) { assert(isArr(x));
usz ia = IA(x);
CHECK_IA(ia, sizeof(B));
HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harr, ia); if(ia) NOGC_S;
arr_shCopy((Arr*)r, x);
return harrP_parts(r);
}
static HArr_p m_harrUp(usz ia) {
SHOULD_INLINE HArr_p m_harrUp(usz ia) {
CHECK_IA(ia, sizeof(B));
HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harr, ia); if(ia) NOGC_S;
return harrP_parts(r);

View File

@ -4,14 +4,14 @@
typedef TyArr JOIN(TU,Arr);
typedef TP(,) JOIN(TU,Atom);
static B TP(m_,arrv) (TEl** p, usz ia) {
SHOULD_INLINE B TP(m_,arrv) (TEl** p, usz ia) {
CHECK_IA(ia, sizeof(TEl));
TyArr* r = m_arr(TYARR_SZ2(TU,ia), T_ARR, ia);
arr_shVec((Arr*)r);
*p = (TEl*)r->a;
return taga(r);
}
static B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x));
SHOULD_INLINE B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x));
usz ia = IA(x);
CHECK_IA(ia, sizeof(TEl));
TyArr* r = m_arr(TYARR_SZ2(TU,ia), T_ARR, IA(x));
@ -19,7 +19,7 @@ static B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x));
arr_shCopy((Arr*)r, x);
return taga(r);
}
static Arr* TP(m_,arrp) (TEl** p, usz ia) {
SHOULD_INLINE Arr* TP(m_,arrp) (TEl** p, usz ia) {
CHECK_IA(ia, sizeof(TEl));
TyArr* r = m_arr(TYARR_SZ2(TU,ia), T_ARR, ia);
*p = (TEl*)r->a;

View File

@ -134,6 +134,11 @@ typedef size_t ux;
#define NOINLINE __attribute__((noinline))
#define FORCE_INLINE __attribute__((always_inline)) static inline
#define NORETURN __attribute__((noreturn))
#if DEBUG
#define SHOULD_INLINE static
#else
#define SHOULD_INLINE FORCE_INLINE
#endif
#define AUTO __auto_type
#define CLZ(X) __builtin_clzll(X)
#define CTZ(X) __builtin_ctzll(X)

View File

@ -175,8 +175,11 @@ DEF_G(void, fill, B , (void* a, usz ms, B x, usz l), ms, x, l) {
#define BIT_COPY(T) for (usz i = 0; i < l; i++) rp[i] = bitp_get(xp, xs+i); return;
#define PTR_COPY(X, R) for (usz i = 0; i < l; i++) ((R*)rp)[i] = ((X*)xp)[i+xs]; return;
void bit_cpyN(u64* r, usz rs, u64* x, usz xs, usz l) {
bit_cpy(r, rs, x, xs, l);
}
DEF_COPY(bit, { bit_cpy((u64*)a, ms, bitarr_ptr(x), xs, l); return; })
DEF_COPY(bit, { bit_cpyN((u64*)a, ms, bitarr_ptr(x), xs, l); return; })
DEF_COPY(i8 , { i8* rp = ms+(i8*)a; void* xp = tyany_ptr(x);
switch (xt) { default: UD;
case t_bitarr: BIT_COPY(i8)
@ -319,7 +322,7 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
TCOPY_FN(u32,c32, 0)
TCOPY_FN(f64,f64, 1)
static void m_copyG_bit(void* a, usz ms, B x, usz xs, usz l) {
bit_cpy((u64*)a, ms, bitarr_ptr(x), xs, l);
bit_cpyN((u64*)a, ms, bitarr_ptr(x), xs, l);
}
#else
#define MAKE_ICPY(T,E) Arr* cpy##T##Arr(B x) { \

View File

@ -113,7 +113,8 @@ static void mut_copy(Mut* m, usz ms, B x, usz xs, usz l) { assert(isArr(x)); m->
// Companion to bit_cpy when uniform syntax is wanted
#define MEM_CPY(R,RI,X,XI,L) memcpy((u8*)(R)+(RI), (u8*)(X)+(XI), (L))
static void bit_cpy(u64* r, usz rs, u64* x, usz xs, usz l) {
void bit_cpyN(u64* r, usz rs, u64* x, usz xs, usz l);
SHOULD_INLINE void bit_cpy(u64* r, usz rs, u64* x, usz xs, usz l) {
u64 re = rs+(u64)l;
i64 d = (i64)xs-(i64)rs;