more explicit inline requiring
This commit is contained in:
parent
8b05cd243f
commit
0aca470931
@ -163,7 +163,7 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) {
|
||||
GROUP_CHUNKED(MEM_CPY)
|
||||
} else {
|
||||
allocBitGroups(rp, ria, z, xr, xsh, len, width);
|
||||
GROUP_CHUNKED(bit_cpy)
|
||||
GROUP_CHUNKED(bit_cpyN)
|
||||
}
|
||||
#undef GROUP_CHUNKED
|
||||
decG(ind);
|
||||
@ -202,8 +202,8 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) {
|
||||
if (xr==1) GROUP_SORT(MEM_CPY, m_tyarrv(rp+j, width, l, xt))
|
||||
else GROUP_SORT(MEM_CPY, rp[j] = m_shChangeLen(xt, xr, xsh, l, width, csz))
|
||||
} else {
|
||||
if (xr==1) GROUP_SORT(bit_cpy, rp[j] = taga(arr_shVec(m_bitarr_nop(l))))
|
||||
else GROUP_SORT(bit_cpy, rp[j] = taga(arr_shChangeLen(m_bitarr_nop(l*width), xr, xsh, l)))
|
||||
if (xr==1) GROUP_SORT(bit_cpyN, rp[j] = taga(arr_shVec(m_bitarr_nop(l))))
|
||||
else GROUP_SORT(bit_cpyN, rp[j] = taga(arr_shChangeLen(m_bitarr_nop(l*width), xr, xsh, l)))
|
||||
}
|
||||
#undef GROUP_SORT
|
||||
} else if (notB && xk <= 3) { // Cells of 𝕩 are CPU-sized
|
||||
|
||||
@ -286,7 +286,7 @@ B internalTemp_c1(B t, B x) {
|
||||
#endif
|
||||
#ifdef TEST_BITCPY
|
||||
SGetU(x)
|
||||
bit_cpy(bitarr_ptr(GetU(x,0)), o2s(GetU(x,1)), bitarr_ptr(GetU(x,2)), o2s(GetU(x,3)), o2s(GetU(x,4)));
|
||||
bit_cpyN(bitarr_ptr(GetU(x,0)), o2s(GetU(x,1)), bitarr_ptr(GetU(x,2)), o2s(GetU(x,3)), o2s(GetU(x,4)));
|
||||
#endif
|
||||
return x;
|
||||
}
|
||||
|
||||
@ -259,11 +259,11 @@ B shape_c2(B t, B w, B x) {
|
||||
}
|
||||
for (; b%8; b*=2) {
|
||||
if (b>nw*32) {
|
||||
if (b<nia) bit_cpy(rq, b, rq, 0, nia-b);
|
||||
if (b<nia) bit_cpyN(rq, b, rq, 0, nia-b);
|
||||
b = 64*nw; // Ensure bi>=bf since bf is rounded up
|
||||
break;
|
||||
}
|
||||
bit_cpy(rq, b, rq, 0, b);
|
||||
bit_cpyN(rq, b, rq, 0, b);
|
||||
}
|
||||
} else {
|
||||
memcpy(rp, xp, b/8);
|
||||
|
||||
@ -217,7 +217,7 @@ static B compress_grouped(u64* wp, B x, usz wia, usz wsum, u8 xt) {
|
||||
usz width = csz;
|
||||
u64* xp = tyany_ptr(x);
|
||||
u64* rp; r = m_bitarrv(&rp,wsum*width); a(r)->ia = wsum;
|
||||
COMPRESS_GROUP(bit_cpy)
|
||||
COMPRESS_GROUP(bit_cpyN)
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
@ -104,7 +104,7 @@ static Arr* transpose_noshape(B* px, usz ia, usz w, usz h) {
|
||||
((u32*)r0)[i] = _pext_u64(v, 0x5555555555555555);
|
||||
((u32*)r1)[i] = _pext_u64(v, 0xAAAAAAAAAAAAAAAA);
|
||||
}
|
||||
bit_cpy(r0, h, r1, 0, h);
|
||||
bit_cpyN(r0, h, r1, 0, h);
|
||||
TFREE(r1);
|
||||
} else
|
||||
#endif
|
||||
|
||||
@ -32,22 +32,22 @@ static void* tyany_ptr(B x) { assert(IS_ARR(TY(x)) || IS_SLICE(TY(x)));
|
||||
return ((TyArr*)r)->a; \
|
||||
}
|
||||
// width in bytes; overalloc is a byte count
|
||||
static void* m_tyarrp (Arr** rp, usz w, usz ia, u8 type ) M_TYARR(0, , , r, )
|
||||
static void* m_tyarrpO(Arr** rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, , r, )
|
||||
static void* m_tyarrv (B* rp, usz w, usz ia, u8 type ) M_TYARR(0, , arr_shVec((Arr*)r);, taga(r), )
|
||||
static void* m_tyarrvO(B* rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, arr_shVec((Arr*)r);, taga(r), )
|
||||
static void* m_tyarrc (B* rp, usz w, B x, u8 type ) M_TYARR(0, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
|
||||
static void* m_tyarrcO(B* rp, usz w, B x, u8 type, usz over) M_TYARR(0,+over, arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
|
||||
SHOULD_INLINE void* m_tyarrp (Arr** rp, usz w, usz ia, u8 type ) M_TYARR(0, , , r, )
|
||||
SHOULD_INLINE void* m_tyarrpO(Arr** rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, , r, )
|
||||
SHOULD_INLINE void* m_tyarrv (B* rp, usz w, usz ia, u8 type ) M_TYARR(0, , arr_shVec((Arr*)r);, taga(r), )
|
||||
SHOULD_INLINE void* m_tyarrvO(B* rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, arr_shVec((Arr*)r);, taga(r), )
|
||||
SHOULD_INLINE void* m_tyarrc (B* rp, usz w, B x, u8 type ) M_TYARR(0, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
|
||||
SHOULD_INLINE void* m_tyarrcO(B* rp, usz w, B x, u8 type, usz over) M_TYARR(0,+over, arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
|
||||
|
||||
// width in log2(bytes)
|
||||
static void* m_tyarrlp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(1, , , r, )
|
||||
static void* m_tyarrlv(B* rp, usz w, usz ia, u8 type) M_TYARR(1, , arr_shVec((Arr*)r);, taga(r), )
|
||||
static void* m_tyarrlc(B* rp, usz w, B x, u8 type) M_TYARR(1, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
|
||||
SHOULD_INLINE void* m_tyarrlp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(1, , , r, )
|
||||
SHOULD_INLINE void* m_tyarrlv(B* rp, usz w, usz ia, u8 type) M_TYARR(1, , arr_shVec((Arr*)r);, taga(r), )
|
||||
SHOULD_INLINE void* m_tyarrlc(B* rp, usz w, B x, u8 type) M_TYARR(1, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
|
||||
|
||||
// width in log2(bits)
|
||||
static void* m_tyarrlbp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(2, , , r, )
|
||||
static void* m_tyarrlbv(B* rp, usz w, usz ia, u8 type) M_TYARR(2, , arr_shVec((Arr*)r);, taga(r), )
|
||||
static void* m_tyarrlbc(B* rp, usz w, B x, u8 type) M_TYARR(2, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
|
||||
SHOULD_INLINE void* m_tyarrlbp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(2, , , r, )
|
||||
SHOULD_INLINE void* m_tyarrlbv(B* rp, usz w, usz ia, u8 type) M_TYARR(2, , arr_shVec((Arr*)r);, taga(r), )
|
||||
SHOULD_INLINE void* m_tyarrlbc(B* rp, usz w, B x, u8 type) M_TYARR(2, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);)
|
||||
|
||||
extern u8 elType2type[];
|
||||
#define el2t(X) elType2type[X] // TODO maybe reorganize array types such that this can just be addition?
|
||||
@ -62,7 +62,7 @@ extern u8 arrTypeBitsLog[];
|
||||
#define arrNewType(X) el2t(TIi(X,elType))
|
||||
|
||||
// Log of width in bits: max of 7, and also return 7 if not power of 2
|
||||
static u8 cellWidthLog(B x) {
|
||||
SHOULD_INLINE u8 cellWidthLog(B x) {
|
||||
assert(isArr(x) && RNK(x)>=1);
|
||||
u8 lw = arrTypeBitsLog(TY(x));
|
||||
if (LIKELY(RNK(x)==1)) return lw;
|
||||
|
||||
@ -28,7 +28,7 @@ NOINLINE void harr_pfree(B x, usz am); // am - item after last written
|
||||
#define HARR_ADDA(N, V) ({ B v_ = (V); N##_v.a[N##_i] = v_; *N##_ia = ++N##_i; v_; })
|
||||
#define HARR_O(N) N##_v
|
||||
#define HARR_I(N) N##_i
|
||||
static HArr_p m_harr_impl(usz ia) {
|
||||
SHOULD_INLINE HArr_p m_harr_impl(usz ia) {
|
||||
CHECK_IA(ia, sizeof(B));
|
||||
HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harrPartial, ia);
|
||||
r->ia = 0;
|
||||
@ -43,27 +43,27 @@ static HArr_p m_harr_impl(usz ia) {
|
||||
#define HARR_FCD(N, X) ({ assert(N##_v.c->ia == N##_len); harr_fcd_impl(N##_v, X); })
|
||||
#define HARR_FA(N, R) ({ assert(N##_v.c->ia == N##_len); harr_fa_impl(N##_v, R); })
|
||||
#define HARR_ABANDON(N) harr_abandon_impl(N##_v.c)
|
||||
static B harr_fv_impl(HArr_p p) { VTY(p.b, t_harrPartial);
|
||||
SHOULD_INLINE B harr_fv_impl(HArr_p p) { VTY(p.b, t_harrPartial);
|
||||
p.c->type = t_harr;
|
||||
p.c->sh = &p.c->ia;
|
||||
SRNK(p.b, 1);
|
||||
gsPop();
|
||||
return p.b;
|
||||
}
|
||||
static B harr_fc_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial);
|
||||
SHOULD_INLINE B harr_fc_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial);
|
||||
p.c->type = t_harr;
|
||||
arr_shCopy((Arr*)p.c, x);
|
||||
gsPop();
|
||||
return p.b;
|
||||
}
|
||||
static B harr_fcd_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial);
|
||||
SHOULD_INLINE B harr_fcd_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial);
|
||||
p.c->type = t_harr;
|
||||
arr_shCopy((Arr*)p.c, x);
|
||||
decG(x);
|
||||
gsPop();
|
||||
return p.b;
|
||||
}
|
||||
static usz* harr_fa_impl(HArr_p p, ur r) { VTY(p.b, t_harrPartial);
|
||||
SHOULD_INLINE usz* harr_fa_impl(HArr_p p, ur r) { VTY(p.b, t_harrPartial);
|
||||
p.c->type = t_harr;
|
||||
gsPop();
|
||||
return arr_shAlloc((Arr*)p.c, r);
|
||||
@ -75,20 +75,20 @@ void harr_abandon_impl(HArr* p);
|
||||
#define m_harr0v(N) ({ usz n_ = (N); HArr_p r_ = m_harrUv(n_); for(usz i=0;i<n_;i++)r_.a[i]=m_f64(0); NOGC_E; r_; })
|
||||
#define m_harr0c(X) ({ B x_ = (X); usz n_ = IA(x_); HArr_p r_ = m_harrUc(x_); for(usz i=0;i<n_;i++)r_.a[i]=m_f64(0); NOGC_E; r_; })
|
||||
#define m_harr0p(N) ({ usz n_ = (N); HArr_p r_ = m_harrUp(n_); for(usz i=0;i<n_;i++)r_.a[i]=m_f64(0); NOGC_E; r_; })
|
||||
static HArr_p m_harrUv(usz ia) {
|
||||
SHOULD_INLINE HArr_p m_harrUv(usz ia) {
|
||||
CHECK_IA(ia, sizeof(B));
|
||||
HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harr, ia); if(ia) NOGC_S;
|
||||
arr_shVec((Arr*)r);
|
||||
return harrP_parts(r);
|
||||
}
|
||||
static HArr_p m_harrUc(B x) { assert(isArr(x));
|
||||
SHOULD_INLINE HArr_p m_harrUc(B x) { assert(isArr(x));
|
||||
usz ia = IA(x);
|
||||
CHECK_IA(ia, sizeof(B));
|
||||
HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harr, ia); if(ia) NOGC_S;
|
||||
arr_shCopy((Arr*)r, x);
|
||||
return harrP_parts(r);
|
||||
}
|
||||
static HArr_p m_harrUp(usz ia) {
|
||||
SHOULD_INLINE HArr_p m_harrUp(usz ia) {
|
||||
CHECK_IA(ia, sizeof(B));
|
||||
HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harr, ia); if(ia) NOGC_S;
|
||||
return harrP_parts(r);
|
||||
|
||||
@ -4,14 +4,14 @@
|
||||
typedef TyArr JOIN(TU,Arr);
|
||||
typedef TP(,) JOIN(TU,Atom);
|
||||
|
||||
static B TP(m_,arrv) (TEl** p, usz ia) {
|
||||
SHOULD_INLINE B TP(m_,arrv) (TEl** p, usz ia) {
|
||||
CHECK_IA(ia, sizeof(TEl));
|
||||
TyArr* r = m_arr(TYARR_SZ2(TU,ia), T_ARR, ia);
|
||||
arr_shVec((Arr*)r);
|
||||
*p = (TEl*)r->a;
|
||||
return taga(r);
|
||||
}
|
||||
static B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x));
|
||||
SHOULD_INLINE B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x));
|
||||
usz ia = IA(x);
|
||||
CHECK_IA(ia, sizeof(TEl));
|
||||
TyArr* r = m_arr(TYARR_SZ2(TU,ia), T_ARR, IA(x));
|
||||
@ -19,7 +19,7 @@ static B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x));
|
||||
arr_shCopy((Arr*)r, x);
|
||||
return taga(r);
|
||||
}
|
||||
static Arr* TP(m_,arrp) (TEl** p, usz ia) {
|
||||
SHOULD_INLINE Arr* TP(m_,arrp) (TEl** p, usz ia) {
|
||||
CHECK_IA(ia, sizeof(TEl));
|
||||
TyArr* r = m_arr(TYARR_SZ2(TU,ia), T_ARR, ia);
|
||||
*p = (TEl*)r->a;
|
||||
|
||||
5
src/h.h
5
src/h.h
@ -134,6 +134,11 @@ typedef size_t ux;
|
||||
#define NOINLINE __attribute__((noinline))
|
||||
#define FORCE_INLINE __attribute__((always_inline)) static inline
|
||||
#define NORETURN __attribute__((noreturn))
|
||||
#if DEBUG
|
||||
#define SHOULD_INLINE static
|
||||
#else
|
||||
#define SHOULD_INLINE FORCE_INLINE
|
||||
#endif
|
||||
#define AUTO __auto_type
|
||||
#define CLZ(X) __builtin_clzll(X)
|
||||
#define CTZ(X) __builtin_ctzll(X)
|
||||
|
||||
@ -175,8 +175,11 @@ DEF_G(void, fill, B , (void* a, usz ms, B x, usz l), ms, x, l) {
|
||||
#define BIT_COPY(T) for (usz i = 0; i < l; i++) rp[i] = bitp_get(xp, xs+i); return;
|
||||
#define PTR_COPY(X, R) for (usz i = 0; i < l; i++) ((R*)rp)[i] = ((X*)xp)[i+xs]; return;
|
||||
|
||||
void bit_cpyN(u64* r, usz rs, u64* x, usz xs, usz l) {
|
||||
bit_cpy(r, rs, x, xs, l);
|
||||
}
|
||||
|
||||
DEF_COPY(bit, { bit_cpy((u64*)a, ms, bitarr_ptr(x), xs, l); return; })
|
||||
DEF_COPY(bit, { bit_cpyN((u64*)a, ms, bitarr_ptr(x), xs, l); return; })
|
||||
DEF_COPY(i8 , { i8* rp = ms+(i8*)a; void* xp = tyany_ptr(x);
|
||||
switch (xt) { default: UD;
|
||||
case t_bitarr: BIT_COPY(i8)
|
||||
@ -319,7 +322,7 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
|
||||
TCOPY_FN(u32,c32, 0)
|
||||
TCOPY_FN(f64,f64, 1)
|
||||
static void m_copyG_bit(void* a, usz ms, B x, usz xs, usz l) {
|
||||
bit_cpy((u64*)a, ms, bitarr_ptr(x), xs, l);
|
||||
bit_cpyN((u64*)a, ms, bitarr_ptr(x), xs, l);
|
||||
}
|
||||
#else
|
||||
#define MAKE_ICPY(T,E) Arr* cpy##T##Arr(B x) { \
|
||||
|
||||
@ -113,7 +113,8 @@ static void mut_copy(Mut* m, usz ms, B x, usz xs, usz l) { assert(isArr(x)); m->
|
||||
// Companion to bit_cpy when uniform syntax is wanted
|
||||
#define MEM_CPY(R,RI,X,XI,L) memcpy((u8*)(R)+(RI), (u8*)(X)+(XI), (L))
|
||||
|
||||
static void bit_cpy(u64* r, usz rs, u64* x, usz xs, usz l) {
|
||||
void bit_cpyN(u64* r, usz rs, u64* x, usz xs, usz l);
|
||||
SHOULD_INLINE void bit_cpy(u64* r, usz rs, u64* x, usz xs, usz l) {
|
||||
u64 re = rs+(u64)l;
|
||||
i64 d = (i64)xs-(i64)rs;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user