diff --git a/src/builtins/group.c b/src/builtins/group.c index 9fdd54b2..866485ab 100644 --- a/src/builtins/group.c +++ b/src/builtins/group.c @@ -163,7 +163,7 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) { GROUP_CHUNKED(MEM_CPY) } else { allocBitGroups(rp, ria, z, xr, xsh, len, width); - GROUP_CHUNKED(bit_cpy) + GROUP_CHUNKED(bit_cpyN) } #undef GROUP_CHUNKED decG(ind); @@ -202,8 +202,8 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) { if (xr==1) GROUP_SORT(MEM_CPY, m_tyarrv(rp+j, width, l, xt)) else GROUP_SORT(MEM_CPY, rp[j] = m_shChangeLen(xt, xr, xsh, l, width, csz)) } else { - if (xr==1) GROUP_SORT(bit_cpy, rp[j] = taga(arr_shVec(m_bitarr_nop(l)))) - else GROUP_SORT(bit_cpy, rp[j] = taga(arr_shChangeLen(m_bitarr_nop(l*width), xr, xsh, l))) + if (xr==1) GROUP_SORT(bit_cpyN, rp[j] = taga(arr_shVec(m_bitarr_nop(l)))) + else GROUP_SORT(bit_cpyN, rp[j] = taga(arr_shChangeLen(m_bitarr_nop(l*width), xr, xsh, l))) } #undef GROUP_SORT } else if (notB && xk <= 3) { // Cells of 𝕩 are CPU-sized diff --git a/src/builtins/internal.c b/src/builtins/internal.c index 61ebebda..65e14777 100644 --- a/src/builtins/internal.c +++ b/src/builtins/internal.c @@ -286,7 +286,7 @@ B internalTemp_c1(B t, B x) { #endif #ifdef TEST_BITCPY SGetU(x) - bit_cpy(bitarr_ptr(GetU(x,0)), o2s(GetU(x,1)), bitarr_ptr(GetU(x,2)), o2s(GetU(x,3)), o2s(GetU(x,4))); + bit_cpyN(bitarr_ptr(GetU(x,0)), o2s(GetU(x,1)), bitarr_ptr(GetU(x,2)), o2s(GetU(x,3)), o2s(GetU(x,4))); #endif return x; } diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index 094ac2d1..36bef070 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -259,11 +259,11 @@ B shape_c2(B t, B w, B x) { } for (; b%8; b*=2) { if (b>nw*32) { - if (b=bf since bf is rounded up break; } - bit_cpy(rq, b, rq, 0, b); + bit_cpyN(rq, b, rq, 0, b); } } else { memcpy(rp, xp, b/8); diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 7b90386f..9d127312 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -217,7 +217,7 @@ static B compress_grouped(u64* wp, B x, usz wia, usz wsum, u8 xt) { usz width = csz; u64* xp = tyany_ptr(x); u64* rp; r = m_bitarrv(&rp,wsum*width); a(r)->ia = wsum; - COMPRESS_GROUP(bit_cpy) + COMPRESS_GROUP(bit_cpyN) } return r; } diff --git a/src/builtins/transpose.c b/src/builtins/transpose.c index e57276bd..7d718a17 100644 --- a/src/builtins/transpose.c +++ b/src/builtins/transpose.c @@ -104,7 +104,7 @@ static Arr* transpose_noshape(B* px, usz ia, usz w, usz h) { ((u32*)r0)[i] = _pext_u64(v, 0x5555555555555555); ((u32*)r1)[i] = _pext_u64(v, 0xAAAAAAAAAAAAAAAA); } - bit_cpy(r0, h, r1, 0, h); + bit_cpyN(r0, h, r1, 0, h); TFREE(r1); } else #endif diff --git a/src/core/arrFns.h b/src/core/arrFns.h index a83f8f62..4c7062b1 100644 --- a/src/core/arrFns.h +++ b/src/core/arrFns.h @@ -32,22 +32,22 @@ static void* tyany_ptr(B x) { assert(IS_ARR(TY(x)) || IS_SLICE(TY(x))); return ((TyArr*)r)->a; \ } // width in bytes; overalloc is a byte count -static void* m_tyarrp (Arr** rp, usz w, usz ia, u8 type ) M_TYARR(0, , , r, ) -static void* m_tyarrpO(Arr** rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, , r, ) -static void* m_tyarrv (B* rp, usz w, usz ia, u8 type ) M_TYARR(0, , arr_shVec((Arr*)r);, taga(r), ) -static void* m_tyarrvO(B* rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, arr_shVec((Arr*)r);, taga(r), ) -static void* m_tyarrc (B* rp, usz w, B x, u8 type ) M_TYARR(0, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);) -static void* m_tyarrcO(B* rp, usz w, B x, u8 type, usz over) M_TYARR(0,+over, arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);) +SHOULD_INLINE void* m_tyarrp (Arr** rp, usz w, usz ia, u8 type ) M_TYARR(0, , , r, ) +SHOULD_INLINE void* m_tyarrpO(Arr** rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, , r, ) +SHOULD_INLINE void* m_tyarrv (B* rp, usz w, usz ia, u8 type ) M_TYARR(0, , arr_shVec((Arr*)r);, taga(r), ) +SHOULD_INLINE void* m_tyarrvO(B* rp, usz w, usz ia, u8 type, usz over) M_TYARR(0,+over, arr_shVec((Arr*)r);, taga(r), ) +SHOULD_INLINE void* m_tyarrc (B* rp, usz w, B x, u8 type ) M_TYARR(0, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);) +SHOULD_INLINE void* m_tyarrcO(B* rp, usz w, B x, u8 type, usz over) M_TYARR(0,+over, arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);) // width in log2(bytes) -static void* m_tyarrlp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(1, , , r, ) -static void* m_tyarrlv(B* rp, usz w, usz ia, u8 type) M_TYARR(1, , arr_shVec((Arr*)r);, taga(r), ) -static void* m_tyarrlc(B* rp, usz w, B x, u8 type) M_TYARR(1, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);) +SHOULD_INLINE void* m_tyarrlp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(1, , , r, ) +SHOULD_INLINE void* m_tyarrlv(B* rp, usz w, usz ia, u8 type) M_TYARR(1, , arr_shVec((Arr*)r);, taga(r), ) +SHOULD_INLINE void* m_tyarrlc(B* rp, usz w, B x, u8 type) M_TYARR(1, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);) // width in log2(bits) -static void* m_tyarrlbp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(2, , , r, ) -static void* m_tyarrlbv(B* rp, usz w, usz ia, u8 type) M_TYARR(2, , arr_shVec((Arr*)r);, taga(r), ) -static void* m_tyarrlbc(B* rp, usz w, B x, u8 type) M_TYARR(2, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);) +SHOULD_INLINE void* m_tyarrlbp(Arr** rp, usz w, usz ia, u8 type) M_TYARR(2, , , r, ) +SHOULD_INLINE void* m_tyarrlbv(B* rp, usz w, usz ia, u8 type) M_TYARR(2, , arr_shVec((Arr*)r);, taga(r), ) +SHOULD_INLINE void* m_tyarrlbc(B* rp, usz w, B x, u8 type) M_TYARR(2, , arr_shCopy((Arr*)r,x);, taga(r), usz ia = IA(x);) extern u8 elType2type[]; #define el2t(X) elType2type[X] // TODO maybe reorganize array types such that this can just be addition? @@ -62,7 +62,7 @@ extern u8 arrTypeBitsLog[]; #define arrNewType(X) el2t(TIi(X,elType)) // Log of width in bits: max of 7, and also return 7 if not power of 2 -static u8 cellWidthLog(B x) { +SHOULD_INLINE u8 cellWidthLog(B x) { assert(isArr(x) && RNK(x)>=1); u8 lw = arrTypeBitsLog(TY(x)); if (LIKELY(RNK(x)==1)) return lw; diff --git a/src/core/harr.h b/src/core/harr.h index dbb27cb0..6fcb73f5 100644 --- a/src/core/harr.h +++ b/src/core/harr.h @@ -28,7 +28,7 @@ NOINLINE void harr_pfree(B x, usz am); // am - item after last written #define HARR_ADDA(N, V) ({ B v_ = (V); N##_v.a[N##_i] = v_; *N##_ia = ++N##_i; v_; }) #define HARR_O(N) N##_v #define HARR_I(N) N##_i -static HArr_p m_harr_impl(usz ia) { +SHOULD_INLINE HArr_p m_harr_impl(usz ia) { CHECK_IA(ia, sizeof(B)); HArr* r = m_arr(fsizeof(HArr,a,B,ia), t_harrPartial, ia); r->ia = 0; @@ -43,27 +43,27 @@ static HArr_p m_harr_impl(usz ia) { #define HARR_FCD(N, X) ({ assert(N##_v.c->ia == N##_len); harr_fcd_impl(N##_v, X); }) #define HARR_FA(N, R) ({ assert(N##_v.c->ia == N##_len); harr_fa_impl(N##_v, R); }) #define HARR_ABANDON(N) harr_abandon_impl(N##_v.c) -static B harr_fv_impl(HArr_p p) { VTY(p.b, t_harrPartial); +SHOULD_INLINE B harr_fv_impl(HArr_p p) { VTY(p.b, t_harrPartial); p.c->type = t_harr; p.c->sh = &p.c->ia; SRNK(p.b, 1); gsPop(); return p.b; } -static B harr_fc_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial); +SHOULD_INLINE B harr_fc_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial); p.c->type = t_harr; arr_shCopy((Arr*)p.c, x); gsPop(); return p.b; } -static B harr_fcd_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial); +SHOULD_INLINE B harr_fcd_impl(HArr_p p, B x) { VTY(p.b, t_harrPartial); p.c->type = t_harr; arr_shCopy((Arr*)p.c, x); decG(x); gsPop(); return p.b; } -static usz* harr_fa_impl(HArr_p p, ur r) { VTY(p.b, t_harrPartial); +SHOULD_INLINE usz* harr_fa_impl(HArr_p p, ur r) { VTY(p.b, t_harrPartial); p.c->type = t_harr; gsPop(); return arr_shAlloc((Arr*)p.c, r); @@ -75,20 +75,20 @@ void harr_abandon_impl(HArr* p); #define m_harr0v(N) ({ usz n_ = (N); HArr_p r_ = m_harrUv(n_); for(usz i=0;ia; return taga(r); } -static B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x)); +SHOULD_INLINE B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x)); usz ia = IA(x); CHECK_IA(ia, sizeof(TEl)); TyArr* r = m_arr(TYARR_SZ2(TU,ia), T_ARR, IA(x)); @@ -19,7 +19,7 @@ static B TP(m_,arrc) (TEl** p, B x) { assert(isArr(x)); arr_shCopy((Arr*)r, x); return taga(r); } -static Arr* TP(m_,arrp) (TEl** p, usz ia) { +SHOULD_INLINE Arr* TP(m_,arrp) (TEl** p, usz ia) { CHECK_IA(ia, sizeof(TEl)); TyArr* r = m_arr(TYARR_SZ2(TU,ia), T_ARR, ia); *p = (TEl*)r->a; diff --git a/src/h.h b/src/h.h index 77f8d085..cfd7768f 100644 --- a/src/h.h +++ b/src/h.h @@ -134,6 +134,11 @@ typedef size_t ux; #define NOINLINE __attribute__((noinline)) #define FORCE_INLINE __attribute__((always_inline)) static inline #define NORETURN __attribute__((noreturn)) +#if DEBUG + #define SHOULD_INLINE static +#else + #define SHOULD_INLINE FORCE_INLINE +#endif #define AUTO __auto_type #define CLZ(X) __builtin_clzll(X) #define CTZ(X) __builtin_ctzll(X) diff --git a/src/utils/mut.c b/src/utils/mut.c index 881781a0..ad419fd6 100644 --- a/src/utils/mut.c +++ b/src/utils/mut.c @@ -175,8 +175,11 @@ DEF_G(void, fill, B , (void* a, usz ms, B x, usz l), ms, x, l) { #define BIT_COPY(T) for (usz i = 0; i < l; i++) rp[i] = bitp_get(xp, xs+i); return; #define PTR_COPY(X, R) for (usz i = 0; i < l; i++) ((R*)rp)[i] = ((X*)xp)[i+xs]; return; +void bit_cpyN(u64* r, usz rs, u64* x, usz xs, usz l) { + bit_cpy(r, rs, x, xs, l); +} -DEF_COPY(bit, { bit_cpy((u64*)a, ms, bitarr_ptr(x), xs, l); return; }) +DEF_COPY(bit, { bit_cpyN((u64*)a, ms, bitarr_ptr(x), xs, l); return; }) DEF_COPY(i8 , { i8* rp = ms+(i8*)a; void* xp = tyany_ptr(x); switch (xt) { default: UD; case t_bitarr: BIT_COPY(i8) @@ -319,7 +322,7 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x TCOPY_FN(u32,c32, 0) TCOPY_FN(f64,f64, 1) static void m_copyG_bit(void* a, usz ms, B x, usz xs, usz l) { - bit_cpy((u64*)a, ms, bitarr_ptr(x), xs, l); + bit_cpyN((u64*)a, ms, bitarr_ptr(x), xs, l); } #else #define MAKE_ICPY(T,E) Arr* cpy##T##Arr(B x) { \ diff --git a/src/utils/mut.h b/src/utils/mut.h index 12b9242e..aa016432 100644 --- a/src/utils/mut.h +++ b/src/utils/mut.h @@ -113,7 +113,8 @@ static void mut_copy(Mut* m, usz ms, B x, usz xs, usz l) { assert(isArr(x)); m-> // Companion to bit_cpy when uniform syntax is wanted #define MEM_CPY(R,RI,X,XI,L) memcpy((u8*)(R)+(RI), (u8*)(X)+(XI), (L)) -static void bit_cpy(u64* r, usz rs, u64* x, usz xs, usz l) { +void bit_cpyN(u64* r, usz rs, u64* x, usz xs, usz l); +SHOULD_INLINE void bit_cpy(u64* r, usz rs, u64* x, usz xs, usz l) { u64 re = rs+(u64)l; i64 d = (i64)xs-(i64)rs;