diff --git a/src/builtins/slash.c b/src/builtins/slash.c index be7d9d3e..efd45fa3 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -305,6 +305,7 @@ static B where(B x, usz xia, u64 s) { #if SINGELI && defined(__BMI2__) i8* rp = m_tyarrvO(&r, 1, s, t_i8arr, 8); bmipopc_1slash8(xp, rp, xia); + REINIT_TAIL_A(r, s, 8); #else i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s,0,); #endif @@ -313,11 +314,13 @@ static B where(B x, usz xia, u64 s) { if (s >= xia/8) { i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16); bmipopc_1slash16(xp, rp, xia); + REINIT_TAIL_A(r, s*2, 16); } #else if (s >= xia/4+xia/8) { i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 2); WHERE_DENSE(xp, rp, xia, 0); + REINIT_TAIL_A(r, s*2, 2); } #endif else { @@ -362,6 +365,7 @@ static B where(B x, usz xia, u64 s) { rq+= bs; xp+= b/64; } + REINIT_TAIL_A(r, s*4, 4); TFREE(buf); } else { f64* rp; r = m_f64arrv(&rp, s); @@ -439,7 +443,8 @@ B grade_bool(B x, usz xia, bool up) { #define BMI_GRADE(W) \ i##W* rp = m_tyarrvO(&r, W/8, xia, t_i##W##arr, W); \ bmipopc_1slash##W(xp0, rp , xia); \ - bmipopc_1slash##W(xp1, rp+l0, xia); + bmipopc_1slash##W(xp1, rp+l0, xia); \ + REINIT_TAIL_A(r, xia*(W/8), W); if (xia <= 128) { BMI_GRADE(8) } else { BMI_GRADE(16) } #undef BMI_GRADE decG(notx); @@ -548,8 +553,8 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { else { DENSE; } \ break; } #if SINGELI - case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia)) - case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia)) + case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); REINIT_TAIL_A(r, wsum, 8)) + case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); REINIT_TAIL_A(r, wsum*2, 16)) #else case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; ia) = data; + REINIT_TAIL(r, offsetof(TyArr,a)+sz, offsetof(TyArr,a)+sizeof(u64)); arr_shAlloc((Arr*)r, 0); return taga(r); } diff --git a/src/core/mm.c b/src/core/mm.c index 0df400a9..be657e48 100644 --- a/src/core/mm.c +++ b/src/core/mm.c @@ -1,6 +1,4 @@ -#if ALLOC_NOINLINE - #define ALLOC_IMPL 1 -#endif +#define MM_C 1 #include "../core.h" diff --git a/src/core/stuff.h b/src/core/stuff.h index 821e3ea6..7804c1d6 100644 --- a/src/core/stuff.h +++ b/src/core/stuff.h @@ -280,7 +280,7 @@ static BBB2B c2fn(B f) { NOINLINE NORETURN void thrOOMTest(void); #endif -FORCE_INLINE void onAlloc(usz sz, u8 type) { +FORCE_INLINE void preAlloc(usz sz, u8 type) { #ifdef OOM_TEST if (--oomTestLeft==0) thrOOMTest(); #endif @@ -301,10 +301,22 @@ FORCE_INLINE void onAlloc(usz sz, u8 type) { talloc+= sz; #endif } -FORCE_INLINE void onFree(Value* x) { +#if VERIFY_TAIL +void tailVerifyAlloc(void* ptr, u64 origSz, i64 logAlloc, u8 type); +void tailVerifyFree(void* ptr); +void tailVerifyReinit(void* ptr, u64 s, u64 e); +#define REINIT_TAIL(P, S, E) tailVerifyReinit(P, S, E) +#else +#define REINIT_TAIL(P, S, E) +#endif +#define REINIT_TAIL_A(A, S, L) REINIT_TAIL(a(A), offsetof(TyArr,a)+(S), offsetof(TyArr,a)+(S)+(L)); +FORCE_INLINE void preFree(Value* x, bool mmx) { #ifdef ALLOC_STAT ctr_f[x->type]++; #endif + #if VERIFY_TAIL + if (!mmx) tailVerifyFree(x); + #endif #ifdef DEBUG if (x->type==t_empty) err("double-free"); // u32 undef; diff --git a/src/h.h b/src/h.h index b649b7d1..210ed0ee 100644 --- a/src/h.h +++ b/src/h.h @@ -101,17 +101,6 @@ #error "can't have both RT_PERF and RT_VERIFY" #endif #endif -#if defined(OBJ_TRACK) - #define OBJ_COUNTER 1 -#endif -#if ALLOC_STAT - #define ALLOC_NOINLINE -#endif -#if ALLOC_NOINLINE - #define ALLOC_FN -#else - #define ALLOC_FN static -#endif typedef int8_t i8; typedef uint8_t u8; @@ -224,6 +213,24 @@ typedef union B { #define IF_WRAP(X) #endif +#if defined(OBJ_TRACK) + #define OBJ_COUNTER 1 +#endif +#if DEBUG && !defined(VERIFY_TAIL) && MM!=2 + #define VERIFY_TAIL 64 +#endif +#if ALLOC_STAT || VERIFY_TAIL + #define ALLOC_NOINLINE 1 +#endif +#if ALLOC_NOINLINE + #if MM_C + #define ALLOC_IMPL 1 + #endif + #define ALLOC_FN +#else + #define ALLOC_FN static +#endif + #define FOR_TYPE(F) \ /* 0*/ F(empty) \ /* 1*/ F(funBI) F(funBl) \ diff --git a/src/jit/nvm_x86_64.c b/src/jit/nvm_x86_64.c index 09966de6..1139b9a9 100644 --- a/src/jit/nvm_x86_64.c +++ b/src/jit/nvm_x86_64.c @@ -15,7 +15,7 @@ #endif // objdump -b binary -m i386 -M x86-64,intel --insn-width=10 -D --adjust-vma=$(cat asm_off) asm_bin | tail -n+8 | sed "$(cat asm_sed);s/\\t/ /g;s/.*: //" -#define ALLOC_IMPL_ALWAYS 1 +#define ALLOC_IMPL_MMX 1 // separate memory management system for executable code; isn't garbage-collected EmptyValue* mmX_buckets[64]; u64 mmX_ctrs[64]; @@ -64,7 +64,7 @@ static void* mmap_nvm(u64 sz) { static void* mmX_allocN(usz sz, u8 type) { assert(sz>=16); return mmX_allocL(64-CLZ(sz-1ull), type); } #undef BN #undef BSZ -#undef ALLOC_IMPL_ALWAYS +#undef ALLOC_IMPL_MMX // all the instructions to be called by the generated code diff --git a/src/opt/mm_2buddy.c b/src/opt/mm_2buddy.c index 9863a359..0b52f37d 100644 --- a/src/opt/mm_2buddy.c +++ b/src/opt/mm_2buddy.c @@ -6,6 +6,9 @@ #ifdef OBJ_COUNTER u64 currObjCounter; #endif +#if VERIFY_TAIL + #error MM=2 doesn't support VERIFY_TAIL +#endif u64 mm_ctrs[128]; EmptyValue* mm_buckets[128]; diff --git a/src/opt/mm_2buddy.h b/src/opt/mm_2buddy.h index f85d598a..f5d55450 100644 --- a/src/opt/mm_2buddy.h +++ b/src/opt/mm_2buddy.h @@ -19,6 +19,8 @@ extern EmptyValue* mm_buckets[128]; #define LOG2(X) ((u8)(64-CLZ((X)-1ull))) + +#if !ALLOC_NOINLINE || ALLOC_IMPL || ALLOC_IMPL_MMX static void* mm_alloc(u64 sz, u8 type) { assert(sz>=16); u32 log = LOG2(sz); @@ -26,6 +28,7 @@ static void* mm_alloc(u64 sz, u8 type) { bool b2 = sz <= (3ull< VERIFY_TAIL*4) return VERIFY_TAIL*4; + return l; +} + +#define ITER_TAIL(F) { \ + u64 o = filled; u8* d = (u8*)ptr; \ + if ((o&7) && o>((o&7)*8)), o, 1); \ + } \ + for(;o+8<=end;o+=8) F(*(u64*)(d+o), ptrHash(ptr, o), o, 8); \ + if (o!=end) { \ + u64 h = ptrHash(ptr, o&~7ULL); \ + for(;o!=end;o++) F(d[o], (u8)(h>>((o&7)*8)), o, 1); \ + } \ +} + +static void tailVerifyInit(void* ptr, u64 filled, u64 end, u64 allocEnd) { + #define F(W, X, O, L) W = X + ITER_TAIL(F) + #undef F +} +void tailVerifyAlloc(void* ptr, u64 filled, i64 logAlloc, u8 type) { + u64 end = 1ULL<sz) { printf("Bad used range: "N64u".."N64u", allocation size "N64u"\n", start, end, sz); exit(1); } +} +void tailVerifyReinit(void* ptr, u64 filled, u64 end) { + if(filled<=8) { printf("Bad reinit start: "N64u".."N64u"\n", filled, end); exit(1); } + verifyEnd(ptr, mm_size(ptr), filled, end); + tailVerifyInit(ptr, filled, end, mm_size(ptr)); +} +void g_iv(void*); +NOINLINE void dumpByte(bool exp, bool has, void* ptr, u64 o) { + u8 h = ptrHash(ptr, o&~7ULL)>>((o&7)*8); + u8 m = ((u8*)ptr)[o]; + u8 v = exp? h : m; + if (!(o&7)) printf(" "); + bool col = exp? !has : has && h!=m; + if (col) printf(exp? "\x1b[38;5;240m" : "\x1b[38;5;203m"); + printf("%02x", v); + if (col) printf("\x1b[0m"); +} +NOINLINE NORETURN void tailFail(u64 got, u64 exp, void* ptr, u64 off, int len, u64 allocFilled, u64 allocTotal) { + printf("Corrupted tail @ %p + "N64d", checked length %d\n", ptr, off, len); + printf("Allocation filled with "N64d" bytes, total space "N64d"\n", allocFilled, allocTotal); + printf("Expected: x=%016"SCNx64" / u="N64u"\nGot: x=%016"SCNx64" / u="N64u"\n\n", exp, exp, got, got); + fflush(stdout); fflush(stderr); + + u64 dS = off<32? 0 : off-32; + if (dS>0) dS&= ~7ULL; + u64 dE = off+32; if (dE>allocTotal) dE = allocTotal; dE&= ~7ULL; + printf("%-+6"SCNd64,dS); for (u64 i=dS; i=allocFilled, ptr, i); printf("\n"); + printf("got: "); for (u64 i=dS; i=allocFilled, ptr, i); printf("\n"); + fflush(stdout); fflush(stderr); + + if (((Value*)ptr)->refc==0) ((Value*)ptr)->refc = 1010101009; // this allocation was just about to be freed, so refcount 0 is understandable + g_iv(ptr); + fflush(stdout); fflush(stderr); + __builtin_trap(); +} +void tailVerifyFree(void* ptr) { + u64 filled; Arr* xa = ptr; + switch(PTY(xa)) { default: return; + case t_bitarr: filled = BITARR_SZ(PIA(xa)); break; + case t_i8arr: filled = TYARR_SZ(I8, PIA(xa)); break; + case t_i16arr: filled = TYARR_SZ(I16, PIA(xa)); break; + case t_i32arr: filled = TYARR_SZ(I32, PIA(xa)); break; + case t_f64arr: filled = TYARR_SZ(F64, PIA(xa)); break; + case t_harr: filled = fsizeof(HArr,a,B,PIA(xa)); break; + case t_fillarr: filled = fsizeof(FillArr,a,B,PIA(xa)); break; + } + u64 end = mm_size(ptr); + verifyEnd(ptr, end, 8, filled); + #define F(G, X, O, L) if ((G) != (X)) tailFail(G, X, ptr, O, L, filled, end) + ITER_TAIL(F) + #undef F +} + +#endif diff --git a/src/opt/mm_buddy.h b/src/opt/mm_buddy.h index a2b739f3..15453b8b 100644 --- a/src/opt/mm_buddy.h +++ b/src/opt/mm_buddy.h @@ -20,11 +20,20 @@ extern EmptyValue* mm_buckets[64]; #define LOG2(X) ((u8)(64-CLZ((X)-1ull))) -#if !ALLOC_NOINLINE || ALLOC_IMPL || ALLOC_IMPL_ALWAYS +#if !ALLOC_NOINLINE || ALLOC_IMPL || ALLOC_IMPL_MMX ALLOC_FN void* mm_alloc(u64 sz, u8 type) { assert(sz>=16); - onAlloc(sz, type); - return mm_allocL(LOG2(sz), type); + preAlloc(sz, type); + #if VERIFY_TAIL + i64 logAlloc = LOG2(sz + VERIFY_TAIL); + #else + i64 logAlloc = LOG2(sz); + #endif + void* res = mm_allocL(logAlloc, type); + #if VERIFY_TAIL && !ALLOC_IMPL_MMX + tailVerifyAlloc(res, sz, logAlloc, type); + #endif + return res; } #endif diff --git a/src/opt/mm_buddyTemplate.h b/src/opt/mm_buddyTemplate.h index aac99708..badc70da 100644 --- a/src/opt/mm_buddyTemplate.h +++ b/src/opt/mm_buddyTemplate.h @@ -1,8 +1,12 @@ #define buckets BN(buckets) -#if !ALLOC_NOINLINE || ALLOC_IMPL || ALLOC_IMPL_ALWAYS +#if !ALLOC_NOINLINE || ALLOC_IMPL || ALLOC_IMPL_MMX ALLOC_FN void BN(free)(Value* x) { - onFree(x); + #if ALLOC_IMPL_MMX + preFree(x, true); + #else + preFree(x, false); + #endif #ifdef DONT_FREE if (x->type!=t_freed) x->flags = x->type; #else @@ -25,17 +29,6 @@ static void* BN(allocL)(i64 bucket, u8 type) { x->refc = 1; x->type = type; x->mmInfo = bucket; - #if defined(SET_HEAP) - u8* p = (u8*)x; - u8* s = p + sizeof(Value); - u8* e = p + BSZ(bucket); - memset(s, SET_HEAP, e-s); - #elif defined(DEBUG) && !defined(DONT_FREE) - u64* p = (u64*)x; - u64* s = p + sizeof(Value)/8; - u64* e = p + BSZ(bucket)/8; - while(suid = currObjCounter++; #ifdef OBJ_TRACK diff --git a/src/opt/mm_malloc.h b/src/opt/mm_malloc.h index 9a4c014e..99840232 100644 --- a/src/opt/mm_malloc.h +++ b/src/opt/mm_malloc.h @@ -4,13 +4,13 @@ extern u64 mm_heapAlloc; extern u64 mm_heapMax; static void mm_free(Value* x) { - onFree(x); + preFree(x); free(x); } static void* mm_alloc(u64 sz, u8 type) { Value* x = malloc(sz); - onAlloc(sz, type); + preAlloc(sz, type); x->flags = x->extra = x->mmInfo = x->type = 0; x->refc = 1; x->type = type; diff --git a/src/opt/single.c b/src/opt/single.c index d8df8092..91f44ee4 100644 --- a/src/opt/single.c +++ b/src/opt/single.c @@ -1,3 +1,5 @@ +#define MM_C 1 + #include "../core.h" #include "../load.c" #include "../core/tyarr.c" diff --git a/src/utils/mut.h b/src/utils/mut.h index 6f8b3447..b963efe7 100644 --- a/src/utils/mut.h +++ b/src/utils/mut.h @@ -167,6 +167,9 @@ FORCE_INLINE B arr_join_inline(B w, B x, bool consume, bool* reusedW) { u64 ria = wia+xia; if (!reusable(w)) goto no; u64 wsz = mm_size(v(w)); + #if VERIFY_TAIL + wsz-= VERIFY_TAIL; + #endif u8 wt = TY(w); u8 we = TI(w, elType); // TODO f64∾i32, i32∾i8, c32∾c8 etc @@ -204,6 +207,9 @@ static inline bool inplace_add(B w, B x) { // consumes x if returns true; fails usz ria = wia+1; if (reusable(w)) { u64 wsz = mm_size(v(w)); + #if VERIFY_TAIL + wsz-= VERIFY_TAIL; + #endif u8 wt = TY(w); switch (wt) { case t_bitarr: if (BITARR_SZ( ria)