From 16274e5952b3dcaa3a0ea588b4435f8c53ec28c0 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sat, 4 Mar 2023 20:09:12 +0200 Subject: [PATCH] bucket coalescing on GC --- src/core/heap.c | 29 +++++++++++++++----- src/core/heap.h | 2 -- src/h.h | 2 +- src/jit/nvm_x86_64.c | 3 +++ src/main.c | 19 +++++++------ src/opt/gc.c | 18 ++++++------- src/opt/mm_2buddy.c | 6 +++++ src/opt/mm_buddy.c | 1 + src/opt/mm_buddyTemplate.c | 55 +++++++++++++++++++++++++++++++++++--- src/opt/mm_buddyTemplate.h | 13 ++++++--- src/opt/mm_malloc.c | 4 +-- src/opt/mm_malloc.h | 4 +-- 12 files changed, 118 insertions(+), 38 deletions(-) diff --git a/src/core/heap.c b/src/core/heap.c index 36b7094c..0d40634e 100644 --- a/src/core/heap.c +++ b/src/core/heap.c @@ -65,7 +65,7 @@ void heap_PIFreedFn(Value* v) { } void mm_forFreedHeap(V2v f); -void heap_printInfo(bool sizes, bool types, bool freed) { +void heap_printInfo(bool sizes, bool types, bool freed, bool chain) { u64 total = mm_heapAlloc; u64 used = mm_heapUsed(); printf("RAM allocated: "N64u"\n", total); @@ -88,14 +88,31 @@ void heap_printInfo(bool sizes, bool types, bool freed) { if (count>0) printf("type %d/%s: count "N64u", total size "N64u"\n", i, type_repr(i), count, heap_PISizes[i]); } } - if (freed) { - for (i32 i = 0; i < MM*64; i++) heap_PIFreed[i] = 0; - mm_forFreedHeap(heap_PIFreedFn); + if (freed || chain) { + if (freed) { + for (i32 i = 0; i < MM*64; i++) heap_PIFreed[i] = 0; + mm_forFreedHeap(heap_PIFreedFn); + } + for (i32 i = 2; i < 64; i++) { for (i32 j = 0; j < MM; j++) { i32 o = i-j; - u64 count = heap_PIFreed[o + j*64]; - if (count>0) printf("freed size %llu: "N64u"\n", (1ULL<next; } + } + } + if (cf!=0 || cc!=0) { + if (chain && freed) printf("freed size %llu: count "N64u", chain "N64u"\n", (1ULL<=16); return mmX_allocL(64-CLZ(sz-1ull), type); } #undef BN #undef BSZ diff --git a/src/main.c b/src/main.c index 675facf3..a0ca35b4 100644 --- a/src/main.c +++ b/src/main.c @@ -128,7 +128,7 @@ static NOINLINE i64 readInt(char** p) { ")exit", ")off", ")vars", ")gc", ")gc off", ")gc on", - ")internalPrint ", + ")internalPrint ", ")heapdump", #if NATIVE_COMPILER && !ONLY_NATIVE_COMP ")switchCompiler", #endif @@ -577,6 +577,7 @@ static B simple_unescape(B x) { return c1(escape_parser, x); } +void heap_printInfo(bool sizes, bool types, bool freed, bool chain); void cbqn_runLine0(char* ln, i64 read) { if (ln[0]==0 || read==0) return; @@ -624,16 +625,15 @@ void cbqn_runLine0(char* ln, i64 read) { time = am; output = 0; } else if (isCmd(cmdS, &cmdE, "mem ")) { - bool sizes = 0; - bool types = 0; - bool freed = 0; + bool sizes=0, types=0; + i32 freed=0; char c; while ((c=*(cmdE++)) != 0) { if (c=='t') types=1; if (c=='s') sizes=1; - if (c=='f') freed=1; + if (c=='f') freed++; } - heap_printInfo(sizes, types, freed); + heap_printInfo(sizes, types, freed!=0, freed>=2); return; } else if (isCmd(cmdS, &cmdE, "erase ")) { char* name = cmdE; @@ -663,6 +663,9 @@ void cbqn_runLine0(char* ln, i64 read) { } else if (isCmd(cmdS, &cmdE, "clearImportCache ")) { clearImportCache(); return; + } else if (isCmd(cmdS, &cmdE, "heapdump ")) { + cbqn_heapDump(NULL); + return; #if USE_REPLXX } else if (isCmd(cmdS, &cmdE, "kb ")) { if (*cmdE == 0) { @@ -825,7 +828,7 @@ void cbqn_runLine(char* ln, i64 len) { #ifdef HEAP_VERIFY heapVerify(); #endif - gc_maybeGC(); + gc_maybeGC(true); return; } cbqn_takeInterrupts(true); @@ -833,7 +836,7 @@ void cbqn_runLine(char* ln, i64 len) { #ifdef HEAP_VERIFY heapVerify(); #endif - gc_maybeGC(); + gc_maybeGC(true); cbqn_takeInterrupts(false); popCatch(); } diff --git a/src/opt/gc.c b/src/opt/gc.c index 8a9b6761..2b8c03b3 100644 --- a/src/opt/gc.c +++ b/src/opt/gc.c @@ -1,5 +1,6 @@ #include "gc.h" +void mm_freeFreedAndMerge(void); #ifdef LOG_GC #include "../utils/time.h" #endif @@ -32,10 +33,6 @@ void gc_add_ref(B* x) { -static void gc_freeFreed(Value* v) { - if (v->type==t_freed) mm_free(v); -} - static void gc_resetTag(Value* x) { x->mmInfo&= 0x7F; } @@ -161,14 +158,14 @@ static void gc_tryFree(Value* v) { } mm_forHeap(gc_tryFree); - mm_forHeap(gc_freeFreed); + mm_freeFreedAndMerge(); } #else static void gc_run(bool toplevel) { mm_forHeap(gc_resetTag); gc_visitRoots(); mm_forHeap(gc_tryFree); - mm_forHeap(gc_freeFreed); + mm_freeFreedAndMerge(); } #endif @@ -195,12 +192,13 @@ void gc_forceGC(bool toplevel) { #endif } - -bool gc_maybeGC() { +static bool gc_wantTopLevelGC; +bool gc_maybeGC(bool toplevel) { if (gc_depth) return false; u64 used = mm_heapUsed(); - if (used > gc_lastAlloc*2) { - gc_forceGC(false); + if (used > gc_lastAlloc*2 || (toplevel && gc_wantTopLevelGC)) { + if (toplevel) gc_wantTopLevelGC = false; + gc_forceGC(toplevel); return true; } return false; diff --git a/src/opt/mm_2buddy.c b/src/opt/mm_2buddy.c index c0de4b92..18070b4a 100644 --- a/src/opt/mm_2buddy.c +++ b/src/opt/mm_2buddy.c @@ -17,6 +17,7 @@ EmptyValue* mm_buckets[128]; #define b1_allocL mm_allocL #define ALSZ 20 #define BSZ(X) (1ull<<(X)) +#define MUL 1 #define MMI(X) X #define BN(X) b1_##X #include "mm_buddyTemplate.c" @@ -27,6 +28,7 @@ EmptyValue* mm_buckets[128]; #define b3_allocL mm_allocL #define ALSZ 20 #define BSZ(X) (3ull<<(X)) +#define MUL 3 #define MMI(X) ((X)|64) #define BN(X) b3_##X #include "mm_buddyTemplate.c" @@ -45,6 +47,10 @@ void mm_forFreedHeap(V2v f) { b1_forFreedHeap(f); b3_forFreedHeap(f); } +void mm_freeFreedAndMerge() { + b1_freeFreedAndMerge(); + b3_freeFreedAndMerge(); +} void mm_dumpHeap(FILE* f) { b1_dumpHeap(f); b3_dumpHeap(f); diff --git a/src/opt/mm_buddy.c b/src/opt/mm_buddy.c index 6fa4b396..d5d7478c 100644 --- a/src/opt/mm_buddy.c +++ b/src/opt/mm_buddy.c @@ -12,6 +12,7 @@ u64 mm_ctrs[64]; EmptyValue* mm_buckets[64]; #define ALSZ 20 #define BSZ(X) (1ull<<(X)) +#define MUL 1 #define MMI(X) X #define BN(X) mm_##X #include "mm_buddyTemplate.c" diff --git a/src/opt/mm_buddyTemplate.c b/src/opt/mm_buddyTemplate.c index e34b610f..4a30cb19 100644 --- a/src/opt/mm_buddyTemplate.c +++ b/src/opt/mm_buddyTemplate.c @@ -36,7 +36,7 @@ FORCE_INLINE void BN(splitTo)(EmptyValue* c, i64 from, i64 to, bool notEqual) { buckets[from] = c; } -#if GC_VISIT_V2 +#if GC_VISIT_V2 && !BUDDY_NO_GC static bool BN(allocMore_rec); #endif @@ -44,12 +44,12 @@ static NOINLINE void* BN(allocateMore)(i64 bucket, u8 type, i64 from, i64 to) { u64 sz = BSZ(from); CHECK_INTERRUPT; - #if GC_VISIT_V2 - if (gc_maybeGC()) goto alloc_rec; + #if GC_VISIT_V2 && !BUDDY_NO_GC + if (gc_maybeGC(false)) goto alloc_rec; #endif if (mm_heapAlloc+sz >= mm_heapMax) { - #if GC_VISIT_V2 + #if GC_VISIT_V2 && !BUDDY_NO_GC if (!BN(allocMore_rec)) { gc_forceGC(false); BN(allocMore_rec) = true; @@ -59,6 +59,7 @@ static NOINLINE void* BN(allocateMore)(i64 bucket, u8 type, i64 from, i64 to) { return r; } BN(allocMore_rec) = false; + gc_wantTopLevelGC = true; #endif thrOOM(); } @@ -129,6 +130,51 @@ void BN(forFreedHeap)(V2v f) { } } } +static void BN(freeFreedAndMerge)() { + for (u64 i = 0; i < 64; i++) buckets[i] = NULL; + + for (u64 i = 0; i < alSize; i++) { + AllocInfo ci = al[i]; + u64 emptySize = 0; + Value* c = ci.p; + Value* e = (Value*)(ci.sz + (u8*)ci.p); + assert(c!=e); + while (true) { + if (vg_def_v(c->type)==t_freed) BN(freeLink)(c, false); + + Value* next = (Value*)(BSZ(vg_def_v(c->mmInfo)&63) + (u8*)c); + if (vg_def_v(c->type)==t_empty) { + emptySize+= 1ULL<<(vg_def_v(c->mmInfo)&63); + } else if (emptySize > 0) { + emptyTail:; + u8* emptyStart = ((u8*)c) - emptySize*MUL; + while(emptySize) { + u64 left = emptySize & (emptySize-1); + u64 curr = emptySize ^ left; + + EmptyValue* cv = (EmptyValue*)emptyStart; + u64 b = 63-CLZ(curr); + *cv = (EmptyValue){ + .type = 0, + .mmInfo = MMI(b), + .next = buckets[b] + }; + buckets[b] = cv; + + emptyStart+= curr*MUL; + emptySize = left; + } + emptySize = 0; + } + + c = next; + if (c==e) { + if (emptySize!=0) goto emptyTail; + break; + } + } + } +} void writeNum(FILE* f, u64 v, i32 len); void BN(dumpHeap)(FILE* f) { @@ -151,4 +197,5 @@ void BN(dumpHeap)(FILE* f) { #undef alSize #undef alCap #undef MMI +#undef MUL #undef ALSZ diff --git a/src/opt/mm_buddyTemplate.h b/src/opt/mm_buddyTemplate.h index badc70da..f789e2c4 100644 --- a/src/opt/mm_buddyTemplate.h +++ b/src/opt/mm_buddyTemplate.h @@ -1,7 +1,8 @@ #define buckets BN(buckets) #if !ALLOC_NOINLINE || ALLOC_IMPL || ALLOC_IMPL_MMX -ALLOC_FN void BN(free)(Value* x) { + +FORCE_INLINE void BN(freeLink)(Value* x, bool link) { #if ALLOC_IMPL_MMX preFree(x, true); #else @@ -12,13 +13,19 @@ ALLOC_FN void BN(free)(Value* x) { #else u8 b = x->mmInfo&127; BN(ctrs)[b]--; - ((EmptyValue*)x)->next = buckets[b]; - buckets[b] = (EmptyValue*)x; + if (link) { + ((EmptyValue*)x)->next = buckets[b]; + buckets[b] = (EmptyValue*)x; + } #endif x->type = t_empty; vg_undef_p(x, BSZ(x->mmInfo&127)); } +ALLOC_FN void BN(free)(Value* x) { + BN(freeLink)(x, true); +} + NOINLINE void* BN(allocS)(i64 bucket, u8 type); static void* BN(allocL)(i64 bucket, u8 type) { EmptyValue* x = buckets[bucket]; diff --git a/src/opt/mm_malloc.c b/src/opt/mm_malloc.c index 2b101a8c..ffa96ac0 100644 --- a/src/opt/mm_malloc.c +++ b/src/opt/mm_malloc.c @@ -9,8 +9,8 @@ void gc_add(B x) { } void gc_addFn(vfn f) { } void gc_add_ref(B* x) { } -bool gc_maybeGC() { return false; } -void gc_forceGC() { } +bool gc_maybeGC(bool toplevel) { return false; } +void gc_forceGC(bool toplevel) { } void mm_forHeap(V2v f) { } u64 mm_heapUsed() { return 123; } // idk void mm_dumpHeap(FILE* f) { } diff --git a/src/opt/mm_malloc.h b/src/opt/mm_malloc.h index 02bef799..29d8a370 100644 --- a/src/opt/mm_malloc.h +++ b/src/opt/mm_malloc.h @@ -23,8 +23,8 @@ static void gc_enable() { } static void mm_visit(B x) { } static void mm_visitP(void* x) { } -void gc_maybeGC(void); -void gc_forceGC(void); +void gc_maybeGC(bool); +void gc_forceGC(bool); void mm_forHeap(V2v f); void mm_dumpHeap(FILE* f);