From 6e6169530dd6363be6f06b264fb946e7e3b5d273 Mon Sep 17 00:00:00 2001 From: dzaima Date: Fri, 24 Feb 2023 21:40:52 +0200 Subject: [PATCH] GC v2 --- src/core/heap.c | 1 + src/h.h | 3 +- src/main.c | 5 +- src/opt/gc.c | 133 +++++++++++++++++++++++++++++++------ src/opt/gc.h | 64 ++++++++++-------- src/opt/mm_buddyTemplate.c | 35 ++++++++-- src/opt/mm_malloc.c | 3 +- src/opt/mm_malloc.h | 1 - 8 files changed, 183 insertions(+), 62 deletions(-) diff --git a/src/core/heap.c b/src/core/heap.c index 474ade44..435e8dac 100644 --- a/src/core/heap.c +++ b/src/core/heap.c @@ -29,6 +29,7 @@ void heap_getReferents(Value* v) { if (TIv(v,isArr) && PRNK(v)>1) heapVerify_visitP(shObjP(v)); TIv(v,visit)(v); } +void gc_visitRoots(void); void heapVerify() { heap_observed = 0; heapVerify_mode=0; mm_forHeap(heapVerify_callVisit); gc_visitRoots(); diff --git a/src/h.h b/src/h.h index b068773c..c666e504 100644 --- a/src/h.h +++ b/src/h.h @@ -355,9 +355,8 @@ typedef void (*vfn)(void); void gc_add(B x); // add permanent root object void gc_addFn(vfn f); // add function that calls mm_visit/mm_visitP for dynamic roots void gc_add_ref(B* x); // add x as a root reference -void gc_maybeGC(void); // gc if that seems necessary +bool gc_maybeGC(void); // gc if that seems necessary; returns if did gc void gc_forceGC(void); // force a gc; who knows what happens if gc is disabled (probably should error) -void gc_visitRoots(void); // some primitive actions static const B bi_N = b((u64)0x7FF2000000000000ull); // tag(0,TAG_TAG); // make gcc happy diff --git a/src/main.c b/src/main.c index 533bb5fc..d1c4aa2c 100644 --- a/src/main.c +++ b/src/main.c @@ -427,9 +427,6 @@ static bool isCmd(char* s, char** e, const char* cmd) { return path_rel_dec(p, m_c8vec_0(name)); } - static void replxx_gcFn() { - mm_visit(b_pv); - } static void cbqn_init_replxx() { B cfg = get_config_path(true, "cbqn_repl.txt"); cfg_path = cfg; gc_add(cfg); @@ -464,7 +461,7 @@ static bool isCmd(char* s, char** e, const char* cmd) { } gc_add(sysvalNames); gc_add(sysvalNamesNorm); - gc_addFn(replxx_gcFn); + gc_add_ref(&b_pv); global_replxx = replxx_init(); } diff --git a/src/opt/gc.c b/src/opt/gc.c index 42b14980..02147b69 100644 --- a/src/opt/gc.c +++ b/src/opt/gc.c @@ -29,9 +29,22 @@ void gc_add_ref(B* x) { } -#ifdef LOG_GC -u64 gc_visitBytes, gc_visitCount, gc_freedBytes, gc_freedCount; -#endif + + + +static void gc_freeFreed(Value* v) { + if (v->type==t_freed) mm_free(v); +} + +static void gc_resetTag(Value* x) { + x->mmInfo&= 0x7F; +} + +void gc_visitRoots() { + for (u32 i = 0; i < gc_rootSz; i++) gc_roots[i](); + for (u32 i = 0; i < gc_rootObjSz; i++) mm_visitP(gc_rootObjs[i]); + for (u32 i = 0; i < gc_rootBRefsSz; i++) mm_visit(*gc_rootBRefs[i]); +} static void gc_tryFree(Value* v) { u8 t = v->type; @@ -59,19 +72,89 @@ static void gc_tryFree(Value* v) { } } -static void gc_freeFreed(Value* v) { - if (v->type==t_freed) mm_free(v); -} +#ifdef LOG_GC + u64 gc_visitBytes, gc_visitCount, gc_freedBytes, gc_freedCount, gc_unkRefsBytes, gc_unkRefsCount; +#endif -static void gc_resetTag(Value* x) { - x->mmInfo&= 0x7F; -} +#if GC_VISIT_V2 + i32 visit_mode; + enum { + GC_DEC_REFC, // decrement refcount + GC_INC_REFC, // increment refcount + GC_MARK, // if unmarked, mark & visit + }; + + void gc_onVisit(Value* x) { + switch (visit_mode) { default: UD; + case GC_DEC_REFC: x->refc--; return; + case GC_INC_REFC: x->refc++; return; + case GC_MARK: { + if (x->mmInfo&0x80) return; + x->mmInfo|= 0x80; + #ifdef LOG_GC + gc_visitBytes+= mm_size(x); gc_visitCount++; + #endif + TIv(x,visit)(x); + return; + } + } + } + + static Value** gcv2_bufS; + static Value** gcv2_bufC; + static Value** gcv2_bufE; + FORCE_INLINE void gcv2_storeRemainingEnd(Value* x) { + *gcv2_bufC = x; + gcv2_bufC++; + #ifdef LOG_GC + gc_unkRefsBytes+= mm_size(x); gc_unkRefsCount++; + #endif + } + static NOINLINE void gcv2_storeRemainingR(Value* x) { + ux i = gcv2_bufC - gcv2_bufS; + ux n = (gcv2_bufE-gcv2_bufS)*2; + gcv2_bufS = realloc(gcv2_bufS, n*sizeof(Value*)); + gcv2_bufC = gcv2_bufS + i; + gcv2_bufE = gcv2_bufS + n; + gcv2_storeRemainingEnd(x); + } + static void gcv2_storeRemaining(Value* x) { + gc_resetTag(x); + if (x->refc == 0) return; + if (gcv2_bufC == gcv2_bufE) return gcv2_storeRemainingR(x); + gcv2_storeRemainingEnd(x); + } + static void gcv2_visit(Value* x) { TIv(x,visit)(x); } + + static void gc_run() { + visit_mode = GC_DEC_REFC; + mm_forHeap(gcv2_visit); + + gcv2_bufS = gcv2_bufC = malloc(1<<20); + gcv2_bufE = gcv2_bufS + ((1<<20) / sizeof(Value*)); + mm_forHeap(gcv2_storeRemaining); // incl. unmark + + visit_mode = GC_INC_REFC; + mm_forHeap(gcv2_visit); + + visit_mode = GC_MARK; + gc_visitRoots(); + Value** c = gcv2_bufS; + while (c < gcv2_bufC) mm_visitP(*(c++)); + free(gcv2_bufS); + + mm_forHeap(gc_tryFree); + mm_forHeap(gc_freeFreed); + } +#else + static void gc_run() { + mm_forHeap(gc_resetTag); + gc_visitRoots(); + mm_forHeap(gc_tryFree); + mm_forHeap(gc_freeFreed); + } +#endif -void gc_visitRoots() { - for (u32 i = 0; i < gc_rootSz; i++) gc_roots[i](); - for (u32 i = 0; i < gc_rootObjSz; i++) mm_visitP(gc_rootObjs[i]); - for (u32 i = 0; i < gc_rootBRefsSz; i++) mm_visit(*gc_rootBRefs[i]); -} u64 gc_lastAlloc; void gc_forceGC() { #if ENABLE_GC @@ -79,21 +162,29 @@ void gc_forceGC() { u64 start = nsTime(); gc_visitBytes = 0; gc_freedBytes = 0; gc_visitCount = 0; gc_freedCount = 0; + gc_unkRefsBytes = 0; gc_unkRefsCount = 0; u64 startSize = mm_heapUsed(); #endif - mm_forHeap(gc_resetTag); - gc_visitRoots(); - mm_forHeap(gc_tryFree); - mm_forHeap(gc_freeFreed); + gc_run(); u64 endSize = mm_heapUsed(); #ifdef LOG_GC - fprintf(stderr, "GC kept "N64d"B from "N64d" objects, freed "N64d"B, including directly "N64d"B from "N64d" objects; took %.3fms\n", gc_visitBytes, gc_visitCount, startSize-endSize, gc_freedBytes, gc_freedCount, (nsTime()-start)/1e6); + fprintf(stderr, "GC kept "N64d"B/"N64d" objs, freed "N64d"B, incl. directly "N64d"B/"N64d" objs", gc_visitBytes, gc_visitCount, startSize-endSize, gc_freedBytes, gc_freedCount); + #if GC_VISIT_V2 + fprintf(stderr, "; unknown refs: "N64d"B/"N64d" objs", gc_unkRefsBytes, gc_unkRefsCount); + #endif + fprintf(stderr, "; took %.3fms\n", (nsTime()-start)/1e6); #endif gc_lastAlloc = endSize; #endif } -void gc_maybeGC() { - if (!gc_depth && mm_heapUsed() > gc_lastAlloc*2) gc_forceGC(); +bool gc_maybeGC() { + if (gc_depth) return false; + u64 used = mm_heapUsed(); + if (used > gc_lastAlloc*2) { + gc_forceGC(); + return true; + } + return false; } diff --git a/src/opt/gc.h b/src/opt/gc.h index 8eaa38da..e88cb6c3 100644 --- a/src/opt/gc.h +++ b/src/opt/gc.h @@ -8,30 +8,40 @@ static void gc_enable() { gc_depth--; } extern u64 gc_visitBytes, gc_visitCount, gc_freedBytes, gc_freedCount; #endif -static void mm_visit(B x) { - #ifdef HEAP_VERIFY - if(heapVerify_visit(x)) return; - #endif - - if (!isVal(x)) return; - Value* vx = v(x); - if (vx->mmInfo&0x80) return; - vx->mmInfo|= 0x80; - #ifdef LOG_GC - gc_visitBytes+= mm_size(vx); gc_visitCount++; - #endif - TI(x,visit)(vx); -} -static void mm_visitP(void* xp) { - #ifdef HEAP_VERIFY - if(heapVerify_visitP(xp)) return; - #endif - - Value* x = (Value*)xp; - if (x->mmInfo&0x80) return; - x->mmInfo|= 0x80; - #ifdef LOG_GC - gc_visitBytes+= mm_size(x); gc_visitCount++; - #endif - TIv(x,visit)(x); -} +#if GC_VISIT_V2 + extern void gc_onVisit(Value*); + static void mm_visit(B x) { + if (isVal(x)) gc_onVisit(v(x)); + } + static void mm_visitP(void* xp) { + gc_onVisit(xp); + } +#else + static void mm_visit(B x) { + #ifdef HEAP_VERIFY + if(heapVerify_visit(x)) return; + #endif + + if (!isVal(x)) return; + Value* vx = v(x); + if (vx->mmInfo&0x80) return; + vx->mmInfo|= 0x80; + #ifdef LOG_GC + gc_visitBytes+= mm_size(vx); gc_visitCount++; + #endif + TI(x,visit)(vx); + } + static void mm_visitP(void* xp) { + #ifdef HEAP_VERIFY + if(heapVerify_visitP(xp)) return; + #endif + + Value* x = (Value*)xp; + if (x->mmInfo&0x80) return; + x->mmInfo|= 0x80; + #ifdef LOG_GC + gc_visitBytes+= mm_size(x); gc_visitCount++; + #endif + TIv(x,visit)(x); + } +#endif \ No newline at end of file diff --git a/src/opt/mm_buddyTemplate.c b/src/opt/mm_buddyTemplate.c index f12e13cb..86545f96 100644 --- a/src/opt/mm_buddyTemplate.c +++ b/src/opt/mm_buddyTemplate.c @@ -9,6 +9,8 @@ usz getPageSize(void); #define MMAP(SZ) mmap(NULL, (SZ)+getPageSize(), PROT_READ|PROT_WRITE, MAP_NORESERVE|MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) #endif +#define str0(X) #X +#define str1(X) str0(X) typedef struct AllocInfo { Value* p; @@ -34,18 +36,43 @@ FORCE_INLINE void BN(splitTo)(EmptyValue* c, i64 from, i64 to, bool notEqual) { buckets[from] = c; } +static bool BN(allocMore_rec); + static NOINLINE void* BN(allocateMore)(i64 bucket, u8 type, i64 from, i64 to) { u64 sz = BSZ(from); - if (mm_heapAlloc+sz >= mm_heapMax) thrOOM(); CHECK_INTERRUPT; - mm_heapAlloc+= sz; - // gc_maybeGC(); + + #if GC_VISIT_V2 + if (gc_maybeGC()) goto alloc_rec; + #endif + + if (mm_heapAlloc+sz >= mm_heapMax) { + #if GC_VISIT_V2 + if (!BN(allocMore_rec)) { + gc_forceGC(); + alloc_rec:; + BN(allocMore_rec) = true; + void* r = BN(allocL)(bucket, type); + BN(allocMore_rec) = false; + return r; + } + BN(allocMore_rec) = false; + #else + thrOOM(); + #endif + } + #if NO_MMAP EmptyValue* c = calloc(sz+getPageSize(), 1); #else EmptyValue* c = MMAP(sz); if (c==MAP_FAILED) thrOOM(); #endif + mm_heapAlloc+= sz; + + #if LOG_GC || LOG_MM_MORE + fprintf(stderr, "allocating "N64u" more " str1(BN()) " heap (from allocation of "N64u"B/bucket %d)\n", sz, (u64)BSZ(bucket), (int)bucket); + #endif if (alSize+1>=alCap) { alCap = alCap? alCap*2 : 1024; al = realloc(al, sizeof(AllocInfo)*alCap); @@ -102,8 +129,6 @@ void BN(forFreedHeap)(V2v f) { } } -#define str0(X) #X -#define str1(X) str0(X) void writeNum(FILE* f, u64 v, i32 len); void BN(dumpHeap)(FILE* f) { for (u64 i = 0; i < alSize; i++) { diff --git a/src/opt/mm_malloc.c b/src/opt/mm_malloc.c index 8196d5e8..2b101a8c 100644 --- a/src/opt/mm_malloc.c +++ b/src/opt/mm_malloc.c @@ -9,9 +9,8 @@ void gc_add(B x) { } void gc_addFn(vfn f) { } void gc_add_ref(B* x) { } -void gc_maybeGC() { } +bool gc_maybeGC() { return false; } void gc_forceGC() { } -void gc_visitRoots() { } void mm_forHeap(V2v f) { } u64 mm_heapUsed() { return 123; } // idk void mm_dumpHeap(FILE* f) { } diff --git a/src/opt/mm_malloc.h b/src/opt/mm_malloc.h index fcb35bd6..02bef799 100644 --- a/src/opt/mm_malloc.h +++ b/src/opt/mm_malloc.h @@ -25,7 +25,6 @@ static void mm_visitP(void* x) { } void gc_maybeGC(void); void gc_forceGC(void); -void gc_visitRoots(void); void mm_forHeap(V2v f); void mm_dumpHeap(FILE* f);