diff --git a/src/jit/nvm_x86_64.c b/src/jit/nvm_x86_64.c index 9538190c..01786ee1 100644 --- a/src/jit/nvm_x86_64.c +++ b/src/jit/nvm_x86_64.c @@ -4,6 +4,7 @@ #include "../utils/file.h" #include "../utils/talloc.h" #include "../utils/mut.h" +#include "../utils/wyhash.h" #include "../vm.h" #ifndef USE_PERF @@ -24,8 +25,37 @@ u64 mmX_ctrs[64]; #include "../opt/mm_buddyTemplate.h" #define MMI(X) X #define ALSZ 17 -#define PROT PROT_READ|PROT_WRITE|PROT_EXEC -#define FLAGS MAP_NORESERVE|MAP_PRIVATE|MAP_ANON|MAP_32BIT + +static u64 nvm_mmap_seed = 0; +#ifdef __clang__ +#if __clang_major__ < 11 // clang 10 gets stuck in an infinite loop while optimizing this +__attribute__((optnone)) +#endif +#endif +static void* mmap_nvm(u64 sz) { + u64 near = (u64)&bqn_exec; + u64 MAX_DIST = 1ULL<<30; + if (near < MAX_DIST) return mmap(NULL, sz, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_NORESERVE|MAP_PRIVATE|MAP_ANON|MAP_32BIT, -1, 0); + u64 ps = getPageSize(); + + i32 attempt = 0; + // printf("binary at %p\n", &bqn_exec); + while(true) { + if (attempt++ > 200) err("Failed to allocate memory for JIT 200 times; stopping trying"); + u64 randOff = wyrand(&nvm_mmap_seed) & (MAX_DIST>>1)-1; + u64 loc = near+randOff & ~(ps-1); + // printf("request %d: %p\n", attempt, (void*)loc); + + void* c = mmap((void*)loc, sz, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_NORESERVE|MAP_PRIVATE|MAP_ANON|MAP_FIXED_NOREPLACE, -1, 0); + if (c==NULL) continue; + + i64 dist = (i64)near - (i64)c; + if ((dist<0?-dist:dist) < MAX_DIST) return c; + munmap(c, sz); + } +} + +#define MMAP(SZ) mmap_nvm(sz); #include "../opt/mm_buddyTemplate.c" static void* mmX_allocN(usz sz, u8 type) { assert(sz>=16); return mmX_allocL(64-CLZ(sz-1ull), type); } #undef BN @@ -512,8 +542,7 @@ Nvm_res m_nvm(Body* body) { } } - if ((u64)i_RETD > I32_MAX || (u64)&gStack > I32_MAX || (u64)&envEnd > I32_MAX) thrM("JIT: Refusing to run with CBQN code outside of the 32-bit address range"); - #define CCALL(F) { u64 f=(u64)(F); if(f>I32_MAX)thrM("JIT: Function address too large for call"); CALLi(f); } + #define CCALL(F) CALLi((u64)(F)) u32* origBC = body->bc; OptRes optRes = opt(origBC); i32 depth = 0; diff --git a/src/jit/x86_64.h b/src/jit/x86_64.h index 7d5b4a60..ae493298 100644 --- a/src/jit/x86_64.h +++ b/src/jit/x86_64.h @@ -500,6 +500,6 @@ ASMI(PUSH, Reg O) { nREX4(O,0); ASM1(0x50+((O)&7)); } ASMI(POP , Reg O) { nREX4(O,0); ASM1(0x58+((O)&7)); } ASMI(CALL, Reg i) { nREX4(i,0); ASM1(0xFF); nA_REG(i,2); } -ASMI(CALLi, u64 pos) { ASM1(0xE8); if (pos>I32_MAX)err("immediate call outside of 32-bit range!"); ASM4(pos-4); } +ASMI(CALLi, u64 pos) { ASM1(0xE8); ASM4(pos-4); } #define IMM(A,B) MOVi(A,(u64)(B)) diff --git a/src/opt/mm_buddyTemplate.c b/src/opt/mm_buddyTemplate.c index f581d45f..8fb77630 100644 --- a/src/opt/mm_buddyTemplate.c +++ b/src/opt/mm_buddyTemplate.c @@ -4,11 +4,8 @@ #define alCap BN(alCap) #define alSize BN(alSize) #define str(X) #X -#ifndef PROT - #define PROT PROT_READ|PROT_WRITE -#endif -#ifndef FLAGS - #define FLAGS MAP_NORESERVE|MAP_PRIVATE|MAP_ANON +#ifndef MMAP + #define MMAP(SZ) mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_NORESERVE|MAP_PRIVATE|MAP_ANON, -1, 0) #endif typedef struct AllocInfo { @@ -35,7 +32,7 @@ static inline void BN(guaranteeEmpty)(u8 bucket) { if (mm_heapAlloc+sz >= mm_heapMax) { printf("Heap size limit reached\n"); exit(1); } mm_heapAlloc+= sz; // gc_maybeGC(); - c = mmap(NULL, sz, PROT, FLAGS, -1, 0); + c = MMAP(sz); #ifdef USE_VALGRIND VALGRIND_MAKE_MEM_UNDEFINED(c, sz); #endif @@ -80,8 +77,7 @@ void BN(forHeap)(V2v f) { } } -#undef FLAGS -#undef PROT +#undef MMAP #undef AllocInfo #undef buckets #undef al diff --git a/src/vm.c b/src/vm.c index cca6a24c..2bbd9260 100644 --- a/src/vm.c +++ b/src/vm.c @@ -935,13 +935,18 @@ B block_decompose(B x) { return m_v2(m_i32(1), x); } B bl_m1d(B m, B f ) { Md1Block* c = c(Md1Block,m); Block* bl=c(Md1Block, m)->bl; return c->bl->imm? execBlock(bl, bl->bodies[0], c(Md1Block, m)->sc, 2, (B[]){m, f }) : m_md1D(m,f ); } B bl_m2d(B m, B f, B g) { Md2Block* c = c(Md2Block,m); Block* bl=c(Md2Block, m)->bl; return c->bl->imm? execBlock(bl, bl->bodies[0], c(Md2Block, m)->sc, 3, (B[]){m, f, g}) : m_md2D(m,f,g); } +static usz pageSizeV; +usz getPageSize() { + if (pageSizeV==0) pageSizeV = sysconf(_SC_PAGESIZE); + return pageSizeV; +} void allocStack(void** curr, void** start, void** end, i32 elSize, i32 count) { - usz pageSize = sysconf(_SC_PAGESIZE); - u64 sz = (elSize*count + pageSize-1)/pageSize * pageSize; + usz ps = getPageSize(); + u64 sz = (elSize*count + ps-1)/ps * ps; assert(sz%elSize == 0); - *curr = *start = mmap(NULL, sz+pageSize, PROT_READ|PROT_WRITE, MAP_NORESERVE|MAP_PRIVATE|MAP_ANON, -1, 0); + *curr = *start = mmap(NULL, sz+ps, PROT_READ|PROT_WRITE, MAP_NORESERVE|MAP_PRIVATE|MAP_ANON, -1, 0); *end = ((char*)*start)+sz; - mprotect(*end, pageSize, PROT_NONE); // idk first way i found to force erroring on overflow + mprotect(*end, ps, PROT_NONE); // idk first way i found to force erroring on overflow } void print_vmStack() { #ifdef DEBUG_VM diff --git a/src/vm.h b/src/vm.h index 0f1b88a6..bf56cce3 100644 --- a/src/vm.h +++ b/src/vm.h @@ -231,6 +231,7 @@ NOINLINE B vm_fmtPoint(B src, B prepend, B path, usz cs, usz ce); // consumes pr NOINLINE void printErrMsg(B msg); NOINLINE void unwindEnv(Env* envNew); // envNew==envStart-1 for emptying the env stack NOINLINE void unwindCompiler(void); // unwind to the env of the invocation of the compiler; UB when not in compiler! +usz getPageSize(); typedef struct FldAlias {