#include "../core.h" #include "../core/gstack.h" #include "../ns.h" #include "../utils/file.h" #ifndef USE_PERF #define USE_PERF 0 // enable writing symbols to /tmp/perf-.map #endif #ifndef CSTACK #define CSTACK 1 #endif #ifdef GS_REALLOC #undef CSTACK #define CSTACK 0 #endif // separate memory management system for executable code; isn't garbage-collected #define BSZ(X) (1ull<<(X)) #define BSZI(X) ((u8)(64-__builtin_clzl((X)-1ull))) #define MMI(X) X #define BN(X) mmX_##X #define buckets mmX_buckets #include "../opt/mm_buddyTemplate.h" #define MMI(X) X #define ALSZ 17 #define PROT PROT_READ|PROT_WRITE|PROT_EXEC #define FLAGS MAP_NORESERVE|MAP_PRIVATE|MAP_ANON|MAP_32BIT #include "../opt/mm_buddyTemplate.c" static void* mmX_allocN(usz sz, u8 type) { assert(sz>=16); return mmX_allocL(BSZI(sz), type); } #undef mmX_buckets #undef BN #undef BSZI #undef BSZ // all the instructions to be called by the generated code #if CSTACK #define GA1 ,B* cStack #define GSP (*--cStack) #define GS_UPD { gStack=cStack; } #else #define GA1 #define GSP (*--gStack) #define GS_UPD #endif #define P(N) B N=GSP; #if VM_POS #define POS_UPD (envCurr-1)->bcL = bc-1; #else #define POS_UPD #endif #define INS NOINLINE INS void i_POPS(B x) { dec(x); } INS B i_ADDI(u64 v) { B o = b(v); ptr_inc(v(o)); return o; } INS B i_ADDU(u64 v) { return b(v); } INS B i_FN1C(B f, u32* bc GA1) { P(x) // TODO figure out a way to instead pass an offset in bc, so that shorter `mov`s can be used to pass it GS_UPD;POS_UPD; B r = c1(f, x); dec(f); return r; } INS B i_FN1O(B f, u32* bc GA1) { P(x) GS_UPD;POS_UPD; B r = isNothing(x)? x : c1(f, x); dec(f); return r; } INS B i_FN2C(B w, u32* bc GA1) { P(f)P(x) GS_UPD;POS_UPD; B r = c2(f, w, x); dec(f); return r; } INS B i_FN2O(B w, u32* bc GA1) { P(f)P(x) GS_UPD;POS_UPD; B r; if (isNothing(x)) { dec(w); r = x; } else r = isNothing(w)? c1(f, x) : c2(f, w, x); dec(f); return r; } INS B i_ARR_0() { // TODO combine with ADDI return inc(bi_emptyHVec); } INS B i_ARR_p(B el0, i64 sz GA1) { assert(sz>0); HArr_p r = m_harrUv(sz); bool allNum = isNum(el0); r.a[sz-1] = el0; for (i64 i = 1; i < sz; i++) if (!isNum(r.a[sz-i-1] = GSP)) allNum = false; if (allNum) { GS_UPD; return withFill(r.b, m_f64(0)); } else return r.b; } INS B i_DFND_0(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; return m_funBlock(bl, *pscs); } INS B i_DFND_1(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; return m_md1Block(bl, *pscs); } INS B i_DFND_2(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; return m_md2Block(bl, *pscs); } INS B i_OP1D(B f, u32* bc GA1) { P(m) GS_UPD;POS_UPD; return m1_d (m,f ); } INS B i_OP2D(B f, u32* bc GA1) { P(m)P(g) GS_UPD;POS_UPD; return m2_d (m,f,g); } INS B i_OP2H(B m GA1) { P(g) return m2_h (m, g); } INS B i_TR2D(B g GA1) { P(h) return m_atop( g,h); } INS B i_TR3D(B f GA1) { P(g)P(h) return m_fork(f,g,h); } INS B i_TR3O(B f GA1) { P(g)P(h) B r; if (isNothing(f)) { r=m_atop(g,h); dec(f); } else { r=m_fork(f,g,h); } return r; } INS B i_LOCM(u32 d, u32 p) { return tag((u64)d<<32 | (u32)p, VAR_TAG); } INS B i_LOCO(u32 d, u32 p, Scope** pscs, u32* bc GA1) { B l = pscs[d]->vars[p]; if(l.u==bi_noVar.u) { POS_UPD; thrM("Reading variable before its defined"); } // TODO probably should GS_UPD (also EXTO) return inc(l); } INS B i_LOCU(u32 d, u32 p, Scope** pscs) { B* vars = pscs[d]->vars; B r = vars[p]; vars[p] = bi_optOut; return r; } INS B i_EXTM(u32 d, u32 p) { return tag((u64)d<<32 | (u32)p, EXT_TAG); } INS B i_EXTO(u32 d, u32 p, Scope** pscs, u32* bc GA1) { B l = pscs[d]->ext->vars[p]; if(l.u==bi_noVar.u) { POS_UPD; thrM("Reading variable before its defined"); } return inc(l); } INS B i_EXTU(u32 d, u32 p, Scope** pscs) { B* vars = pscs[d]->ext->vars; B r = vars[p]; vars[p] = bi_optOut; return r; } INS B i_SETN(B s, Scope** pscs, u32* bc GA1) { P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, false); dec(s); return x; } INS B i_SETU(B s, Scope** pscs, u32* bc GA1) { P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, true ); dec(s); return x; } INS B i_SETM(B s, Scope** pscs, u32* bc GA1) { P(f)P(x) GS_UPD; POS_UPD; B w = v_get(pscs, s); B r = c2(f,w,x); dec(f); v_set(pscs, s, r, true); dec(s); return r; } INS B i_FLDO(B ns, u32 p, Scope** pscs GA1) { GS_UPD; if (!isNsp(ns)) thrM("Trying to read a field from non-namespace"); B r = inc(ns_getU(ns, pscs[0]->body->nsDesc->nameList, p)); dec(ns); return r; } INS B i_NSPM(B o, u32 l) { B a = mm_alloc(sizeof(FldAlias), t_fldAlias, ftag(OBJ_TAG)); c(FldAlias,a)->obj = o; c(FldAlias,a)->p = l; return a; } INS B i_RETD(Scope** pscs GA1) { Scope* sc = pscs[0]; Body* b = sc->body; ptr_inc(sc); ptr_inc(b->nsDesc); GS_UPD; return m_ns(sc, b->nsDesc); } #undef INS #undef P #undef GSP #undef GS_UPD #undef POS_UPD #undef GA1 #include "x86_64.h" #if USE_PERF #include #include "../utils/file.h" FILE* perf_map; u32 perfid = 0; #endif static void* nvm_alloc(u64 sz) { // void* r = mmap(NULL, sz, PROT_EXEC|PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON|MAP_32BIT, -1, 0); // if (r==MAP_FAILED) thrM("JIT: Failed to allocate executable memory"); // return r; TmpFile* src = mmX_allocN(fsizeof(TmpFile,a,u8,sz), t_i8arr); arr_shVec(tag(src,ARR_TAG), sz); return src->a; } void nvm_free(u8* ptr) { if (!USE_PERF) mmX_free((Value*)RFLD(ptr, TmpFile, a)); } static void write_asm(u8* p, u64 sz) { // for debugging; view with objdump -b binary -m i386 -M x86-64,intel -D --adjust-vma=$(cat tmp_off) tmp_bin | tail -n+7 i32* rp; B r = m_i32arrv(&rp, sz); for (u64 i = 0; i < sz; i++) rp[i] = p[i]; file_wBytes(m_str32(U"tmp_bin"), r); char off[20]; snprintf(off, 20, "%p", p); B o = m_str8l(off); file_wChars(m_str32(U"tmp_off"), o); dec(r); dec(o); } static u32 readBytes4(u8* d) { return d[0] | d[1]<<8 | d[2]<<16 | d[3]<<24; } typedef B JITFn(B* cStack, Scope** pscs); static inline i32 maxi32(i32 a, i32 b) { return a>b?a:b; } u8* m_nvm(Body* body) { TSALLOC(u8, bin, 64); TSALLOC(u32, rel, 64); #define r_TMP 3 #define r_PSCS 14 #define r_CS 15 ASM(PUSH, r_TMP, -); ASM(PUSH, r_PSCS, -); ASM(PUSH, r_CS, -); ASM(MOV, r_CS , REG_ARG0); ASM(MOV, r_PSCS, REG_ARG1); // #define CCALL(F) { IMM(r_TMP, F); ASM(CALL, r_TMP, -); } #define CCALL(F) { if((u64)F < 1ULL<<31) { TSADD(rel, TSSIZE(bin)); ASM(CALLI, (u32)F, -); } else { IMM(r_TMP, F); ASM(CALL, r_TMP, -); } } u32* bc = body->bc; Block** blocks = body->blocks->a; i32 depth = 0; while (true) { u32* s = bc; u32* n = nextBC(bc); bool ret = false; #define L64 ({ u64 r = bc[0] | ((u64)bc[1])<<32; bc+= 2; r; }) // #define LEA0(O,I,OFF) { ASM(MOV,O,I); ADDI(O,OFF); } #define LEA0(O,I,OFF,Q) ({ i32 o=(OFF); u8 r; if(!o) { r=I; if(Q)ASM(MOV,O,I); } else { r=O; if(o==(i8)o) { ASM3(LEAo1,O,I,o); } else { ASM(MOV,O,I); ADDI(O,o); } } r; }) #define SPOS(R,N,Q) LEA0(R, r_CS, maxi32(0, depth+(N)-1)*sizeof(B),Q) #if CSTACK // #define INV(N,D,F) ASM(MOV,REG_ARG##N,r_CS); ADDI(r_CS,(D)*sizeof(B)); CCALL(F) #define INV(N,D,F) SPOS(REG_ARG##N, D, 1); CCALL(F) #else #define INV(N,D,F) CCALL(F) // N - stack argument number; D - expected stack delta; F - called function; TODO instrs which don't need stack (POPS and things with stack delta 1) #endif #define TOPp ASM(MOV,REG_ARG0,REG_RES) #define TOPs if (depth) { u8 t = SPOS(r_TMP, 0, 0); ASM(MOV_MR0, t, REG_RES); } switch (*bc++) { case POPS: TOPp; CCALL(i_POPS); if (depth>1) { u8 t = SPOS(r_TMP, -1, 0); ASM(MOV_RM0, REG_RES, t); } break; case ADDI: TOPs; IMM(REG_ARG0, L64); CCALL(i_ADDI); break; // (u64 v, S) case ADDU: TOPs; // (u64 v, S) #if CSTACK IMM(REG_RES, L64); #else IMM(REG_ARG0, L64); CCALL(i_ADDU); #endif break; case FN1C: TOPp; IMM(REG_ARG1, s); INV(2,0,i_FN1C); break; // (B, u32* bc, S) case FN2C: TOPp; IMM(REG_ARG1, s); INV(2,0,i_FN2C); break; // (B, u32* bc, S) case FN1O: TOPp; IMM(REG_ARG1, s); INV(2,0,i_FN1O); break; // (B, u32* bc, S) case FN2O: TOPp; IMM(REG_ARG1, s); INV(2,0,i_FN2O); break; // (B, u32* bc, S) case ARRM: case ARRO: u32 sz = *bc++; if (sz) { TOPp; IMM(REG_ARG1, sz); INV(2,0,i_ARR_p); } // (B, i64 sz, S) else { TOPs; CCALL(i_ARR_0); } // (S) break; case DFND: TOPs; // (u32* bc, Scope** pscs, Block* bl, S) Block* bl = blocks[*bc++]; u64 fn = (u64)(bl->ty==0? i_DFND_0 : bl->ty==1? i_DFND_1 : bl->ty==2? i_DFND_2 : NULL); if (fn==0) thrM("JIT: Bad DFND argument"); IMM(REG_ARG0,s); ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,bl); INV(3,1,fn); break; case OP1D: TOPp; IMM(REG_ARG1,s); INV(2,0,i_OP1D); break; // (B, u32* bc, S) case OP2D: TOPp; IMM(REG_ARG1,s); INV(2,0,i_OP2D); break; // (B, u32* bc, S) case OP2H: TOPp; INV(1,0,i_OP2H); break; // (B, S) case TR2D: TOPp; INV(1,0,i_TR2D); break; // (B, S) case TR3D: TOPp; INV(1,0,i_TR3D); break; // (B, S) case TR3O: TOPp; INV(1,0,i_TR3O); break; // (B, S) case LOCM: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); CCALL(i_LOCM); break; // (u32 d, u32 p, S) case LOCO: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); INV(4,1,i_LOCO); break; // (u32 d, u32 p, Scope** pscs, u32* bc, S) case LOCU: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); CCALL(i_LOCU); break; // (u32 d, u32 p, Scope** pscs, S) case EXTM: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); CCALL(i_EXTM); break; // (u32 d, u32 p, S) case EXTO: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); INV(4,1,i_EXTO); break; // (u32 d, u32 p, Scope** pscs, u32* bc, S) case EXTU: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); CCALL(i_EXTU); break; // (u32 d, u32 p, Scope** pscs, S) case SETN: TOPp; ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,s); INV(3,0,i_SETN); break; // (B, Scope** pscs, u32* bc, S) case SETU: TOPp; ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,s); INV(3,0,i_SETU); break; // (B, Scope** pscs, u32* bc, S) case SETM: TOPp; ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,s); INV(3,0,i_SETM); break; // (B, Scope** pscs, u32* bc, S) case FLDO: TOPp; IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); INV(3,0,i_FLDO); break; // (B, u32 p, Scope** pscs, S) case NSPM: TOPp; IMM(REG_ARG1,*bc++); CCALL(i_NSPM); break; // (B, u32 l, S) case RETD: ASM(MOV,REG_ARG0,r_PSCS); INV(1,1,i_RETD); ret=true; break; // (Scope** pscs, S); stack diff 0 is wrong, but updating it is useless case RETN: IMM(r_TMP, &gStack); ASM(MOV_MR0, r_TMP, r_CS); ret=true; break; default: thrF("JIT: Unsupported bytecode %i", *s); } #undef TOPs #undef TOPp #undef INV #undef SPOS #undef L64 if (n!=bc) thrM("JIT: Wrong parsing of bytecode"); depth+= stackDiff(s); if (ret) break; } ASM(POP, r_CS, -); ASM(POP, r_PSCS, -); ASM(POP, r_TMP, -); ASM_RAW(bin, RET); #undef CCALL u64 sz = TSSIZE(bin); u8* binEx = nvm_alloc(sz); #if USE_PERF if (!perf_map) { B s = m_str32(U"/tmp/perf-"); AFMT("%l.map", getpid()); perf_map = file_open(s, "open", "wa"); print(s); printf(": map\n"); dec(s); } u32 bcPos = body->map[0]; // printf("JIT %d:\n", perfid); // vm_printPos(body->comp, bcPos, -1); fprintf(perf_map, "%lx %lx JIT %d: BC@%u\n", (u64)binEx, sz, perfid++, bcPos); #endif memcpy(binEx, bin, sz); u64 relAm = TSSIZE(rel); // printf("allocated at %p; i_ADDU: %p\n", binEx, i_ADDU); for (u64 i = 0; i < relAm; i++) { u8* ins = binEx+rel[i]; u32 o = readBytes4(ins+1); u32 n = o-(u32)ins-5; memcpy(ins+1, (u8[]){BYTES4(n)}, 4); } // write_asm(binEx, sz); TSFREE(bin); TSFREE(rel); return binEx; } B evalJIT(Body* b, Scope* sc, u8* ptr) { // doesn't consume u32* bc = b->bc; pushEnv(sc, bc); gsReserve(b->maxStack); Scope* pscs[b->maxPSC+1]; pscs[0] = sc; for (i32 i = 0; i < b->maxPSC; i++) pscs[i+1] = pscs[i]->psc; // write_asm(ptr, RFLD(ptr, TmpFile, a)->ia); B* sp = gStack; B r = ((JITFn*)ptr)(gStack, pscs); if (sp!=gStack) thrM("uh oh"); popEnv(); return r; }