359 lines
12 KiB
C
359 lines
12 KiB
C
#include "../core.h"
|
|
#include "../core/gstack.h"
|
|
#include "../ns.h"
|
|
#include "../utils/file.h"
|
|
|
|
#ifndef USE_PERF
|
|
#define USE_PERF 0 // enable writing symbols to /tmp/perf-<pid>.map
|
|
#endif
|
|
#ifndef CSTACK
|
|
#define CSTACK 1
|
|
#endif
|
|
#ifdef GS_REALLOC
|
|
#undef CSTACK
|
|
#define CSTACK 0
|
|
#endif
|
|
|
|
// separate memory management system for executable code; isn't garbage-collected
|
|
#define BSZ(X) (1ull<<(X))
|
|
#define BSZI(X) ((u8)(64-__builtin_clzl((X)-1ull)))
|
|
#define MMI(X) X
|
|
#define BN(X) mmX_##X
|
|
#define buckets mmX_buckets
|
|
#include "../opt/mm_buddyTemplate.h"
|
|
#define MMI(X) X
|
|
#define ALSZ 17
|
|
#define PROT PROT_READ|PROT_WRITE|PROT_EXEC
|
|
#define FLAGS MAP_NORESERVE|MAP_PRIVATE|MAP_ANON|MAP_32BIT
|
|
#include "../opt/mm_buddyTemplate.c"
|
|
static void* mmX_allocN(usz sz, u8 type) { assert(sz>=16); return mmX_allocL(BSZI(sz), type); }
|
|
#undef mmX_buckets
|
|
#undef BN
|
|
#undef BSZI
|
|
#undef BSZ
|
|
|
|
|
|
// all the instructions to be called by the generated code
|
|
#if CSTACK
|
|
#define GA1 ,B* cStack
|
|
#define GSP (*--cStack)
|
|
#define GS_UPD { gStack=cStack; }
|
|
#else
|
|
#define GA1
|
|
#define GSP (*--gStack)
|
|
#define GS_UPD
|
|
#endif
|
|
#define P(N) B N=GSP;
|
|
#if VM_POS
|
|
#define POS_UPD (envCurr-1)->bcL = bc-1;
|
|
#else
|
|
#define POS_UPD
|
|
#endif
|
|
#define INS NOINLINE
|
|
INS void i_POPS(B x) {
|
|
dec(x);
|
|
}
|
|
INS B i_ADDI(u64 v) {
|
|
B o = b(v);
|
|
ptr_inc(v(o));
|
|
return o;
|
|
}
|
|
INS B i_ADDU(u64 v) {
|
|
return b(v);
|
|
}
|
|
INS B i_FN1C(B f, u32* bc GA1) { P(x) // TODO figure out a way to instead pass an offset in bc, so that shorter `mov`s can be used to pass it
|
|
GS_UPD;POS_UPD;
|
|
B r = c1(f, x);
|
|
dec(f); return r;
|
|
}
|
|
INS B i_FN1O(B f, u32* bc GA1) { P(x)
|
|
GS_UPD;POS_UPD;
|
|
B r = isNothing(x)? x : c1(f, x);
|
|
dec(f); return r;
|
|
}
|
|
INS B i_FN2C(B w, u32* bc GA1) { P(f)P(x)
|
|
GS_UPD;POS_UPD;
|
|
B r = c2(f, w, x);
|
|
dec(f); return r;
|
|
}
|
|
INS B i_FN2O(B w, u32* bc GA1) { P(f)P(x)
|
|
GS_UPD;POS_UPD; B r;
|
|
if (isNothing(x)) { dec(w); r = x; }
|
|
else r = isNothing(w)? c1(f, x) : c2(f, w, x);
|
|
dec(f);
|
|
return r;
|
|
}
|
|
INS B i_ARR_0() { // TODO combine with ADDI
|
|
return inc(bi_emptyHVec);
|
|
}
|
|
INS B i_ARR_p(B el0, i64 sz GA1) { assert(sz>0);
|
|
HArr_p r = m_harrUv(sz);
|
|
bool allNum = isNum(el0);
|
|
r.a[sz-1] = el0;
|
|
for (i64 i = 1; i < sz; i++) if (!isNum(r.a[sz-i-1] = GSP)) allNum = false;
|
|
if (allNum) {
|
|
GS_UPD;
|
|
return withFill(r.b, m_f64(0));
|
|
} else return r.b;
|
|
}
|
|
INS B i_DFND_0(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; return m_funBlock(bl, *pscs); }
|
|
INS B i_DFND_1(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; return m_md1Block(bl, *pscs); }
|
|
INS B i_DFND_2(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; return m_md2Block(bl, *pscs); }
|
|
INS B i_OP1D(B f, u32* bc GA1) { P(m) GS_UPD;POS_UPD; return m1_d (m,f ); }
|
|
INS B i_OP2D(B f, u32* bc GA1) { P(m)P(g) GS_UPD;POS_UPD; return m2_d (m,f,g); }
|
|
INS B i_OP2H(B m GA1) { P(g) return m2_h (m, g); }
|
|
INS B i_TR2D(B g GA1) { P(h) return m_atop( g,h); }
|
|
INS B i_TR3D(B f GA1) { P(g)P(h) return m_fork(f,g,h); }
|
|
INS B i_TR3O(B f GA1) { P(g)P(h) B r;
|
|
if (isNothing(f)) { r=m_atop(g,h); dec(f); }
|
|
else { r=m_fork(f,g,h); }
|
|
return r;
|
|
}
|
|
INS B i_LOCM(u32 d, u32 p) {
|
|
return tag((u64)d<<32 | (u32)p, VAR_TAG);
|
|
}
|
|
INS B i_LOCO(u32 d, u32 p, Scope** pscs, u32* bc GA1) {
|
|
B l = pscs[d]->vars[p];
|
|
if(l.u==bi_noVar.u) { POS_UPD; thrM("Reading variable before its defined"); } // TODO probably should GS_UPD (also EXTO)
|
|
return inc(l);
|
|
}
|
|
INS B i_LOCU(u32 d, u32 p, Scope** pscs) {
|
|
B* vars = pscs[d]->vars;
|
|
B r = vars[p];
|
|
vars[p] = bi_optOut;
|
|
return r;
|
|
}
|
|
INS B i_EXTM(u32 d, u32 p) {
|
|
return tag((u64)d<<32 | (u32)p, EXT_TAG);
|
|
}
|
|
INS B i_EXTO(u32 d, u32 p, Scope** pscs, u32* bc GA1) {
|
|
B l = pscs[d]->ext->vars[p];
|
|
if(l.u==bi_noVar.u) { POS_UPD; thrM("Reading variable before its defined"); }
|
|
return inc(l);
|
|
}
|
|
INS B i_EXTU(u32 d, u32 p, Scope** pscs) {
|
|
B* vars = pscs[d]->ext->vars;
|
|
B r = vars[p];
|
|
vars[p] = bi_optOut;
|
|
return r;
|
|
}
|
|
INS B i_SETN(B s, Scope** pscs, u32* bc GA1) { P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, false); dec(s); return x; }
|
|
INS B i_SETU(B s, Scope** pscs, u32* bc GA1) { P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, true ); dec(s); return x; }
|
|
INS B i_SETM(B s, Scope** pscs, u32* bc GA1) { P(f)P(x) GS_UPD; POS_UPD;
|
|
B w = v_get(pscs, s);
|
|
B r = c2(f,w,x); dec(f);
|
|
v_set(pscs, s, r, true); dec(s);
|
|
return r;
|
|
}
|
|
INS B i_FLDO(B ns, u32 p, Scope** pscs GA1) { GS_UPD;
|
|
if (!isNsp(ns)) thrM("Trying to read a field from non-namespace");
|
|
B r = inc(ns_getU(ns, pscs[0]->body->nsDesc->nameList, p));
|
|
dec(ns);
|
|
return r;
|
|
}
|
|
INS B i_NSPM(B o, u32 l) {
|
|
B a = mm_alloc(sizeof(FldAlias), t_fldAlias, ftag(OBJ_TAG));
|
|
c(FldAlias,a)->obj = o;
|
|
c(FldAlias,a)->p = l;
|
|
return a;
|
|
}
|
|
INS B i_RETD(Scope** pscs GA1) {
|
|
Scope* sc = pscs[0];
|
|
Body* b = sc->body;
|
|
ptr_inc(sc);
|
|
ptr_inc(b->nsDesc);
|
|
GS_UPD;
|
|
return m_ns(sc, b->nsDesc);
|
|
}
|
|
|
|
#undef INS
|
|
#undef P
|
|
#undef GSP
|
|
#undef GS_UPD
|
|
#undef POS_UPD
|
|
#undef GA1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#include "x86_64.h"
|
|
|
|
#if USE_PERF
|
|
#include <unistd.h>
|
|
#include "../utils/file.h"
|
|
FILE* perf_map;
|
|
u32 perfid = 0;
|
|
#endif
|
|
|
|
static void* nvm_alloc(u64 sz) {
|
|
// void* r = mmap(NULL, sz, PROT_EXEC|PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON|MAP_32BIT, -1, 0);
|
|
// if (r==MAP_FAILED) thrM("JIT: Failed to allocate executable memory");
|
|
// return r;
|
|
TmpFile* src = mmX_allocN(fsizeof(TmpFile,a,u8,sz), t_i8arr);
|
|
arr_shVec(tag(src,ARR_TAG), sz);
|
|
return src->a;
|
|
}
|
|
void nvm_free(u8* ptr) {
|
|
if (!USE_PERF) mmX_free((Value*)RFLD(ptr, TmpFile, a));
|
|
}
|
|
static void write_asm(u8* p, u64 sz) { // for debugging; view with objdump -b binary -m i386 -M x86-64,intel -D --adjust-vma=$(cat tmp_off) tmp_bin | tail -n+7
|
|
i32* rp; B r = m_i32arrv(&rp, sz);
|
|
for (u64 i = 0; i < sz; i++) rp[i] = p[i];
|
|
file_wBytes(m_str32(U"tmp_bin"), r);
|
|
char off[20]; snprintf(off, 20, "%p", p);
|
|
B o = m_str8l(off);
|
|
file_wChars(m_str32(U"tmp_off"), o);
|
|
dec(r); dec(o);
|
|
}
|
|
|
|
|
|
static u32 readBytes4(u8* d) {
|
|
return d[0] | d[1]<<8 | d[2]<<16 | d[3]<<24;
|
|
}
|
|
|
|
typedef B JITFn(B* cStack, Scope** pscs);
|
|
static inline i32 maxi32(i32 a, i32 b) { return a>b?a:b; }
|
|
u8* m_nvm(Body* body) {
|
|
TSALLOC(u8, bin, 64);
|
|
TSALLOC(u32, rel, 64);
|
|
#define r_TMP 3
|
|
#define r_PSCS 14
|
|
#define r_CS 15
|
|
ASM(PUSH, r_TMP, -);
|
|
ASM(PUSH, r_PSCS, -);
|
|
ASM(PUSH, r_CS, -);
|
|
ASM(MOV, r_CS , REG_ARG0);
|
|
ASM(MOV, r_PSCS, REG_ARG1);
|
|
// #define CCALL(F) { IMM(r_TMP, F); ASM(CALL, r_TMP, -); }
|
|
#define CCALL(F) { if((u64)F < 1ULL<<31) { TSADD(rel, TSSIZE(bin)); ASM(CALLI, (u32)F, -); } else { IMM(r_TMP, F); ASM(CALL, r_TMP, -); } }
|
|
u32* bc = body->bc;
|
|
Block** blocks = body->blocks->a;
|
|
i32 depth = 0;
|
|
while (true) {
|
|
u32* s = bc;
|
|
u32* n = nextBC(bc);
|
|
bool ret = false;
|
|
#define L64 ({ u64 r = bc[0] | ((u64)bc[1])<<32; bc+= 2; r; })
|
|
// #define LEA0(O,I,OFF) { ASM(MOV,O,I); ADDI(O,OFF); }
|
|
#define LEA0(O,I,OFF,Q) ({ i32 o=(OFF); u8 r; if(!o) { r=I; if(Q)ASM(MOV,O,I); } else { r=O; if(o==(i8)o) { ASM3(LEAo1,O,I,o); } else { ASM(MOV,O,I); ADDI(O,o); } } r; })
|
|
#define SPOS(R,N,Q) LEA0(R, r_CS, maxi32(0, depth+(N)-1)*sizeof(B),Q)
|
|
#if CSTACK
|
|
// #define INV(N,D,F) ASM(MOV,REG_ARG##N,r_CS); ADDI(r_CS,(D)*sizeof(B)); CCALL(F)
|
|
#define INV(N,D,F) SPOS(REG_ARG##N, D, 1); CCALL(F)
|
|
#else
|
|
#define INV(N,D,F) CCALL(F) // N - stack argument number; D - expected stack delta; F - called function; TODO instrs which don't need stack (POPS and things with stack delta 1)
|
|
#endif
|
|
#define TOPp ASM(MOV,REG_ARG0,REG_RES)
|
|
#define TOPs if (depth) { u8 t = SPOS(r_TMP, 0, 0); ASM(MOV_MR0, t, REG_RES); }
|
|
switch (*bc++) {
|
|
case POPS: TOPp;
|
|
CCALL(i_POPS);
|
|
if (depth>1) { u8 t = SPOS(r_TMP, -1, 0); ASM(MOV_RM0, REG_RES, t); }
|
|
break;
|
|
case ADDI: TOPs; IMM(REG_ARG0, L64); CCALL(i_ADDI); break; // (u64 v, S)
|
|
case ADDU: TOPs; // (u64 v, S)
|
|
#if CSTACK
|
|
IMM(REG_RES, L64);
|
|
#else
|
|
IMM(REG_ARG0, L64); CCALL(i_ADDU);
|
|
#endif
|
|
break;
|
|
case FN1C: TOPp; IMM(REG_ARG1, s); INV(2,0,i_FN1C); break; // (B, u32* bc, S)
|
|
case FN2C: TOPp; IMM(REG_ARG1, s); INV(2,0,i_FN2C); break; // (B, u32* bc, S)
|
|
case FN1O: TOPp; IMM(REG_ARG1, s); INV(2,0,i_FN1O); break; // (B, u32* bc, S)
|
|
case FN2O: TOPp; IMM(REG_ARG1, s); INV(2,0,i_FN2O); break; // (B, u32* bc, S)
|
|
case ARRM: case ARRO:
|
|
u32 sz = *bc++;
|
|
if (sz) { TOPp; IMM(REG_ARG1, sz); INV(2,0,i_ARR_p); } // (B, i64 sz, S)
|
|
else { TOPs; CCALL(i_ARR_0); } // (S)
|
|
break;
|
|
case DFND: TOPs; // (u32* bc, Scope** pscs, Block* bl, S)
|
|
Block* bl = blocks[*bc++];
|
|
u64 fn = (u64)(bl->ty==0? i_DFND_0 : bl->ty==1? i_DFND_1 : bl->ty==2? i_DFND_2 : NULL);
|
|
if (fn==0) thrM("JIT: Bad DFND argument");
|
|
IMM(REG_ARG0,s); ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,bl); INV(3,1,fn);
|
|
break;
|
|
case OP1D: TOPp; IMM(REG_ARG1,s); INV(2,0,i_OP1D); break; // (B, u32* bc, S)
|
|
case OP2D: TOPp; IMM(REG_ARG1,s); INV(2,0,i_OP2D); break; // (B, u32* bc, S)
|
|
case OP2H: TOPp; INV(1,0,i_OP2H); break; // (B, S)
|
|
case TR2D: TOPp; INV(1,0,i_TR2D); break; // (B, S)
|
|
case TR3D: TOPp; INV(1,0,i_TR3D); break; // (B, S)
|
|
case TR3O: TOPp; INV(1,0,i_TR3O); break; // (B, S)
|
|
case LOCM: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); CCALL(i_LOCM); break; // (u32 d, u32 p, S)
|
|
case LOCO: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); INV(4,1,i_LOCO); break; // (u32 d, u32 p, Scope** pscs, u32* bc, S)
|
|
case LOCU: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); CCALL(i_LOCU); break; // (u32 d, u32 p, Scope** pscs, S)
|
|
case EXTM: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); CCALL(i_EXTM); break; // (u32 d, u32 p, S)
|
|
case EXTO: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); INV(4,1,i_EXTO); break; // (u32 d, u32 p, Scope** pscs, u32* bc, S)
|
|
case EXTU: TOPs; IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); CCALL(i_EXTU); break; // (u32 d, u32 p, Scope** pscs, S)
|
|
case SETN: TOPp; ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,s); INV(3,0,i_SETN); break; // (B, Scope** pscs, u32* bc, S)
|
|
case SETU: TOPp; ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,s); INV(3,0,i_SETU); break; // (B, Scope** pscs, u32* bc, S)
|
|
case SETM: TOPp; ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,s); INV(3,0,i_SETM); break; // (B, Scope** pscs, u32* bc, S)
|
|
case FLDO: TOPp; IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); INV(3,0,i_FLDO); break; // (B, u32 p, Scope** pscs, S)
|
|
case NSPM: TOPp; IMM(REG_ARG1,*bc++); CCALL(i_NSPM); break; // (B, u32 l, S)
|
|
case RETD: ASM(MOV,REG_ARG0,r_PSCS); INV(1,1,i_RETD); ret=true; break; // (Scope** pscs, S); stack diff 0 is wrong, but updating it is useless
|
|
case RETN: IMM(r_TMP, &gStack); ASM(MOV_MR0, r_TMP, r_CS); ret=true; break;
|
|
default: thrF("JIT: Unsupported bytecode %i", *s);
|
|
}
|
|
#undef TOPs
|
|
#undef TOPp
|
|
#undef INV
|
|
#undef SPOS
|
|
#undef L64
|
|
if (n!=bc) thrM("JIT: Wrong parsing of bytecode");
|
|
depth+= stackDiff(s);
|
|
if (ret) break;
|
|
}
|
|
ASM(POP, r_CS, -);
|
|
ASM(POP, r_PSCS, -);
|
|
ASM(POP, r_TMP, -);
|
|
ASM_RAW(bin, RET);
|
|
#undef CCALL
|
|
|
|
u64 sz = TSSIZE(bin);
|
|
u8* binEx = nvm_alloc(sz);
|
|
#if USE_PERF
|
|
if (!perf_map) {
|
|
B s = m_str32(U"/tmp/perf-"); AFMT("%l.map", getpid());
|
|
perf_map = file_open(s, "open", "wa");
|
|
print(s); printf(": map\n");
|
|
dec(s);
|
|
}
|
|
u32 bcPos = body->map[0];
|
|
// printf("JIT %d:\n", perfid);
|
|
// vm_printPos(body->comp, bcPos, -1);
|
|
fprintf(perf_map, "%lx %lx JIT %d: BC@%u\n", (u64)binEx, sz, perfid++, bcPos);
|
|
#endif
|
|
memcpy(binEx, bin, sz);
|
|
u64 relAm = TSSIZE(rel);
|
|
// printf("allocated at %p; i_ADDU: %p\n", binEx, i_ADDU);
|
|
for (u64 i = 0; i < relAm; i++) {
|
|
u8* ins = binEx+rel[i];
|
|
u32 o = readBytes4(ins+1);
|
|
u32 n = o-(u32)ins-5;
|
|
memcpy(ins+1, (u8[]){BYTES4(n)}, 4);
|
|
}
|
|
// write_asm(binEx, sz);
|
|
TSFREE(bin);
|
|
TSFREE(rel);
|
|
return binEx;
|
|
}
|
|
B evalJIT(Body* b, Scope* sc, u8* ptr) { // doesn't consume
|
|
u32* bc = b->bc;
|
|
pushEnv(sc, bc);
|
|
gsReserve(b->maxStack);
|
|
Scope* pscs[b->maxPSC+1];
|
|
pscs[0] = sc;
|
|
for (i32 i = 0; i < b->maxPSC; i++) pscs[i+1] = pscs[i]->psc;
|
|
// write_asm(ptr, RFLD(ptr, TmpFile, a)->ia);
|
|
|
|
B* sp = gStack;
|
|
B r = ((JITFn*)ptr)(gStack, pscs);
|
|
if (sp!=gStack) thrM("uh oh");
|
|
|
|
popEnv();
|
|
return r;
|
|
}
|