local stack pointer in JIT

This commit is contained in:
dzaima 2021-06-10 01:42:29 +03:00
parent 550b91688a
commit 981ba0e261
3 changed files with 122 additions and 82 deletions

View File

@ -201,7 +201,7 @@ static bool atomEqual(B w, B x) { // doesn't consume (not that that matters real
#ifdef DEBUG
static Value* VALIDATEP(Value* x) {
static NOINLINE Value* VALIDATEP(Value* x) {
if (x->refc<=0 || (x->refc>>28) == 'a' || x->type==t_empty) {
printf("bad refcount for type %d: %d\nattempting to print: ", x->type, x->refc); fflush(stdout);
print(tag(x,OBJ_TAG)); putchar('\n'); fflush(stdout);
@ -214,7 +214,7 @@ static bool atomEqual(B w, B x) { // doesn't consume (not that that matters real
}
return x;
}
static B VALIDATE(B x) {
static NOINLINE B VALIDATE(B x) {
if (!isVal(x)) return x;
VALIDATEP(v(x));
if(isArr(x)!=TI(x).isArr && v(x)->type!=t_freed && v(x)->type!=t_harrPartial) {

View File

@ -6,6 +6,13 @@
#ifndef USE_PERF
#define USE_PERF 0 // enable writing symbols to /tmp/perf-<pid>.map
#endif
#ifndef CSTACK
#define CSTACK 1
#endif
#ifdef GS_REALLOC
#undef CSTACK
#define CSTACK 0
#endif
// separate memory management system for executable code; isn't garbage-collected
#define BSZ(X) (1ull<<(X))
@ -27,52 +34,62 @@ static void* mmX_allocN(usz sz, u8 type) { assert(sz>=16); return mmX_allocL(BSZ
// all the instructions to be called by the generated code
#define GSA(X) { B tr=X; *(gStack++) = tr; }
#define GSP (*--gStack)
#if CSTACK
#define GA0 B* cStack
#define GA1 ,B* cStack
#define GSA(X) { B tr=X; *(cStack++) = tr; }
#define GSP (*--cStack)
#define GS_UPD { gStack=cStack; }
#else
#define GA0
#define GA1
#define GSA(X) { B tr=X; *(gStack++) = tr; }
#define GSP (*--gStack)
#define GS_UPD
#endif
#define P(N) B N=GSP;
#define GS_UPD
#if VM_POS
#define POS_UPD (envCurr-1)->bcL = bc-1;
#else
#define POS_UPD
#endif
#define INS NOINLINE
INS void i_POPS() {
INS void i_POPS(GA0) {
dec(GSP);
}
INS void i_PUSH() {
thrM("NYI PUSH in nvm");
}
INS void i_ADDI(u64 v) {
// INS void i_PUSH(GA0) {
// thrM("NYI PUSH in nvm");
// }
INS void i_ADDI(u64 v GA1) {
B o = b(v);
ptr_inc(v(o));
GSA(o);
}
INS void i_ADDU(u64 v) {
INS void i_ADDU(u64 v GA1) {
GSA(b(v));
}
INS void i_FN1C(u32* bc) { P(f)P(x) // TODO figure out a way to instead pass an offset in bc, so that shorter `mov`s can be used to pass it
INS void i_FN1C(u32* bc GA1) { P(f)P(x) // TODO figure out a way to instead pass an offset in bc, so that shorter `mov`s can be used to pass it
GS_UPD;POS_UPD;
GSA(c1(f, x)); dec(f);
}
INS void i_FN1O(u32* bc) { P(f)P(x)
INS void i_FN1O(u32* bc GA1) { P(f)P(x)
GS_UPD;POS_UPD;
GSA(isNothing(x)? x : c1(f, x)); dec(f);
}
INS void i_FN2C(u32* bc) { P(w)P(f)P(x)
INS void i_FN2C(u32* bc GA1) { P(w)P(f)P(x)
GS_UPD;POS_UPD;
GSA(c2(f, w, x)); dec(f);
}
INS void i_FN2O(u32* bc) { P(w)P(f)P(x)
INS void i_FN2O(u32* bc GA1) { P(w)P(f)P(x)
GS_UPD;POS_UPD;
if (isNothing(x)) { dec(w); GSA(x); }
else GSA(isNothing(w)? c1(f, x) : c2(f, w, x));
dec(f);
}
INS void i_ARR_0() {
INS void i_ARR_0(GA0) {
GSA(inc(bi_emptyHVec));
}
INS void i_ARR_p(i64 sz) {
INS void i_ARR_p(i64 sz GA1) {
HArr_p r = m_harrUv(sz);
bool allNum = true;
assert(sz>0);
@ -82,71 +99,73 @@ INS void i_ARR_p(i64 sz) {
GSA(withFill(r.b, m_f64(0)));
} else GSA(r.b);
}
INS void i_DFND_0(u32* bc, Scope** pscs, Block* bl) { GS_UPD;POS_UPD; GSA(m_funBlock(bl, *pscs)); }
INS void i_DFND_1(u32* bc, Scope** pscs, Block* bl) { GS_UPD;POS_UPD; GSA(m_md1Block(bl, *pscs)); }
INS void i_DFND_2(u32* bc, Scope** pscs, Block* bl) { GS_UPD;POS_UPD; GSA(m_md2Block(bl, *pscs)); }
INS void i_OP1D(u32* bc) { P(f)P(m) GS_UPD;POS_UPD; GSA(m1_d (m,f )); }
INS void i_OP2D(u32* bc) { P(f)P(m)P(g) GS_UPD;POS_UPD; GSA(m2_d (m,f,g)); }
INS void i_OP2H( ) { P(m)P(g) GSA(m2_h (m, g)); }
INS void i_TR2D( ) { P(g)P(h) GSA(m_atop( g,h)); }
INS void i_TR3D( ) { P(f)P(g)P(h) GSA(m_fork(f,g,h)); }
INS void i_TR3O( ) { P(f)P(g)P(h)
INS void i_DFND_0(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; GSA(m_funBlock(bl, *pscs)); }
INS void i_DFND_1(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; GSA(m_md1Block(bl, *pscs)); }
INS void i_DFND_2(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; GSA(m_md2Block(bl, *pscs)); }
INS void i_OP1D(u32* bc GA1) { P(f)P(m) GS_UPD;POS_UPD; GSA(m1_d (m,f )); }
INS void i_OP2D(u32* bc GA1) { P(f)P(m)P(g) GS_UPD;POS_UPD; GSA(m2_d (m,f,g)); }
INS void i_OP2H( GA0) { P(m)P(g) GSA(m2_h (m, g)); }
INS void i_TR2D( GA0) { P(g)P(h) GSA(m_atop( g,h)); }
INS void i_TR3D( GA0) { P(f)P(g)P(h) GSA(m_fork(f,g,h)); }
INS void i_TR3O( GA0) { P(f)P(g)P(h)
if (isNothing(f)) { GSA(m_atop(g,h)); dec(f); }
else GSA(m_fork(f,g,h));
}
INS void i_LOCM(u32 d, u32 p) {
INS void i_LOCM(u32 d, u32 p GA1) {
GSA(tag((u64)d<<32 | (u32)p, VAR_TAG));
}
INS void i_LOCO(u32 d, u32 p, Scope** pscs, u32* bc) {
INS void i_LOCO(u32 d, u32 p, Scope** pscs, u32* bc GA1) {
B l = pscs[d]->vars[p];
if(l.u==bi_noVar.u) { POS_UPD; thrM("Reading variable before its defined"); }
GSA(inc(l));
}
INS void i_LOCU(u32 d, u32 p, Scope** pscs) {
INS void i_LOCU(u32 d, u32 p, Scope** pscs GA1) {
B* vars = pscs[d]->vars;
GSA(vars[p]);
vars[p] = bi_optOut;
}
INS void i_EXTM(u32 d, u32 p) {
INS void i_EXTM(u32 d, u32 p GA1) {
GSA(tag((u64)d<<32 | (u32)p, EXT_TAG));
}
INS void i_EXTO(u32 d, u32 p, Scope** pscs, u32* bc) {
INS void i_EXTO(u32 d, u32 p, Scope** pscs, u32* bc GA1) {
B l = pscs[d]->ext->vars[p];
if(l.u==bi_noVar.u) { POS_UPD; thrM("Reading variable before its defined"); }
GSA(inc(l));
}
INS void i_EXTU(u32 d, u32 p, Scope** pscs) {
INS void i_EXTU(u32 d, u32 p, Scope** pscs GA1) {
B* vars = pscs[d]->ext->vars;
GSA(vars[p]);
vars[p] = bi_optOut;
}
INS void i_SETN(Scope** pscs, u32* bc) { P(s) P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, false); dec(s); GSA(x); }
INS void i_SETU(Scope** pscs, u32* bc) { P(s) P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, true ); dec(s); GSA(x); }
INS void i_SETM(Scope** pscs, u32* bc) { P(s)P(f)P(x) GS_UPD; POS_UPD;
INS void i_SETN(Scope** pscs, u32* bc GA1) { P(s) P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, false); dec(s); GSA(x); }
INS void i_SETU(Scope** pscs, u32* bc GA1) { P(s) P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, true ); dec(s); GSA(x); }
INS void i_SETM(Scope** pscs, u32* bc GA1) { P(s)P(f)P(x) GS_UPD; POS_UPD;
B w = v_get(pscs, s);
B r = c2(f,w,x); dec(f);
v_set(pscs, s, r, true); dec(s);
GSA(r);
}
INS void i_FLDO(u32 p, Scope** pscs) { P(ns) GS_UPD;
INS void i_FLDO(u32 p, Scope** pscs GA1) { P(ns) GS_UPD;
if (!isNsp(ns)) thrM("Trying to read a field from non-namespace");
GSA(inc(ns_getU(ns, pscs[0]->body->nsDesc->nameList, p)));
dec(ns);
}
INS void i_NSPM(u32 l) { P(o)
INS void i_NSPM(u32 l GA1) { P(o)
B a = mm_alloc(sizeof(FldAlias), t_fldAlias, ftag(OBJ_TAG));
c(FldAlias,a)->obj = o;
c(FldAlias,a)->p = l;
GSA(a);
}
INS B i_RETD(Scope** pscs) {
INS B i_RETD(Scope** pscs GA1) {
Scope* sc = pscs[0];
Body* b = sc->body;
ptr_inc(sc);
ptr_inc(b->nsDesc);
GS_UPD;
return m_ns(sc, b->nsDesc);
}
INS B i_RETN() { P(v)
INS B i_RETN(GA0) { P(v)
GS_UPD;
return v;
}
@ -156,6 +175,8 @@ INS B i_RETN() { P(v)
#undef GSA
#undef GS_UPD
#undef POS_UPD
#undef GA0
#undef GA1
@ -164,8 +185,6 @@ INS B i_RETN() { P(v)
#include "x86_64.h"
#define IMM(O,I) { u64 v=(u64)(I); if(v==0) ASM_RAW(bin, XOR(O,O)); else if(v>=0 & v<(1ULL<<32)) ASM_RAW(bin, MOV4_RI(O,v)); else ASM_RAW(bin, MOV_RI(O,v)); }
#define ASM(INS, O, I) ASM_RAW(bin, INS(O, I))
#if USE_PERF
#include <unistd.h>
@ -200,17 +219,20 @@ static u32 readBytes4(u8* d) {
return d[0] | d[1]<<8 | d[2]<<16 | d[3]<<24;
}
typedef B JITFn(Scope** pscs);
typedef B JITFn(B* cStack, Scope** pscs);
u8* m_nvm(Body* body) {
TSALLOC(u8, bin, 64);
TSALLOC(u32, rel, 64);
#define r_TMP 12
#define r_PSCS 13
// ASM(PUSH, 12, -);
#define r_CS 14
ASM(PUSH, 12, -);
ASM(PUSH, 13, -);
// ASM(PUSH, 14, -);
ASM(MOV, r_PSCS, REG_ARG0);
// #define CCALL(F) { IMM(12, F); ASM(CALL, 12, -); }
#define CCALL(F) { if((u64)F < 1ULL<<31) { TSADD(rel, TSSIZE(bin)); ASM(CALLI, (u32)F, -); } else { IMM(12, F); ASM(CALL, 12, -); } }
ASM(PUSH, 14, -);
ASM(MOV, r_CS , REG_ARG0);
ASM(MOV, r_PSCS, REG_ARG1);
// #define CCALL(F) { IMM(r_TMP, F); ASM(CALL, r_TMP, -); }
#define CCALL(F) { if((u64)F < 1ULL<<31) { TSADD(rel, TSSIZE(bin)); ASM(CALLI, (u32)F, -); } else { IMM(r_TMP, F); ASM(CALL, r_TMP, -); } }
u32* bc = body->bc;
Block** blocks = body->blocks->a;
while (true) {
@ -218,54 +240,64 @@ u8* m_nvm(Body* body) {
u32* n = nextBC(bc);
bool ret = false;
#define L64 ({ u64 r = bc[0] | ((u64)bc[1])<<32; bc+= 2; r; })
#if CSTACK
#define INV(N,D,F) ASM(MOV,REG_ARG##N, r_CS); ADDI(r_CS,(D)*sizeof(B)); CCALL(F)
#else
#define INV(N,D,F) CCALL(F) // N - stack argument number; D - expected stack delta; F - called function
#endif
switch (*bc++) {
case POPS: CCALL(i_POPS); break;
case ADDI: IMM(REG_ARG0, L64); CCALL(i_ADDI); break; // (u64 v)
case ADDU: IMM(REG_ARG0, L64); CCALL(i_ADDU); break; // (u64 v)
case FN1C: IMM(REG_ARG0, s); CCALL(i_FN1C); break; // (u32* bc)
case FN2C: IMM(REG_ARG0, s); CCALL(i_FN2C); break; // (u32* bc)
case FN1O: IMM(REG_ARG0, s); CCALL(i_FN1O); break; // (u32* bc)
case FN2O: IMM(REG_ARG0, s); CCALL(i_FN2O); break; // (u32* bc)
case ARRM: case ARRO: // (i64 sz)
case POPS: INV(0,-1,i_POPS); break; // (S)
case ADDI: IMM(REG_ARG0, L64); INV(1,1,i_ADDI); break; // (u64 v, S)
case ADDU:
#if CSTACK
IMM(r_TMP, L64); ASM(MOV_MR0, r_CS, r_TMP); ADDI(r_CS,sizeof(B));
#else
IMM(REG_ARG0, L64); INV(1,1,i_ADDU);
#endif
break; // (u64 v, S)
case FN1C: IMM(REG_ARG0, s); INV(1,-1,i_FN1C); break; // (u32* bc, S)
case FN2C: IMM(REG_ARG0, s); INV(1,-2,i_FN2C); break; // (u32* bc, S)
case FN1O: IMM(REG_ARG0, s); INV(1,-1,i_FN1O); break; // (u32* bc, S)
case FN2O: IMM(REG_ARG0, s); INV(1,-2,i_FN2O); break; // (u32* bc, S)
case ARRM: case ARRO: // (i64 sz, S)
u32 sz = *bc++;
if (sz) { IMM(REG_ARG0, sz); CCALL(i_ARR_p); }
else { CCALL(i_ARR_0); }
if (sz) { IMM(REG_ARG0, sz); INV(1,1-(i32)sz,i_ARR_p); }
else { INV(0, 1,i_ARR_0); }
break;
case DFND: // (u32* bc, Scope** pscs, Block* bl)
case DFND: // (u32* bc, Scope** pscs, Block* bl, S)
Block* bl = blocks[*bc++];
IMM(REG_ARG0,s); ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,bl);
u64 fn = (u64)(bl->ty==0? i_DFND_0 : bl->ty==1? i_DFND_1 : bl->ty==2? i_DFND_2 : NULL);
if (fn==0) thrM("JIT: Bad DFND argument");
CCALL(fn);
IMM(REG_ARG0,s); ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,bl); INV(3,1,fn);
break;
case OP1D: IMM(REG_ARG0,s); CCALL(i_OP1D) break; // (u32* bc)
case OP2D: IMM(REG_ARG0,s); CCALL(i_OP2D) break; // (u32* bc)
case OP2H: CCALL(i_OP2H) break;
case TR2D: CCALL(i_TR2D) break;
case TR3D: CCALL(i_TR3D) break;
case TR3O: CCALL(i_TR3O) break;
case LOCM: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); CCALL(i_LOCM); break; // (u32 d, u32 p)
case LOCO: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); CCALL(i_LOCO); break; // (u32 d, u32 p, Scope** pscs, u32* bc)
case LOCU: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); CCALL(i_LOCU); break; // (u32 d, u32 p, Scope** pscs)
case EXTM: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); CCALL(i_EXTM); break; // (u32 d, u32 p)
case EXTO: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); CCALL(i_EXTO); break; // (u32 d, u32 p, Scope** pscs, u32* bc)
case EXTU: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); CCALL(i_EXTU); break; // (u32 d, u32 p, Scope** pscs)
case SETN: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); CCALL(i_SETN); break; // (Scope** pscs, u32* bc)
case SETU: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); CCALL(i_SETU); break; // (Scope** pscs, u32* bc)
case SETM: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); CCALL(i_SETM); break; // (Scope** pscs, u32* bc)
case FLDO: IMM(REG_ARG0,*bc++); ASM(MOV,REG_ARG1,r_PSCS); CCALL(i_FLDO); break; // (u32 p, Scope** pscs)
case NSPM: IMM(REG_ARG0,*bc++); CCALL(i_NSPM); break; // (u32 l)
case RETD: ASM(MOV,REG_ARG0,r_PSCS); CCALL(i_RETD); ret=true; break; // (Scope** pscs)
case RETN: CCALL(i_RETN); ret=true; break;
case OP1D: IMM(REG_ARG0,s); INV(1,-1,i_OP1D) break; // (u32* bc, S)
case OP2D: IMM(REG_ARG0,s); INV(1,-2,i_OP2D) break; // (u32* bc, S)
case OP2H: INV(0,-1,i_OP2H) break; // (S)
case TR2D: INV(0,-1,i_TR2D) break; // (S)
case TR3D: INV(0,-2,i_TR3D) break; // (S)
case TR3O: INV(0,-2,i_TR3O) break; // (S)
case LOCM: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); INV(2,1,i_LOCM); break; // (u32 d, u32 p, S)
case LOCO: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); INV(4,1,i_LOCO); break; // (u32 d, u32 p, Scope** pscs, u32* bc, S)
case LOCU: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); INV(3,1,i_LOCU); break; // (u32 d, u32 p, Scope** pscs, S)
case EXTM: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); INV(2,1,i_EXTM); break; // (u32 d, u32 p, S)
case EXTO: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); INV(4,1,i_EXTO); break; // (u32 d, u32 p, Scope** pscs, u32* bc, S)
case EXTU: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); INV(3,1,i_EXTU); break; // (u32 d, u32 p, Scope** pscs, S)
case SETN: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); INV(2,-1,i_SETN); break; // (Scope** pscs, u32* bc, S)
case SETU: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); INV(2,-1,i_SETU); break; // (Scope** pscs, u32* bc, S)
case SETM: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); INV(2,-2,i_SETM); break; // (Scope** pscs, u32* bc, S)
case FLDO: IMM(REG_ARG0,*bc++); ASM(MOV,REG_ARG1,r_PSCS); INV(2,0,i_FLDO); break; // (u32 p, Scope** pscs, S)
case NSPM: IMM(REG_ARG0,*bc++); INV(1,0,i_NSPM); break; // (u32 l, S)
case RETD: ASM(MOV,REG_ARG0,r_PSCS); INV(1,0,i_RETD); ret=true; break; // (Scope** pscs, S); stack diff 0 is wrong, but updating it is useless
case RETN: INV(0,0,i_RETN); ret=true; break; // (S)
default: thrF("JIT: Unsupported bytecode %i", *s);
}
#undef L64
if (n!=bc) thrM("JIT: Wrong parsing of bytecode");
if (ret) break;
}
// ASM(POP, 14, -);
ASM(POP, 14, -);
ASM(POP, 13, -);
// ASM(POP, 12, -);
ASM(POP, 12, -);
ASM_RAW(bin, RET);
u64 sz = TSSIZE(bin);
@ -305,7 +337,7 @@ B evalJIT(Body* b, Scope* sc, u8* ptr) { // doesn't consume
for (i32 i = 0; i < b->maxPSC; i++) pscs[i+1] = pscs[i]->psc;
// write_asm(ptr, RFLD(ptr, TmpFile, a)->ia);
B r = ((JITFn*)ptr)(pscs);
B r = ((JITFn*)ptr)(gStack, pscs);
popEnv();
return r;

View File

@ -33,6 +33,7 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
// . r11 x
// . r12-r15 x
typedef unsigned char U;
typedef unsigned char UC;
typedef unsigned char Reg;
#define REG_RES 0
@ -244,3 +245,10 @@ typedef unsigned short RegM;
#define CMOVG(I,O) {REX8(O,I),0x0F,0x4F,A_REG(O,I)}
#define RET {0xC3}
#define ASM(INS, O, I) ASM_RAW(bin, INS(O, I))
#define IMM(O,I) { u64 v=(u64)(I); if(v==0) ASM(XOR,O,O); else if(v>=0 & v<(1ULL<<32)) ASM(MOV4_RI,O,v); else ASM(MOV_RI,O,v); }
#define ADDI(O, I) { i32 v=(i32)(I); if(v) { if(v==(i8)v) ASM(ADDI1,O,v); else ASM(ADDI4,O,v); } } // I must fit in i32
#define SUBI(O, I) { i32 v=(i32)(I); if(v) { if(v==(i8)v) ASM(SUBI1,O,v); else ASM(SUBI4,O,v); } } // I must fit in i32