diff --git a/src/core/stuff.h b/src/core/stuff.h index 124fb5ad..307541eb 100644 --- a/src/core/stuff.h +++ b/src/core/stuff.h @@ -201,7 +201,7 @@ static bool atomEqual(B w, B x) { // doesn't consume (not that that matters real #ifdef DEBUG - static Value* VALIDATEP(Value* x) { + static NOINLINE Value* VALIDATEP(Value* x) { if (x->refc<=0 || (x->refc>>28) == 'a' || x->type==t_empty) { printf("bad refcount for type %d: %d\nattempting to print: ", x->type, x->refc); fflush(stdout); print(tag(x,OBJ_TAG)); putchar('\n'); fflush(stdout); @@ -214,7 +214,7 @@ static bool atomEqual(B w, B x) { // doesn't consume (not that that matters real } return x; } - static B VALIDATE(B x) { + static NOINLINE B VALIDATE(B x) { if (!isVal(x)) return x; VALIDATEP(v(x)); if(isArr(x)!=TI(x).isArr && v(x)->type!=t_freed && v(x)->type!=t_harrPartial) { diff --git a/src/jit/nvm.c b/src/jit/nvm.c index 684c0bbf..0f334e53 100644 --- a/src/jit/nvm.c +++ b/src/jit/nvm.c @@ -6,6 +6,13 @@ #ifndef USE_PERF #define USE_PERF 0 // enable writing symbols to /tmp/perf-.map #endif +#ifndef CSTACK + #define CSTACK 1 +#endif +#ifdef GS_REALLOC + #undef CSTACK + #define CSTACK 0 +#endif // separate memory management system for executable code; isn't garbage-collected #define BSZ(X) (1ull<<(X)) @@ -27,52 +34,62 @@ static void* mmX_allocN(usz sz, u8 type) { assert(sz>=16); return mmX_allocL(BSZ // all the instructions to be called by the generated code -#define GSA(X) { B tr=X; *(gStack++) = tr; } -#define GSP (*--gStack) +#if CSTACK + #define GA0 B* cStack + #define GA1 ,B* cStack + #define GSA(X) { B tr=X; *(cStack++) = tr; } + #define GSP (*--cStack) + #define GS_UPD { gStack=cStack; } +#else + #define GA0 + #define GA1 + #define GSA(X) { B tr=X; *(gStack++) = tr; } + #define GSP (*--gStack) + #define GS_UPD +#endif #define P(N) B N=GSP; -#define GS_UPD #if VM_POS #define POS_UPD (envCurr-1)->bcL = bc-1; #else #define POS_UPD #endif #define INS NOINLINE -INS void i_POPS() { +INS void i_POPS(GA0) { dec(GSP); } -INS void i_PUSH() { - thrM("NYI PUSH in nvm"); -} -INS void i_ADDI(u64 v) { +// INS void i_PUSH(GA0) { +// thrM("NYI PUSH in nvm"); +// } +INS void i_ADDI(u64 v GA1) { B o = b(v); ptr_inc(v(o)); GSA(o); } -INS void i_ADDU(u64 v) { +INS void i_ADDU(u64 v GA1) { GSA(b(v)); } -INS void i_FN1C(u32* bc) { P(f)P(x) // TODO figure out a way to instead pass an offset in bc, so that shorter `mov`s can be used to pass it +INS void i_FN1C(u32* bc GA1) { P(f)P(x) // TODO figure out a way to instead pass an offset in bc, so that shorter `mov`s can be used to pass it GS_UPD;POS_UPD; GSA(c1(f, x)); dec(f); } -INS void i_FN1O(u32* bc) { P(f)P(x) +INS void i_FN1O(u32* bc GA1) { P(f)P(x) GS_UPD;POS_UPD; GSA(isNothing(x)? x : c1(f, x)); dec(f); } -INS void i_FN2C(u32* bc) { P(w)P(f)P(x) +INS void i_FN2C(u32* bc GA1) { P(w)P(f)P(x) GS_UPD;POS_UPD; GSA(c2(f, w, x)); dec(f); } -INS void i_FN2O(u32* bc) { P(w)P(f)P(x) +INS void i_FN2O(u32* bc GA1) { P(w)P(f)P(x) GS_UPD;POS_UPD; if (isNothing(x)) { dec(w); GSA(x); } else GSA(isNothing(w)? c1(f, x) : c2(f, w, x)); dec(f); } -INS void i_ARR_0() { +INS void i_ARR_0(GA0) { GSA(inc(bi_emptyHVec)); } -INS void i_ARR_p(i64 sz) { +INS void i_ARR_p(i64 sz GA1) { HArr_p r = m_harrUv(sz); bool allNum = true; assert(sz>0); @@ -82,71 +99,73 @@ INS void i_ARR_p(i64 sz) { GSA(withFill(r.b, m_f64(0))); } else GSA(r.b); } -INS void i_DFND_0(u32* bc, Scope** pscs, Block* bl) { GS_UPD;POS_UPD; GSA(m_funBlock(bl, *pscs)); } -INS void i_DFND_1(u32* bc, Scope** pscs, Block* bl) { GS_UPD;POS_UPD; GSA(m_md1Block(bl, *pscs)); } -INS void i_DFND_2(u32* bc, Scope** pscs, Block* bl) { GS_UPD;POS_UPD; GSA(m_md2Block(bl, *pscs)); } -INS void i_OP1D(u32* bc) { P(f)P(m) GS_UPD;POS_UPD; GSA(m1_d (m,f )); } -INS void i_OP2D(u32* bc) { P(f)P(m)P(g) GS_UPD;POS_UPD; GSA(m2_d (m,f,g)); } -INS void i_OP2H( ) { P(m)P(g) GSA(m2_h (m, g)); } -INS void i_TR2D( ) { P(g)P(h) GSA(m_atop( g,h)); } -INS void i_TR3D( ) { P(f)P(g)P(h) GSA(m_fork(f,g,h)); } -INS void i_TR3O( ) { P(f)P(g)P(h) +INS void i_DFND_0(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; GSA(m_funBlock(bl, *pscs)); } +INS void i_DFND_1(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; GSA(m_md1Block(bl, *pscs)); } +INS void i_DFND_2(u32* bc, Scope** pscs, Block* bl GA1) { GS_UPD;POS_UPD; GSA(m_md2Block(bl, *pscs)); } +INS void i_OP1D(u32* bc GA1) { P(f)P(m) GS_UPD;POS_UPD; GSA(m1_d (m,f )); } +INS void i_OP2D(u32* bc GA1) { P(f)P(m)P(g) GS_UPD;POS_UPD; GSA(m2_d (m,f,g)); } +INS void i_OP2H( GA0) { P(m)P(g) GSA(m2_h (m, g)); } +INS void i_TR2D( GA0) { P(g)P(h) GSA(m_atop( g,h)); } +INS void i_TR3D( GA0) { P(f)P(g)P(h) GSA(m_fork(f,g,h)); } +INS void i_TR3O( GA0) { P(f)P(g)P(h) if (isNothing(f)) { GSA(m_atop(g,h)); dec(f); } else GSA(m_fork(f,g,h)); } -INS void i_LOCM(u32 d, u32 p) { +INS void i_LOCM(u32 d, u32 p GA1) { GSA(tag((u64)d<<32 | (u32)p, VAR_TAG)); } -INS void i_LOCO(u32 d, u32 p, Scope** pscs, u32* bc) { +INS void i_LOCO(u32 d, u32 p, Scope** pscs, u32* bc GA1) { B l = pscs[d]->vars[p]; if(l.u==bi_noVar.u) { POS_UPD; thrM("Reading variable before its defined"); } GSA(inc(l)); } -INS void i_LOCU(u32 d, u32 p, Scope** pscs) { +INS void i_LOCU(u32 d, u32 p, Scope** pscs GA1) { B* vars = pscs[d]->vars; GSA(vars[p]); vars[p] = bi_optOut; } -INS void i_EXTM(u32 d, u32 p) { +INS void i_EXTM(u32 d, u32 p GA1) { GSA(tag((u64)d<<32 | (u32)p, EXT_TAG)); } -INS void i_EXTO(u32 d, u32 p, Scope** pscs, u32* bc) { +INS void i_EXTO(u32 d, u32 p, Scope** pscs, u32* bc GA1) { B l = pscs[d]->ext->vars[p]; if(l.u==bi_noVar.u) { POS_UPD; thrM("Reading variable before its defined"); } GSA(inc(l)); } -INS void i_EXTU(u32 d, u32 p, Scope** pscs) { +INS void i_EXTU(u32 d, u32 p, Scope** pscs GA1) { B* vars = pscs[d]->ext->vars; GSA(vars[p]); vars[p] = bi_optOut; } -INS void i_SETN(Scope** pscs, u32* bc) { P(s) P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, false); dec(s); GSA(x); } -INS void i_SETU(Scope** pscs, u32* bc) { P(s) P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, true ); dec(s); GSA(x); } -INS void i_SETM(Scope** pscs, u32* bc) { P(s)P(f)P(x) GS_UPD; POS_UPD; +INS void i_SETN(Scope** pscs, u32* bc GA1) { P(s) P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, false); dec(s); GSA(x); } +INS void i_SETU(Scope** pscs, u32* bc GA1) { P(s) P(x) GS_UPD; POS_UPD; v_set(pscs, s, x, true ); dec(s); GSA(x); } +INS void i_SETM(Scope** pscs, u32* bc GA1) { P(s)P(f)P(x) GS_UPD; POS_UPD; B w = v_get(pscs, s); B r = c2(f,w,x); dec(f); v_set(pscs, s, r, true); dec(s); GSA(r); } -INS void i_FLDO(u32 p, Scope** pscs) { P(ns) GS_UPD; +INS void i_FLDO(u32 p, Scope** pscs GA1) { P(ns) GS_UPD; if (!isNsp(ns)) thrM("Trying to read a field from non-namespace"); GSA(inc(ns_getU(ns, pscs[0]->body->nsDesc->nameList, p))); dec(ns); } -INS void i_NSPM(u32 l) { P(o) +INS void i_NSPM(u32 l GA1) { P(o) B a = mm_alloc(sizeof(FldAlias), t_fldAlias, ftag(OBJ_TAG)); c(FldAlias,a)->obj = o; c(FldAlias,a)->p = l; GSA(a); } -INS B i_RETD(Scope** pscs) { +INS B i_RETD(Scope** pscs GA1) { Scope* sc = pscs[0]; Body* b = sc->body; ptr_inc(sc); ptr_inc(b->nsDesc); + GS_UPD; return m_ns(sc, b->nsDesc); } -INS B i_RETN() { P(v) +INS B i_RETN(GA0) { P(v) + GS_UPD; return v; } @@ -156,6 +175,8 @@ INS B i_RETN() { P(v) #undef GSA #undef GS_UPD #undef POS_UPD +#undef GA0 +#undef GA1 @@ -164,8 +185,6 @@ INS B i_RETN() { P(v) #include "x86_64.h" -#define IMM(O,I) { u64 v=(u64)(I); if(v==0) ASM_RAW(bin, XOR(O,O)); else if(v>=0 & v<(1ULL<<32)) ASM_RAW(bin, MOV4_RI(O,v)); else ASM_RAW(bin, MOV_RI(O,v)); } -#define ASM(INS, O, I) ASM_RAW(bin, INS(O, I)) #if USE_PERF #include @@ -200,17 +219,20 @@ static u32 readBytes4(u8* d) { return d[0] | d[1]<<8 | d[2]<<16 | d[3]<<24; } -typedef B JITFn(Scope** pscs); +typedef B JITFn(B* cStack, Scope** pscs); u8* m_nvm(Body* body) { TSALLOC(u8, bin, 64); TSALLOC(u32, rel, 64); + #define r_TMP 12 #define r_PSCS 13 - // ASM(PUSH, 12, -); + #define r_CS 14 + ASM(PUSH, 12, -); ASM(PUSH, 13, -); - // ASM(PUSH, 14, -); - ASM(MOV, r_PSCS, REG_ARG0); - // #define CCALL(F) { IMM(12, F); ASM(CALL, 12, -); } - #define CCALL(F) { if((u64)F < 1ULL<<31) { TSADD(rel, TSSIZE(bin)); ASM(CALLI, (u32)F, -); } else { IMM(12, F); ASM(CALL, 12, -); } } + ASM(PUSH, 14, -); + ASM(MOV, r_CS , REG_ARG0); + ASM(MOV, r_PSCS, REG_ARG1); + // #define CCALL(F) { IMM(r_TMP, F); ASM(CALL, r_TMP, -); } + #define CCALL(F) { if((u64)F < 1ULL<<31) { TSADD(rel, TSSIZE(bin)); ASM(CALLI, (u32)F, -); } else { IMM(r_TMP, F); ASM(CALL, r_TMP, -); } } u32* bc = body->bc; Block** blocks = body->blocks->a; while (true) { @@ -218,54 +240,64 @@ u8* m_nvm(Body* body) { u32* n = nextBC(bc); bool ret = false; #define L64 ({ u64 r = bc[0] | ((u64)bc[1])<<32; bc+= 2; r; }) + #if CSTACK + #define INV(N,D,F) ASM(MOV,REG_ARG##N, r_CS); ADDI(r_CS,(D)*sizeof(B)); CCALL(F) + #else + #define INV(N,D,F) CCALL(F) // N - stack argument number; D - expected stack delta; F - called function + #endif switch (*bc++) { - case POPS: CCALL(i_POPS); break; - case ADDI: IMM(REG_ARG0, L64); CCALL(i_ADDI); break; // (u64 v) - case ADDU: IMM(REG_ARG0, L64); CCALL(i_ADDU); break; // (u64 v) - case FN1C: IMM(REG_ARG0, s); CCALL(i_FN1C); break; // (u32* bc) - case FN2C: IMM(REG_ARG0, s); CCALL(i_FN2C); break; // (u32* bc) - case FN1O: IMM(REG_ARG0, s); CCALL(i_FN1O); break; // (u32* bc) - case FN2O: IMM(REG_ARG0, s); CCALL(i_FN2O); break; // (u32* bc) - case ARRM: case ARRO: // (i64 sz) + case POPS: INV(0,-1,i_POPS); break; // (S) + case ADDI: IMM(REG_ARG0, L64); INV(1,1,i_ADDI); break; // (u64 v, S) + case ADDU: + #if CSTACK + IMM(r_TMP, L64); ASM(MOV_MR0, r_CS, r_TMP); ADDI(r_CS,sizeof(B)); + #else + IMM(REG_ARG0, L64); INV(1,1,i_ADDU); + #endif + break; // (u64 v, S) + case FN1C: IMM(REG_ARG0, s); INV(1,-1,i_FN1C); break; // (u32* bc, S) + case FN2C: IMM(REG_ARG0, s); INV(1,-2,i_FN2C); break; // (u32* bc, S) + case FN1O: IMM(REG_ARG0, s); INV(1,-1,i_FN1O); break; // (u32* bc, S) + case FN2O: IMM(REG_ARG0, s); INV(1,-2,i_FN2O); break; // (u32* bc, S) + case ARRM: case ARRO: // (i64 sz, S) u32 sz = *bc++; - if (sz) { IMM(REG_ARG0, sz); CCALL(i_ARR_p); } - else { CCALL(i_ARR_0); } + if (sz) { IMM(REG_ARG0, sz); INV(1,1-(i32)sz,i_ARR_p); } + else { INV(0, 1,i_ARR_0); } break; - case DFND: // (u32* bc, Scope** pscs, Block* bl) + case DFND: // (u32* bc, Scope** pscs, Block* bl, S) Block* bl = blocks[*bc++]; - IMM(REG_ARG0,s); ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,bl); u64 fn = (u64)(bl->ty==0? i_DFND_0 : bl->ty==1? i_DFND_1 : bl->ty==2? i_DFND_2 : NULL); if (fn==0) thrM("JIT: Bad DFND argument"); - CCALL(fn); + IMM(REG_ARG0,s); ASM(MOV,REG_ARG1,r_PSCS); IMM(REG_ARG2,bl); INV(3,1,fn); break; - case OP1D: IMM(REG_ARG0,s); CCALL(i_OP1D) break; // (u32* bc) - case OP2D: IMM(REG_ARG0,s); CCALL(i_OP2D) break; // (u32* bc) - case OP2H: CCALL(i_OP2H) break; - case TR2D: CCALL(i_TR2D) break; - case TR3D: CCALL(i_TR3D) break; - case TR3O: CCALL(i_TR3O) break; - case LOCM: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); CCALL(i_LOCM); break; // (u32 d, u32 p) - case LOCO: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); CCALL(i_LOCO); break; // (u32 d, u32 p, Scope** pscs, u32* bc) - case LOCU: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); CCALL(i_LOCU); break; // (u32 d, u32 p, Scope** pscs) - case EXTM: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); CCALL(i_EXTM); break; // (u32 d, u32 p) - case EXTO: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); CCALL(i_EXTO); break; // (u32 d, u32 p, Scope** pscs, u32* bc) - case EXTU: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); CCALL(i_EXTU); break; // (u32 d, u32 p, Scope** pscs) - case SETN: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); CCALL(i_SETN); break; // (Scope** pscs, u32* bc) - case SETU: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); CCALL(i_SETU); break; // (Scope** pscs, u32* bc) - case SETM: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); CCALL(i_SETM); break; // (Scope** pscs, u32* bc) - case FLDO: IMM(REG_ARG0,*bc++); ASM(MOV,REG_ARG1,r_PSCS); CCALL(i_FLDO); break; // (u32 p, Scope** pscs) - case NSPM: IMM(REG_ARG0,*bc++); CCALL(i_NSPM); break; // (u32 l) - case RETD: ASM(MOV,REG_ARG0,r_PSCS); CCALL(i_RETD); ret=true; break; // (Scope** pscs) - case RETN: CCALL(i_RETN); ret=true; break; + case OP1D: IMM(REG_ARG0,s); INV(1,-1,i_OP1D) break; // (u32* bc, S) + case OP2D: IMM(REG_ARG0,s); INV(1,-2,i_OP2D) break; // (u32* bc, S) + case OP2H: INV(0,-1,i_OP2H) break; // (S) + case TR2D: INV(0,-1,i_TR2D) break; // (S) + case TR3D: INV(0,-2,i_TR3D) break; // (S) + case TR3O: INV(0,-2,i_TR3O) break; // (S) + case LOCM: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); INV(2,1,i_LOCM); break; // (u32 d, u32 p, S) + case LOCO: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); INV(4,1,i_LOCO); break; // (u32 d, u32 p, Scope** pscs, u32* bc, S) + case LOCU: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); INV(3,1,i_LOCU); break; // (u32 d, u32 p, Scope** pscs, S) + case EXTM: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); INV(2,1,i_EXTM); break; // (u32 d, u32 p, S) + case EXTO: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); IMM(REG_ARG3,s); INV(4,1,i_EXTO); break; // (u32 d, u32 p, Scope** pscs, u32* bc, S) + case EXTU: IMM(REG_ARG0,*bc++); IMM(REG_ARG1,*bc++); ASM(MOV,REG_ARG2,r_PSCS); INV(3,1,i_EXTU); break; // (u32 d, u32 p, Scope** pscs, S) + case SETN: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); INV(2,-1,i_SETN); break; // (Scope** pscs, u32* bc, S) + case SETU: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); INV(2,-1,i_SETU); break; // (Scope** pscs, u32* bc, S) + case SETM: ASM(MOV,REG_ARG0,r_PSCS); IMM(REG_ARG1,s); INV(2,-2,i_SETM); break; // (Scope** pscs, u32* bc, S) + case FLDO: IMM(REG_ARG0,*bc++); ASM(MOV,REG_ARG1,r_PSCS); INV(2,0,i_FLDO); break; // (u32 p, Scope** pscs, S) + case NSPM: IMM(REG_ARG0,*bc++); INV(1,0,i_NSPM); break; // (u32 l, S) + case RETD: ASM(MOV,REG_ARG0,r_PSCS); INV(1,0,i_RETD); ret=true; break; // (Scope** pscs, S); stack diff 0 is wrong, but updating it is useless + case RETN: INV(0,0,i_RETN); ret=true; break; // (S) default: thrF("JIT: Unsupported bytecode %i", *s); } #undef L64 if (n!=bc) thrM("JIT: Wrong parsing of bytecode"); if (ret) break; } - // ASM(POP, 14, -); + ASM(POP, 14, -); ASM(POP, 13, -); - // ASM(POP, 12, -); + ASM(POP, 12, -); ASM_RAW(bin, RET); u64 sz = TSSIZE(bin); @@ -305,7 +337,7 @@ B evalJIT(Body* b, Scope* sc, u8* ptr) { // doesn't consume for (i32 i = 0; i < b->maxPSC; i++) pscs[i+1] = pscs[i]->psc; // write_asm(ptr, RFLD(ptr, TmpFile, a)->ia); - B r = ((JITFn*)ptr)(pscs); + B r = ((JITFn*)ptr)(gStack, pscs); popEnv(); return r; diff --git a/src/jit/x86_64.h b/src/jit/x86_64.h index 6e3c298b..68bfba07 100644 --- a/src/jit/x86_64.h +++ b/src/jit/x86_64.h @@ -33,6 +33,7 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. // . r11 x // . r12-r15 x +typedef unsigned char U; typedef unsigned char UC; typedef unsigned char Reg; #define REG_RES 0 @@ -244,3 +245,10 @@ typedef unsigned short RegM; #define CMOVG(I,O) {REX8(O,I),0x0F,0x4F,A_REG(O,I)} #define RET {0xC3} + + +#define ASM(INS, O, I) ASM_RAW(bin, INS(O, I)) +#define IMM(O,I) { u64 v=(u64)(I); if(v==0) ASM(XOR,O,O); else if(v>=0 & v<(1ULL<<32)) ASM(MOV4_RI,O,v); else ASM(MOV_RI,O,v); } +#define ADDI(O, I) { i32 v=(i32)(I); if(v) { if(v==(i8)v) ASM(ADDI1,O,v); else ASM(ADDI4,O,v); } } // I must fit in i32 +#define SUBI(O, I) { i32 v=(i32)(I); if(v) { if(v==(i8)v) ASM(SUBI1,O,v); else ASM(SUBI4,O,v); } } // I must fit in i32 +