diff --git a/src/jit/nvm.c b/src/jit/nvm.c index 75829a3d..f55ae693 100644 --- a/src/jit/nvm.c +++ b/src/jit/nvm.c @@ -403,7 +403,7 @@ static u32 readBytes4(u8* d) { static void write_asm(u8* p, u64 sz); static void asm_test() { ALLOC_ASM(64); - for (int i = 0; i < 16; i++) for (int j = 0; j < 16; j++) for (int k = 0; k < 16; k++) BZHI(i,j,k); + for (int i = 0; i < 16; i++) CALLi(123); GET_ASM(); write_asm(bin, ASM_SIZE); exit(0); @@ -430,29 +430,29 @@ typedef B JITFn(B* cStack, Scope** pscs, Scope* sc); static inline i32 maxi32(i32 a, i32 b) { return a>b?a:b; } Nvm_res m_nvm(Body* body) { ALLOC_ASM(64); - TSALLOC(u32, rel, 64); + TSALLOC(u32, rel, 64); // TODO move to x86_64.h #if ASM_TEST asm_test(); #endif - Reg r_TMP = 3; // TODO this doesn't really need to be non-volatile + Reg r_ENV = 15; Reg r_PSCS = 14; - Reg r_CS = 15; + Reg r_CS = 3; Reg r_SC = 12; PUSH(5); - PUSH(r_TMP); - PUSH(r_PSCS); - PUSH(r_CS); - PUSH(r_SC); + PUSH(r_ENV); + PUSH(r_PSCS); // Scope* pscs[] + PUSH(r_CS); // starting gStack + PUSH(r_SC); // Scope* sc MOV(r_CS , R_A0); MOV(r_PSCS, R_A1); MOV(r_SC , R_A2); + MOV8rp(r_ENV, (u64)&envCurr - 4); TSADD(rel, ASM_SIZE-4); for (i32 i = 1; i < body->maxPSC+1; i++) { MOV8rmo(R_A2, R_A2, offsetof(Scope, psc)); MOV8mro(r_PSCS, R_A2, i*8); } - if ((u64)i_SETN != (u32)(u64)i_SETN) thrM("JIT: Refusing to run with CBQN code outside of the 32-bit address range"); - // #define CCALL(F) { IMM(r_TMP, F); CALL(r_TMP); } - #define CCALL(F) { TSADD(rel, ASM_SIZE); CALLi(F); } + if ((u64)i_RETD > I32_MAX || (u64)&gStack > I32_MAX || (u64)&envEnd > I32_MAX) thrM("JIT: Refusing to run with CBQN code outside of the 32-bit address range"); + #define CCALL(F) { u64 f=(u64)(F); if(f>I32_MAX)thrM("JIT: Function address too large for call"); CALLi(f-4); TSADD(rel, ASM_SIZE-4); } u32* origBC = body->bc; OptRes optRes = opt(origBC); Block** blocks = body->blocks->a; @@ -465,8 +465,9 @@ Nvm_res m_nvm(Body* body) { bool ret = false; #define L64 ({ u64 r = bc[0] | ((u64)bc[1])<<32; bc+= 2; r; }) // #define LEA0(O,I,OFF) { MOV(O,I); ADDI(O,OFF); } - #define LEA0(O,I,OFF,Q) ({ i32 o=(OFF); if(o) LEAi(O,I,o); else if (Q) MOV(O,I); o?O:I; }) - #define SPOS(R,N,Q) LEA0(R, r_CS, maxi32(0, depth+(N)-1)*sizeof(B),Q) + #define LEA0(O,I,OFF,Q) ({ i32 o=(OFF); if (Q||o) LEAi(O,I,o); o?O:I; }) + #define SPOSq(N) (maxi32(0, depth+(N)-1) * sizeof(B)) + #define SPOS(R,N,Q) LEA0(R, r_CS, SPOSq(N), Q) // load stack position N in register R; if !Q, then might not write and instead return another register which will have the wanted value #if CSTACK // #define INV(N,D,F) MOV(R_A##N,r_CS); ADDI(r_CS,(D)*sizeof(B)); CCALL(F) #define INV(N,D,F) SPOS(R_A##N, D, 1); CCALL(F) @@ -474,20 +475,21 @@ Nvm_res m_nvm(Body* body) { #define INV(N,D,F) CCALL(F) // N - stack argument number; D - expected stack delta; F - called function; TODO instrs which don't need stack (POPS and things with stack delta 1) #endif #define TOPp MOV(R_A0,R_RES) - #define TOPs if (depth) { u8 t = SPOS(r_TMP, 0, 0); MOV8mr(t, R_RES); } + #define TOPs if (depth) { u8 t = SPOS(R_A3, 0, 0); MOV8mr(t, R_RES); } #define LSC(R,D) { if(D) MOV8rmo(R,r_PSCS,D*8); else MOV(R,r_SC); } - #define INCV(R) INC4mo(R, offsetof(Value,refc)); // ADD4mi(r_TMP, 1); CCALL(i_INC); + #define INCV(R) INC4mo(R, offsetof(Value,refc)); // ADD4mi(R_A3, 1); CCALL(i_INC); #ifdef __BMI2__ // TODO move to runtime detection maybe #define INCB(R,T,U) IMM(T,0xfffffffffffffull);ADD(T,R);IMM(U,0x7fffffffffffeull);CMP(T,U);{JA(lI);MOVi1l(U,0x30);BZHI(U,R,U);INCV(U);LBL1(lI);} #else #define INCB(R,T,U) IMM(T,0xfffffffffffffull);ADD(T,R);IMM(U,0x7fffffffffffeull);CMP(T,U);{JA(lI);IMM(U,0xffffffffffffull);AND(U,R);INCV(U);LBL1(lI);} #endif + #define POS_UPD(R1,R2) IMM(R1, off); MOV8mro(r_ENV, R1, offsetof(Env,bcL)); switch (*bc++) { case POPS: TOPp; CCALL(i_POPS); - if (depth>1) { u8 t = SPOS(r_TMP, -1, 0); MOV8rm(R_RES, t); } + if (depth>1) { u8 t = SPOS(R_A3, -1, 0); MOV8rm(R_RES, t); } break; - case ADDI: TOPs; { u64 x = L64; IMM(R_RES, x); IMM(r_TMP, v(b(x))); INCV(r_TMP); break; } // (u64 v, S) + case ADDI: TOPs; { u64 x = L64; IMM(R_RES, x); IMM(R_A3, v(b(x))); INCV(R_A3); break; } // (u64 v, S) case ADDU: TOPs; // (u64 v, S) #if CSTACK IMM(R_RES, L64); @@ -499,8 +501,19 @@ Nvm_res m_nvm(Body* body) { case FN2C: TOPp; IMM(R_A1,off); INV(2,0,i_FN2C); break; // (B, u32* bc, S) case FN1O: TOPp; IMM(R_A1,off); INV(2,0,i_FN1O); break; // (B, u32* bc, S) case FN2O: TOPp; IMM(R_A1,off); INV(2,0,i_FN2O); break; // (B, u32* bc, S) - case FN1Ci:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN1Ci); break; // (B, BB2B fm, u32* bc, S) - case FN2Ci:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN2Ci); break; // (B, BBB2B fd, u32* bc, S) + case FN1Ci: { u64 fn = L64; POS_UPD(R_A0,R_A3); + MOV(R_A1, R_RES); + MOV8pr((u64)&gStack - 4, SPOS(R_A3, 0, 0)); TSADD(rel, ASM_SIZE-4); // GS_UPD + CCALL(fn); + } break; + case FN2Ci: { u64 fn = L64; POS_UPD(R_A0,R_A3); + Reg r_sp = SPOS(R_A2, -1, 0); + MOV8pr((u64)&gStack - 4, r_sp); TSADD(rel, ASM_SIZE-4); // GS_UPD + MOV(R_A1, R_RES); MOV8rm(R_A2, r_sp); // load args + CCALL(fn); + } break; + // case FN1Ci:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN1Ci); break; // (B, BB2B fm, u32* bc, S) + // case FN2Ci:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN2Ci); break; // (B, BBB2B fd, u32* bc, S) case FN1Oi:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN1Oi); break; // (B, BB2B fm, u32* bc, S) case FN2Oi:TOPp; IMM(R_A1,L64); IMM(R_A2, L64); IMM(R_A3,off); INV(4,0,i_FN2Oi); break; // (B, BB2B fm, BBB2B fd, u32* bc, S) case ARRM: case ARRO:; @@ -540,7 +553,8 @@ Nvm_res m_nvm(Body* body) { case NSPM: TOPp; IMM(R_A1,*bc++); CCALL(i_NSPM); break; // (B, u32 l, S) case CHKV: TOPp; IMM(R_A1,off); INV(2,0,i_CHKV); break; // (B, u32* bc, S) case RETD: MOV(R_A0,r_SC); INV(1,1,i_RETD); ret=true; break; // (Scope* sc, S); stack diff 0 is wrong, but updating it is useless - case RETN: IMM(r_TMP, &gStack); MOV8mr(r_TMP, r_CS); ret=true; break; + // case RETN: IMM(R_A3, &gStack); MOV8mr(R_A3, r_CS); ret=true; break; + case RETN: MOV8pr((u64)&gStack - 4, r_CS); TSADD(rel, ASM_SIZE-4); ret=true; break; default: thrF("JIT: Unsupported bytecode %i", *s); } #undef INCB @@ -559,7 +573,7 @@ Nvm_res m_nvm(Body* body) { POP(r_SC); POP(r_CS); POP(r_PSCS); - POP(r_TMP); + POP(r_ENV); POP(5); RET(); #undef CCALL @@ -576,19 +590,20 @@ Nvm_res m_nvm(Body* body) { u32 bcPos = body->map[0]; // printf("JIT %d:\n", perfid); // vm_printPos(body->comp, bcPos, -1); - fprintf(perf_map, "%lx %lx JIT %d: BC@%u\n", (u64)binEx, sz, perfid++, bcPos); + fprintf(perf_map, N64x" "N64x" JIT %d: BC@%u\n", (u64)binEx, sz, perfid++, bcPos); #endif memcpy(binEx, bin, sz); u64 relAm = TSSIZE(rel); // printf("allocated at %p; i_ADDU: %p\n", binEx, i_ADDU); for (u64 i = 0; i < relAm; i++) { u8* ins = binEx+rel[i]; - u32 o = readBytes4(ins+1); - u32 n = o-(u32)(u64)ins-5; - memcpy(ins+1, (u8[]){BYTES4(n)}, 4); + u32 o = readBytes4(ins); + u32 n = o-(u32)(u64)ins; + memcpy(ins, (u8[]){BYTES4(n)}, 4); } #if WRITE_ASM write_asm(binEx, sz); + // exit(0); #endif FREE_ASM(); TSFREE(rel); @@ -601,9 +616,9 @@ B evalJIT(Body* b, Scope* sc, u8* ptr) { // doesn't consume Scope* pscs[b->maxPSC+1]; pscs[0] = sc; - B* sp = gStack; + // B* sp = gStack; B r = ((JITFn*)ptr)(gStack, pscs, sc); - if (sp!=gStack) thrM("uh oh"); + // if (sp!=gStack) thrM("uh oh"); popEnv(); return r; diff --git a/src/jit/x86_64.h b/src/jit/x86_64.h index 21cbf1b3..57a8cfcf 100644 --- a/src/jit/x86_64.h +++ b/src/jit/x86_64.h @@ -61,7 +61,6 @@ typedef u8 Reg; #define R_P4 15 // r15 - #define ALLOC_ASM(N) TStack* b_o = (TStack*)mm_allocN(sizeof(TStack)+(N), t_temp); b_o->size=0; b_o->cap=(N); #define GET_ASM() u8* bin = b_o->data; #define AADD(P,N) b_o=asm_add(b_o, P, N) @@ -350,6 +349,7 @@ static inline void asm_a(TStack* o, u64 len, u8 v[]) { if(CHK4(O)) ASM1(0x24); \ if(b4) ASM4(t); else if (b1) ASM1(t); \ } +#define MRMp(OFF,I) ASM1((((I)&7)<<3) + 5); ASM4(OFF); // offset to rip #define nA_0REG(O,I) (((I)&7)<<3) + ((O)&7) #define nA_REG(O,I) ASM1(0xC0 + nA_0REG(O,I)) // aka MRM immediate @@ -359,7 +359,7 @@ static inline void asm_a(TStack* o, u64 len, u8 v[]) { #define AC3(N,A,B,C) (i##N(b_o=asm_r(b_o),A,B,C)) // meaning of lowercase after basic instr name: -// 'r' means the corresponding argument is the direct register contents, 'm' - that it's dereferenced; if all are 'r' or they don't have other options, they can be omitted +// 'r' means the corresponding argument is the direct register contents, 'm' - that it's dereferenced, 'p' - offset to rip; if all are 'r' or they don't have other options, they can be omitted // add 'o' for an offset argument (e.g. the 5 in 'mov rax, [rdi+5]'), 'i' for an immediate (e.g. 5 in 'add r12,5', or in 'lea rax, [rdi+5]') #define INC4mo(O,IMM) AC2(INC4mo,O,IMM) #define INC8mo(O,IMM) AC2(INC8mo,O,IMM) @@ -386,10 +386,12 @@ static inline void asm_a(TStack* o, u64 len, u8 v[]) { #define MOV8mr(I,O) AC2(MOV8mr,I,O) // *(u64*)(nullptr + I) ← O #define MOV4rm(O,I) AC2(MOV4rm,O,I) // O ← *(u32*)(nullptr + I) #define MOV4mr(I,O) AC2(MOV4mr,I,O) // *(u32*)(nullptr + I) ← O -#define MOV8rmo(O,I,IMM) AC3(MOV8rmo,O,I,IMM) // O ← *(u64*)(nullptr + I + IMM) -#define MOV8mro(I,O,IMM) AC3(MOV8mro,I,O,IMM) // *(u64*)(nullptr + I + IMM) ← O -#define MOV4rmo(O,I,IMM) AC3(MOV4rmo,O,I,IMM) -#define MOV4mro(I,O,IMM) AC3(MOV4mro,I,O,IMM) +#define MOV8rmo(O,I,OFF) AC3(MOV8rmo,O,I,OFF) // O ← *(u64*)(nullptr + I + OFF) +#define MOV8mro(I,O,OFF) AC3(MOV8mro,I,O,OFF) // *(u64*)(nullptr + I + OFF) ← O +#define MOV4rmo(O,I,OFF) AC3(MOV4rmo,O,I,OFF) +#define MOV4mro(I,O,OFF) AC3(MOV4mro,I,O,OFF) +#define MOV8pr(OFF,I) AC2(MOV8pr,OFF,I) +#define MOV8rp(I,OFF) AC2(MOV8rp,OFF,I) #define LEAi(O,I,IMM) AC3(LEAi,O,I,IMM) #define BZHI(O,I,N) AC3(BZHI,O,I,N) // requires __BMI2__ @@ -397,7 +399,7 @@ static inline void asm_a(TStack* o, u64 len, u8 v[]) { #define PUSH(O) AC1(PUSH,O) #define POP(O) AC1(POP,O) #define CALL(IMM) AC1(CALL,IMM) -#define CALLi(I) AC1(CALLi,(i32)(i64)(I)) +#define CALLi(I) AC1(CALLi,(i64)(I)) // I must be 32-bit #define RET() {b_o=asm_r(b_o); ASM1(0xC3);} #define JO(L) u64 L=ASM_SIZE; {b_o=asm_r(b_o); ASM1(0x70);ASM1(-2);} @@ -445,11 +447,12 @@ ASMI(MOVi, Reg o, i64 i) { else { nREX8(o,0); ASM1(0xB8+(o&7)); ASM8(i); } } ASMI(MOVi1l, Reg o, i8 imm) { if (o>=4) ASM1(o>=8?0x41:0x40); ASM1(0xb0+(o&7)); ASM1(imm); } - ASMI(MOV8mr, Reg o, Reg i) { nREX8(o,i); ASM1(0x89); MRM(o,i); } ASMI(MOV8mro, Reg o, Reg i, i32 off) { nREX8(o,i); ASM1(0x89); MRMo(o,i, off); } ASMI(MOV8rm, Reg i, Reg o) { nREX8(o,i); ASM1(0x8B); MRM(o,i); } ASMI(MOV8rmo, Reg i, Reg o, i32 off) { nREX8(o,i); ASM1(0x8B); MRMo(o,i, off); } ASMI(MOV4mr, Reg o, Reg i) { nREX4(o,i); ASM1(0x89); MRM(o,i); } ASMI(MOV4mro, Reg o, Reg i, i32 off) { nREX4(o,i); ASM1(0x89); MRMo(o,i, off); } ASMI(MOV4rm, Reg i, Reg o) { nREX4(o,i); ASM1(0x8B); MRM(o,i); } ASMI(MOV4rmo, Reg i, Reg o, i32 off) { nREX4(o,i); ASM1(0x8B); MRMo(o,i, off); } +ASMI(MOV8pr,i32 off, Reg i) { nREX8(0,i); ASM1(0x89); MRMp(off,i); } +ASMI(MOV8rp,i32 off, Reg i) { nREX8(0,i); ASM1(0x8B); MRMp(off,i); } ASMI(LEAi, Reg o, Reg i, i32 imm) { if(imm==0) iMOV(b_o,o,i); else { nREX8(i,o); ASM1(0x8D); MRMo(i,o,imm); } } ASMI(BZHI, Reg o, Reg i, Reg n) { ASM1(0xC4); ASM1(0x42+(i<8)*0x20 + (o<8)*0x80); ASM1(0xf8-n*8); ASM1(0xF5); nA_REG(i, o); } @@ -458,6 +461,6 @@ ASMI(PUSH, Reg O) { nREX4(O,0); ASM1(0x50+((O)&7)); } ASMI(POP , Reg O) { nREX4(O,0); ASM1(0x58+((O)&7)); } ASMI(CALL, Reg i) { nREX4(i,0); ASM1(0xFF); nA_REG(i,2); } -ASMI(CALLi, i32 imm) { ASM1(0xE8); ASM4(imm); } +ASMI(CALLi, i32 imm) { ASM1(0xE8); if (imm>I32_MAX)err("immediate call outside of 32-bit range!"); ASM4(imm); } #define IMM(A,B) MOVi(A,(u64)(B)) \ No newline at end of file diff --git a/src/vm.c b/src/vm.c index 6c6667cc..23217ae5 100644 --- a/src/vm.c +++ b/src/vm.c @@ -95,10 +95,12 @@ B* gStackEnd; void gsPrint() { B* c = gStackStart; i32 i = 0; + printf("gStack %p, height "N64d":\n", gStackStart, gStack-gStackStart); while (c!=gStack) { - printf("%d: ", i); - print(*c); - printf(", refc=%d\n", v(*c)->refc); + printf(" %d: ", i); fflush(stdout); + print(*c); fflush(stdout); + if (isVal(*c)) printf(", refc=%d", v(*c)->refc); fflush(stdout); + printf("\n"); c++; i++; }