inline i_FN1Ci and i_FN2Ci
This commit is contained in:
parent
8b561a065a
commit
eac62d4c29
@ -403,7 +403,7 @@ static u32 readBytes4(u8* d) {
|
||||
static void write_asm(u8* p, u64 sz);
|
||||
static void asm_test() {
|
||||
ALLOC_ASM(64);
|
||||
for (int i = 0; i < 16; i++) for (int j = 0; j < 16; j++) for (int k = 0; k < 16; k++) BZHI(i,j,k);
|
||||
for (int i = 0; i < 16; i++) CALLi(123);
|
||||
GET_ASM();
|
||||
write_asm(bin, ASM_SIZE);
|
||||
exit(0);
|
||||
@ -430,29 +430,29 @@ typedef B JITFn(B* cStack, Scope** pscs, Scope* sc);
|
||||
static inline i32 maxi32(i32 a, i32 b) { return a>b?a:b; }
|
||||
Nvm_res m_nvm(Body* body) {
|
||||
ALLOC_ASM(64);
|
||||
TSALLOC(u32, rel, 64);
|
||||
TSALLOC(u32, rel, 64); // TODO move to x86_64.h
|
||||
#if ASM_TEST
|
||||
asm_test();
|
||||
#endif
|
||||
Reg r_TMP = 3; // TODO this doesn't really need to be non-volatile
|
||||
Reg r_ENV = 15;
|
||||
Reg r_PSCS = 14;
|
||||
Reg r_CS = 15;
|
||||
Reg r_CS = 3;
|
||||
Reg r_SC = 12;
|
||||
PUSH(5);
|
||||
PUSH(r_TMP);
|
||||
PUSH(r_PSCS);
|
||||
PUSH(r_CS);
|
||||
PUSH(r_SC);
|
||||
PUSH(r_ENV);
|
||||
PUSH(r_PSCS); // Scope* pscs[]
|
||||
PUSH(r_CS); // starting gStack
|
||||
PUSH(r_SC); // Scope* sc
|
||||
MOV(r_CS , R_A0);
|
||||
MOV(r_PSCS, R_A1);
|
||||
MOV(r_SC , R_A2);
|
||||
MOV8rp(r_ENV, (u64)&envCurr - 4); TSADD(rel, ASM_SIZE-4);
|
||||
for (i32 i = 1; i < body->maxPSC+1; i++) {
|
||||
MOV8rmo(R_A2, R_A2, offsetof(Scope, psc));
|
||||
MOV8mro(r_PSCS, R_A2, i*8);
|
||||
}
|
||||
if ((u64)i_SETN != (u32)(u64)i_SETN) thrM("JIT: Refusing to run with CBQN code outside of the 32-bit address range");
|
||||
// #define CCALL(F) { IMM(r_TMP, F); CALL(r_TMP); }
|
||||
#define CCALL(F) { TSADD(rel, ASM_SIZE); CALLi(F); }
|
||||
if ((u64)i_RETD > I32_MAX || (u64)&gStack > I32_MAX || (u64)&envEnd > I32_MAX) thrM("JIT: Refusing to run with CBQN code outside of the 32-bit address range");
|
||||
#define CCALL(F) { u64 f=(u64)(F); if(f>I32_MAX)thrM("JIT: Function address too large for call"); CALLi(f-4); TSADD(rel, ASM_SIZE-4); }
|
||||
u32* origBC = body->bc;
|
||||
OptRes optRes = opt(origBC);
|
||||
Block** blocks = body->blocks->a;
|
||||
@ -465,8 +465,9 @@ Nvm_res m_nvm(Body* body) {
|
||||
bool ret = false;
|
||||
#define L64 ({ u64 r = bc[0] | ((u64)bc[1])<<32; bc+= 2; r; })
|
||||
// #define LEA0(O,I,OFF) { MOV(O,I); ADDI(O,OFF); }
|
||||
#define LEA0(O,I,OFF,Q) ({ i32 o=(OFF); if(o) LEAi(O,I,o); else if (Q) MOV(O,I); o?O:I; })
|
||||
#define SPOS(R,N,Q) LEA0(R, r_CS, maxi32(0, depth+(N)-1)*sizeof(B),Q)
|
||||
#define LEA0(O,I,OFF,Q) ({ i32 o=(OFF); if (Q||o) LEAi(O,I,o); o?O:I; })
|
||||
#define SPOSq(N) (maxi32(0, depth+(N)-1) * sizeof(B))
|
||||
#define SPOS(R,N,Q) LEA0(R, r_CS, SPOSq(N), Q) // load stack position N in register R; if !Q, then might not write and instead return another register which will have the wanted value
|
||||
#if CSTACK
|
||||
// #define INV(N,D,F) MOV(R_A##N,r_CS); ADDI(r_CS,(D)*sizeof(B)); CCALL(F)
|
||||
#define INV(N,D,F) SPOS(R_A##N, D, 1); CCALL(F)
|
||||
@ -474,20 +475,21 @@ Nvm_res m_nvm(Body* body) {
|
||||
#define INV(N,D,F) CCALL(F) // N - stack argument number; D - expected stack delta; F - called function; TODO instrs which don't need stack (POPS and things with stack delta 1)
|
||||
#endif
|
||||
#define TOPp MOV(R_A0,R_RES)
|
||||
#define TOPs if (depth) { u8 t = SPOS(r_TMP, 0, 0); MOV8mr(t, R_RES); }
|
||||
#define TOPs if (depth) { u8 t = SPOS(R_A3, 0, 0); MOV8mr(t, R_RES); }
|
||||
#define LSC(R,D) { if(D) MOV8rmo(R,r_PSCS,D*8); else MOV(R,r_SC); }
|
||||
#define INCV(R) INC4mo(R, offsetof(Value,refc)); // ADD4mi(r_TMP, 1); CCALL(i_INC);
|
||||
#define INCV(R) INC4mo(R, offsetof(Value,refc)); // ADD4mi(R_A3, 1); CCALL(i_INC);
|
||||
#ifdef __BMI2__ // TODO move to runtime detection maybe
|
||||
#define INCB(R,T,U) IMM(T,0xfffffffffffffull);ADD(T,R);IMM(U,0x7fffffffffffeull);CMP(T,U);{JA(lI);MOVi1l(U,0x30);BZHI(U,R,U);INCV(U);LBL1(lI);}
|
||||
#else
|
||||
#define INCB(R,T,U) IMM(T,0xfffffffffffffull);ADD(T,R);IMM(U,0x7fffffffffffeull);CMP(T,U);{JA(lI);IMM(U,0xffffffffffffull);AND(U,R);INCV(U);LBL1(lI);}
|
||||
#endif
|
||||
#define POS_UPD(R1,R2) IMM(R1, off); MOV8mro(r_ENV, R1, offsetof(Env,bcL));
|
||||
switch (*bc++) {
|
||||
case POPS: TOPp;
|
||||
CCALL(i_POPS);
|
||||
if (depth>1) { u8 t = SPOS(r_TMP, -1, 0); MOV8rm(R_RES, t); }
|
||||
if (depth>1) { u8 t = SPOS(R_A3, -1, 0); MOV8rm(R_RES, t); }
|
||||
break;
|
||||
case ADDI: TOPs; { u64 x = L64; IMM(R_RES, x); IMM(r_TMP, v(b(x))); INCV(r_TMP); break; } // (u64 v, S)
|
||||
case ADDI: TOPs; { u64 x = L64; IMM(R_RES, x); IMM(R_A3, v(b(x))); INCV(R_A3); break; } // (u64 v, S)
|
||||
case ADDU: TOPs; // (u64 v, S)
|
||||
#if CSTACK
|
||||
IMM(R_RES, L64);
|
||||
@ -499,8 +501,19 @@ Nvm_res m_nvm(Body* body) {
|
||||
case FN2C: TOPp; IMM(R_A1,off); INV(2,0,i_FN2C); break; // (B, u32* bc, S)
|
||||
case FN1O: TOPp; IMM(R_A1,off); INV(2,0,i_FN1O); break; // (B, u32* bc, S)
|
||||
case FN2O: TOPp; IMM(R_A1,off); INV(2,0,i_FN2O); break; // (B, u32* bc, S)
|
||||
case FN1Ci:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN1Ci); break; // (B, BB2B fm, u32* bc, S)
|
||||
case FN2Ci:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN2Ci); break; // (B, BBB2B fd, u32* bc, S)
|
||||
case FN1Ci: { u64 fn = L64; POS_UPD(R_A0,R_A3);
|
||||
MOV(R_A1, R_RES);
|
||||
MOV8pr((u64)&gStack - 4, SPOS(R_A3, 0, 0)); TSADD(rel, ASM_SIZE-4); // GS_UPD
|
||||
CCALL(fn);
|
||||
} break;
|
||||
case FN2Ci: { u64 fn = L64; POS_UPD(R_A0,R_A3);
|
||||
Reg r_sp = SPOS(R_A2, -1, 0);
|
||||
MOV8pr((u64)&gStack - 4, r_sp); TSADD(rel, ASM_SIZE-4); // GS_UPD
|
||||
MOV(R_A1, R_RES); MOV8rm(R_A2, r_sp); // load args
|
||||
CCALL(fn);
|
||||
} break;
|
||||
// case FN1Ci:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN1Ci); break; // (B, BB2B fm, u32* bc, S)
|
||||
// case FN2Ci:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN2Ci); break; // (B, BBB2B fd, u32* bc, S)
|
||||
case FN1Oi:TOPp; IMM(R_A1,L64); IMM(R_A2,off); INV(3,0,i_FN1Oi); break; // (B, BB2B fm, u32* bc, S)
|
||||
case FN2Oi:TOPp; IMM(R_A1,L64); IMM(R_A2, L64); IMM(R_A3,off); INV(4,0,i_FN2Oi); break; // (B, BB2B fm, BBB2B fd, u32* bc, S)
|
||||
case ARRM: case ARRO:;
|
||||
@ -540,7 +553,8 @@ Nvm_res m_nvm(Body* body) {
|
||||
case NSPM: TOPp; IMM(R_A1,*bc++); CCALL(i_NSPM); break; // (B, u32 l, S)
|
||||
case CHKV: TOPp; IMM(R_A1,off); INV(2,0,i_CHKV); break; // (B, u32* bc, S)
|
||||
case RETD: MOV(R_A0,r_SC); INV(1,1,i_RETD); ret=true; break; // (Scope* sc, S); stack diff 0 is wrong, but updating it is useless
|
||||
case RETN: IMM(r_TMP, &gStack); MOV8mr(r_TMP, r_CS); ret=true; break;
|
||||
// case RETN: IMM(R_A3, &gStack); MOV8mr(R_A3, r_CS); ret=true; break;
|
||||
case RETN: MOV8pr((u64)&gStack - 4, r_CS); TSADD(rel, ASM_SIZE-4); ret=true; break;
|
||||
default: thrF("JIT: Unsupported bytecode %i", *s);
|
||||
}
|
||||
#undef INCB
|
||||
@ -559,7 +573,7 @@ Nvm_res m_nvm(Body* body) {
|
||||
POP(r_SC);
|
||||
POP(r_CS);
|
||||
POP(r_PSCS);
|
||||
POP(r_TMP);
|
||||
POP(r_ENV);
|
||||
POP(5);
|
||||
RET();
|
||||
#undef CCALL
|
||||
@ -576,19 +590,20 @@ Nvm_res m_nvm(Body* body) {
|
||||
u32 bcPos = body->map[0];
|
||||
// printf("JIT %d:\n", perfid);
|
||||
// vm_printPos(body->comp, bcPos, -1);
|
||||
fprintf(perf_map, "%lx %lx JIT %d: BC@%u\n", (u64)binEx, sz, perfid++, bcPos);
|
||||
fprintf(perf_map, N64x" "N64x" JIT %d: BC@%u\n", (u64)binEx, sz, perfid++, bcPos);
|
||||
#endif
|
||||
memcpy(binEx, bin, sz);
|
||||
u64 relAm = TSSIZE(rel);
|
||||
// printf("allocated at %p; i_ADDU: %p\n", binEx, i_ADDU);
|
||||
for (u64 i = 0; i < relAm; i++) {
|
||||
u8* ins = binEx+rel[i];
|
||||
u32 o = readBytes4(ins+1);
|
||||
u32 n = o-(u32)(u64)ins-5;
|
||||
memcpy(ins+1, (u8[]){BYTES4(n)}, 4);
|
||||
u32 o = readBytes4(ins);
|
||||
u32 n = o-(u32)(u64)ins;
|
||||
memcpy(ins, (u8[]){BYTES4(n)}, 4);
|
||||
}
|
||||
#if WRITE_ASM
|
||||
write_asm(binEx, sz);
|
||||
// exit(0);
|
||||
#endif
|
||||
FREE_ASM();
|
||||
TSFREE(rel);
|
||||
@ -601,9 +616,9 @@ B evalJIT(Body* b, Scope* sc, u8* ptr) { // doesn't consume
|
||||
Scope* pscs[b->maxPSC+1];
|
||||
pscs[0] = sc;
|
||||
|
||||
B* sp = gStack;
|
||||
// B* sp = gStack;
|
||||
B r = ((JITFn*)ptr)(gStack, pscs, sc);
|
||||
if (sp!=gStack) thrM("uh oh");
|
||||
// if (sp!=gStack) thrM("uh oh");
|
||||
|
||||
popEnv();
|
||||
return r;
|
||||
|
||||
@ -61,7 +61,6 @@ typedef u8 Reg;
|
||||
#define R_P4 15 // r15
|
||||
|
||||
|
||||
|
||||
#define ALLOC_ASM(N) TStack* b_o = (TStack*)mm_allocN(sizeof(TStack)+(N), t_temp); b_o->size=0; b_o->cap=(N);
|
||||
#define GET_ASM() u8* bin = b_o->data;
|
||||
#define AADD(P,N) b_o=asm_add(b_o, P, N)
|
||||
@ -350,6 +349,7 @@ static inline void asm_a(TStack* o, u64 len, u8 v[]) {
|
||||
if(CHK4(O)) ASM1(0x24); \
|
||||
if(b4) ASM4(t); else if (b1) ASM1(t); \
|
||||
}
|
||||
#define MRMp(OFF,I) ASM1((((I)&7)<<3) + 5); ASM4(OFF); // offset to rip
|
||||
#define nA_0REG(O,I) (((I)&7)<<3) + ((O)&7)
|
||||
#define nA_REG(O,I) ASM1(0xC0 + nA_0REG(O,I)) // aka MRM immediate
|
||||
|
||||
@ -359,7 +359,7 @@ static inline void asm_a(TStack* o, u64 len, u8 v[]) {
|
||||
#define AC3(N,A,B,C) (i##N(b_o=asm_r(b_o),A,B,C))
|
||||
|
||||
// meaning of lowercase after basic instr name:
|
||||
// 'r' means the corresponding argument is the direct register contents, 'm' - that it's dereferenced; if all are 'r' or they don't have other options, they can be omitted
|
||||
// 'r' means the corresponding argument is the direct register contents, 'm' - that it's dereferenced, 'p' - offset to rip; if all are 'r' or they don't have other options, they can be omitted
|
||||
// add 'o' for an offset argument (e.g. the 5 in 'mov rax, [rdi+5]'), 'i' for an immediate (e.g. 5 in 'add r12,5', or in 'lea rax, [rdi+5]')
|
||||
#define INC4mo(O,IMM) AC2(INC4mo,O,IMM)
|
||||
#define INC8mo(O,IMM) AC2(INC8mo,O,IMM)
|
||||
@ -386,10 +386,12 @@ static inline void asm_a(TStack* o, u64 len, u8 v[]) {
|
||||
#define MOV8mr(I,O) AC2(MOV8mr,I,O) // *(u64*)(nullptr + I) ← O
|
||||
#define MOV4rm(O,I) AC2(MOV4rm,O,I) // O ← *(u32*)(nullptr + I)
|
||||
#define MOV4mr(I,O) AC2(MOV4mr,I,O) // *(u32*)(nullptr + I) ← O
|
||||
#define MOV8rmo(O,I,IMM) AC3(MOV8rmo,O,I,IMM) // O ← *(u64*)(nullptr + I + IMM)
|
||||
#define MOV8mro(I,O,IMM) AC3(MOV8mro,I,O,IMM) // *(u64*)(nullptr + I + IMM) ← O
|
||||
#define MOV4rmo(O,I,IMM) AC3(MOV4rmo,O,I,IMM)
|
||||
#define MOV4mro(I,O,IMM) AC3(MOV4mro,I,O,IMM)
|
||||
#define MOV8rmo(O,I,OFF) AC3(MOV8rmo,O,I,OFF) // O ← *(u64*)(nullptr + I + OFF)
|
||||
#define MOV8mro(I,O,OFF) AC3(MOV8mro,I,O,OFF) // *(u64*)(nullptr + I + OFF) ← O
|
||||
#define MOV4rmo(O,I,OFF) AC3(MOV4rmo,O,I,OFF)
|
||||
#define MOV4mro(I,O,OFF) AC3(MOV4mro,I,O,OFF)
|
||||
#define MOV8pr(OFF,I) AC2(MOV8pr,OFF,I)
|
||||
#define MOV8rp(I,OFF) AC2(MOV8rp,OFF,I)
|
||||
|
||||
#define LEAi(O,I,IMM) AC3(LEAi,O,I,IMM)
|
||||
#define BZHI(O,I,N) AC3(BZHI,O,I,N) // requires __BMI2__
|
||||
@ -397,7 +399,7 @@ static inline void asm_a(TStack* o, u64 len, u8 v[]) {
|
||||
#define PUSH(O) AC1(PUSH,O)
|
||||
#define POP(O) AC1(POP,O)
|
||||
#define CALL(IMM) AC1(CALL,IMM)
|
||||
#define CALLi(I) AC1(CALLi,(i32)(i64)(I))
|
||||
#define CALLi(I) AC1(CALLi,(i64)(I)) // I must be 32-bit
|
||||
#define RET() {b_o=asm_r(b_o); ASM1(0xC3);}
|
||||
|
||||
#define JO(L) u64 L=ASM_SIZE; {b_o=asm_r(b_o); ASM1(0x70);ASM1(-2);}
|
||||
@ -445,11 +447,12 @@ ASMI(MOVi, Reg o, i64 i) {
|
||||
else { nREX8(o,0); ASM1(0xB8+(o&7)); ASM8(i); }
|
||||
}
|
||||
ASMI(MOVi1l, Reg o, i8 imm) { if (o>=4) ASM1(o>=8?0x41:0x40); ASM1(0xb0+(o&7)); ASM1(imm); }
|
||||
|
||||
ASMI(MOV8mr, Reg o, Reg i) { nREX8(o,i); ASM1(0x89); MRM(o,i); } ASMI(MOV8mro, Reg o, Reg i, i32 off) { nREX8(o,i); ASM1(0x89); MRMo(o,i, off); }
|
||||
ASMI(MOV8rm, Reg i, Reg o) { nREX8(o,i); ASM1(0x8B); MRM(o,i); } ASMI(MOV8rmo, Reg i, Reg o, i32 off) { nREX8(o,i); ASM1(0x8B); MRMo(o,i, off); }
|
||||
ASMI(MOV4mr, Reg o, Reg i) { nREX4(o,i); ASM1(0x89); MRM(o,i); } ASMI(MOV4mro, Reg o, Reg i, i32 off) { nREX4(o,i); ASM1(0x89); MRMo(o,i, off); }
|
||||
ASMI(MOV4rm, Reg i, Reg o) { nREX4(o,i); ASM1(0x8B); MRM(o,i); } ASMI(MOV4rmo, Reg i, Reg o, i32 off) { nREX4(o,i); ASM1(0x8B); MRMo(o,i, off); }
|
||||
ASMI(MOV8pr,i32 off, Reg i) { nREX8(0,i); ASM1(0x89); MRMp(off,i); }
|
||||
ASMI(MOV8rp,i32 off, Reg i) { nREX8(0,i); ASM1(0x8B); MRMp(off,i); }
|
||||
|
||||
ASMI(LEAi, Reg o, Reg i, i32 imm) { if(imm==0) iMOV(b_o,o,i); else { nREX8(i,o); ASM1(0x8D); MRMo(i,o,imm); } }
|
||||
ASMI(BZHI, Reg o, Reg i, Reg n) { ASM1(0xC4); ASM1(0x42+(i<8)*0x20 + (o<8)*0x80); ASM1(0xf8-n*8); ASM1(0xF5); nA_REG(i, o); }
|
||||
@ -458,6 +461,6 @@ ASMI(PUSH, Reg O) { nREX4(O,0); ASM1(0x50+((O)&7)); }
|
||||
ASMI(POP , Reg O) { nREX4(O,0); ASM1(0x58+((O)&7)); }
|
||||
|
||||
ASMI(CALL, Reg i) { nREX4(i,0); ASM1(0xFF); nA_REG(i,2); }
|
||||
ASMI(CALLi, i32 imm) { ASM1(0xE8); ASM4(imm); }
|
||||
ASMI(CALLi, i32 imm) { ASM1(0xE8); if (imm>I32_MAX)err("immediate call outside of 32-bit range!"); ASM4(imm); }
|
||||
|
||||
#define IMM(A,B) MOVi(A,(u64)(B))
|
||||
8
src/vm.c
8
src/vm.c
@ -95,10 +95,12 @@ B* gStackEnd;
|
||||
void gsPrint() {
|
||||
B* c = gStackStart;
|
||||
i32 i = 0;
|
||||
printf("gStack %p, height "N64d":\n", gStackStart, gStack-gStackStart);
|
||||
while (c!=gStack) {
|
||||
printf("%d: ", i);
|
||||
print(*c);
|
||||
printf(", refc=%d\n", v(*c)->refc);
|
||||
printf(" %d: ", i); fflush(stdout);
|
||||
print(*c); fflush(stdout);
|
||||
if (isVal(*c)) printf(", refc=%d", v(*c)->refc); fflush(stdout);
|
||||
printf("\n");
|
||||
c++;
|
||||
i++;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user