uCBQN/src/jit/x86_64.h
dzaima 1b26e85325 don't need to mark static globals as hidden
gcc complains about this
2024-02-01 21:26:08 +02:00

254 lines
10 KiB
C

// based on I: https://github.com/mlochbaum/ILanguage/blob/master/x86_64.h
#pragma once
#include "../core.h"
#include "../utils/talloc.h"
// V - volatile (overwritten by calls)
// 0 rax V result
// 1 rcx V arg 3
// 2 rdx V arg 2
// 3 rbx
// 4 rsp stack
// 5 rbp base
// 6 rsi V arg 1
// 7 rdi V arg 0
// 8 r8 V arg 4
// 9 r9 V arg 5
// . r10 V
// . r11 V
// . r12
// . r13
// . r14
// . r15
typedef u8 U;
typedef u8 UC;
typedef u8 Reg;
#define R_RES 0 // rax
#define R_SP 4 // rsp
#define R_BP 5 // rbp
// aregument registers
#define R_A0 7 // rdi
#define R_A1 6 // rsi
#define R_A2 2 // rdx
#define R_A3 1 // rcx; may be used by non-immediate shifts
#define R_A4 8 // r8
#define R_A5 9 // r9
// volatile, aka caller-saved registers (like argument registers, but never actually arguments)
#define R_V0 10
#define R_V1 11
// non-volatile/callee-saved/preserved registers
#define R_P0 3 // rbx
#define R_P1 14 // r14
#define R_P2 15 // r15
#define R_P3 13 // r13
#define R_P4 12 // r12
typedef struct AsmStk {
u8* s; // actual allocation
u8* c; // position for next write
u8* e; // position past last writable position
} AsmStk;
STATIC_GLOBAL AsmStk asm_ins; // TODO add as root
STATIC_GLOBAL AsmStk asm_rel;
STATIC_GLOBAL i32 asm_depth = 0;
static NOINLINE void asm_allocBuf(AsmStk* stk, u64 sz) {
TAlloc* a = mm_alloc(sizeof(TAlloc) + sz, t_temp);
stk->s = a->data;
stk->c = a->data;
stk->e = a->data + sz;
}
typedef struct AsmRestorer {
struct CustomObj;
i32 depth;
AsmStk ins, rel;
} AsmRestorer;
static void asmRestorer_free(Value* v) {
asm_depth = ((AsmRestorer*)v)->depth;
asm_ins = ((AsmRestorer*)v)->ins;
asm_rel = ((AsmRestorer*)v)->rel;
}
static NOINLINE void asm_init() {
AsmRestorer* r = m_customObj(sizeof(AsmRestorer), noop_visit, asmRestorer_free);
r->depth = asm_depth;
r->ins = asm_ins;
r->rel = asm_rel;
gsAdd(tag(r, OBJ_TAG));
asm_depth++;
asm_allocBuf(&asm_ins, 64);
asm_allocBuf(&asm_rel, 64);
}
static NOINLINE void asm_free() {
mm_free((Value*) TOBJ(asm_ins.s));
mm_free((Value*) TOBJ(asm_rel.s));
assert(asm_depth>0);
B v = gsPop();
assert(TY(v)==t_customObj);
decG(v);
}
static NOINLINE void asm_bufDbl(AsmStk* stk, u64 nsz) {
u8* prevS = stk->s;
u64 size = stk->e - prevS;
u64 used = stk->c - prevS;
while (size < used+nsz) size*= 2;
asm_allocBuf(stk, size);
stk->c+= used;
memcpy(stk->s, prevS, used);
mm_free((Value*) TOBJ(prevS));
}
#define ASM_SIZE (asm_ins.c - asm_ins.s)
static inline void asm_w1(u8* data, i8 v) { *data = v; }
static inline void asm_w2(u8* data, i16 v) { memcpy(data, (i16[]){v}, 2); }
static inline void asm_w4(u8* data, i32 v) { memcpy(data, (i32[]){v}, 4); }
static inline void asm_w8(u8* data, i64 v) { memcpy(data, (i64[]){v}, 8); }
#define ASM1(X) ({ asm_w1(ic, X); ic+= 1; })
#define ASM2(X) ({ asm_w2(ic, X); ic+= 2; })
#define ASM4(X) ({ asm_w4(ic, X); ic+= 4; })
#define ASM8(X) ({ asm_w8(ic, X); ic+= 8; })
static inline i32 asm_r4(u8* data) { i32 v; memcpy(&v, data, 4); return v; }
static inline void asm_r() {
if (RARE(asm_ins.c+32 > asm_ins.e)) asm_bufDbl(&asm_ins, 32);
}
static NOINLINE void asm_addRel(u32 v) {
if (RARE(asm_rel.c == asm_rel.e)) asm_bufDbl(&asm_rel, 4);
asm_w4(asm_rel.c, v);
asm_rel.c+= 4;
}
static NOINLINE void asm_write(u8* P, u64 SZ) {
memcpy(P, asm_ins.s, SZ);
u64 relAm = (asm_rel.c-asm_rel.s)/4;
for (u64 i = 0; i < relAm; i++) {
u8* ins = P+asm_r4(asm_rel.s+i*4);
u32 o = asm_r4(ins);
u32 n = o-(u32)(u64)ins;
asm_w4(ins, n);
}
}
#define REX0(O,I) (((O)>7)+(((I)>7)<<2))
#define REX1(O,I) ({ if((O)>3 || (I)>3) ASM1(0x40+REX0(O,I)); })
#define REX2(O,I) ({ ASM1(0x66); REX4(O,I); })
#define REX4(O,I) {u8 t=REX0(O,I); if(t) ASM1(0x40+t); }
#define REX8(O,I) ASM1(0x48 + REX0(O,I))
#define CHK4(O) (((O)&7)==4)
#define CHK5(O) (((O)&7)==5)
#define CHK45(O) (((O)&7)>>1==2)
// ModR/M mess
#define A_0REG(O,I,A) ((((I)&7)<<3) + ((O)&7) + (A))
#define MRM(O,I) ASM1(A_0REG(O,I,0x40*CHK5(O))); if(CHK45(O)) ASM1(CHK4(O)?0x24:0);
#define MRMo(O,I,OFF) { i64 t=OFF; \
bool b1 = t||CHK5(O); bool b4 = t!=(i8)t; \
ASM1(A_0REG(O, I, b1? b4?0x80:0x40 : 0)); \
if(CHK4(O)) ASM1(0x24); \
if(b4) ASM4(t); else if (b1) ASM1(t); \
}
#define MRMp(OFF,I) ASM1(A_0REG(5,I,0x00)); ASM4(OFF); // offset to rip
#define MRMr(O,I) ASM1(A_0REG(O,I,0xC0)) // only register operands
#define MRM1(O,T) ASM1(((O)&7) + T)
static const u8 cO = 0x0; static const u8 cNO = 0x1;
static const u8 cB = 0x2; static const u8 cAE = 0x3;
static const u8 cE = 0x4; static const u8 cNE = 0x5;
static const u8 cBE = 0x6; static const u8 cA = 0x7;
static const u8 cS = 0x8; static const u8 cNS = 0x9;
static const u8 cP = 0xA; static const u8 cNP = 0xB;
static const u8 cL = 0xC; static const u8 cGE = 0xD;
static const u8 cLE = 0xE; static const u8 cG = 0xF;
#define J1(T, L) u64 L=ASM_SIZE; { ASMS; ASM1((i8)(0x70+(T)));ASM1(-2);ASME; } // -2 comes from the instruction being 2 bytes long and L being defined at the start
#define J4(T, L) u64 L=ASM_SIZE; { ASMS; ASM1(0x0f);ASM1((i8)(0x80+(T)));ASM4(-6);ASME; }
#define LBL1(L) { i64 t= (i8)asm_ins.s[L+1] + ASM_SIZE-(i64)L; if(t!=(i8 )t) fatal("x86-64 codegen: jump too long!"); asm_ins.s[L+1] = t; }
#define LBL4(L) { i64 t=asm_r4(asm_ins.s+(L+2)) + ASM_SIZE-(i64)L; if(t!=(i32)t) fatal("x86-64 codegen: jump too long!"); asm_w4(asm_ins.s+(L+2), t); }
#define ASMS u8* ic=asm_ins.c
#define ASME asm_ins.c = ic; asm_r()
#define ASMI(N, ...) static NOINLINE void N(__VA_ARGS__)
// meaning of lowercase after basic instr name:
// 'r' means the corresponding argument is the direct register contents, 'm' - that it's dereferenced, 'p' - offset to rip; if all are 'r' or they don't have other options, they can be omitted
// add 'o' for an offset argument (e.g. the 5 in 'mov rax, [rdi+5]'), 'i' for an immediate (e.g. 5 in 'add r12,5', or in 'lea rax, [rdi+5]')
// MOV8rm: O ← *(u64*)(nullptr + I)
// MOV8mr: *(u64*)(nullptr + I) ← O
// MOV4rm: O ← *(u32*)(nullptr + I)
// MOV4mr: *(u32*)(nullptr + I) ← O
// MOV8rmo: // O ← *(u64*)(nullptr + I + OFF)
// MOV8mro: // *(u64*)(nullptr + I + OFF) ← O
// MOV4rmo: // O ← *(u32*)(nullptr + I + OFF)
// MOV4mro: // *(u32*)(nullptr + I + OFF) ← O
// BZHI: requires __BMI2__
ASMI(ADD, Reg o, Reg i) { ASMS; REX8(o,i); ASM1(0x01); MRMr(o,i); ASME; }
ASMI(SUB, Reg o, Reg i) { ASMS; REX8(o,i); ASM1(0x29); MRMr(o,i); ASME; }
ASMI( OR, Reg o, Reg i) { ASMS; REX8(o,i); ASM1(0x09); MRMr(o,i); ASME; }
ASMI(XOR, Reg o, Reg i) { ASMS; REX8(o,i); ASM1(0x31); MRMr(o,i); ASME; }
ASMI(AND, Reg o, Reg i) { ASMS; REX8(o,i); ASM1(0x21); MRMr(o,i); ASME; }
ASMI(XOR4, Reg o, Reg i) { ASMS; REX4(o,i); ASM1(0x31); MRMr(o,i); ASME; }
ASMI(CMP, Reg o, Reg i) { ASMS; REX8(o,i); ASM1(0x39); MRMr(o,i); ASME; }
ASMI(ADDi, Reg o, i32 imm) { if(!imm) return; ASMS; REX8(o,0); if(imm==(i8)imm) { ASM1(0x83); MRM1(o,0xc0); ASM1(imm); } else { ASM1(0x81); MRM1(o,0xc0); ASM4(imm); } ASME; }
ASMI(SUBi, Reg o, i32 imm) { if(!imm) return; ASMS; REX8(o,0); if(imm==(i8)imm) { ASM1(0x83); MRM1(o,0xe8); ASM1(imm); } else { ASM1(0x81); MRM1(o,0xe8); ASM4(imm); } ASME; }
ASMI(SHL1i, Reg o, i8 imm) { ASMS; REX1(o,0); ASM1(imm==1?0xD0:0xC0); MRM1(o,0xe0); if (imm!=1) ASM1(imm); ASME; }
ASMI(SHR1i, Reg o, i8 imm) { ASMS; REX1(o,0); ASM1(imm==1?0xD0:0xC0); MRM1(o,0xe8); if (imm!=1) ASM1(imm); ASME; }
ASMI(SHL2i, Reg o, i8 imm) { ASMS; REX2(o,0); ASM1(imm==1?0xD1:0xC1); MRM1(o,0xe0); if (imm!=1) ASM1(imm); ASME; }
ASMI(SHR2i, Reg o, i8 imm) { ASMS; REX2(o,0); ASM1(imm==1?0xD1:0xC1); MRM1(o,0xe8); if (imm!=1) ASM1(imm); ASME; }
ASMI(SHL8i, Reg o, i8 imm) { ASMS; REX8(o,0); ASM1(imm==1?0xD1:0xC1); MRM1(o,0xe0); if (imm!=1) ASM1(imm); ASME; }
ASMI(SHR8i, Reg o, i8 imm) { ASMS; REX8(o,0); ASM1(imm==1?0xD1:0xC1); MRM1(o,0xe8); if (imm!=1) ASM1(imm); ASME; }
ASMI(ADD4mi, Reg o, i32 imm) { ASMS; REX4(o,0); ASM1(0x83); MRM1(o,0xc0); ASM1(imm); ASME; }
ASMI(INC4mo, Reg o, i32 off) { ASMS; REX4(o,0); ASM1(0xff); MRMo(o,0,off); ASME; }
ASMI(INC8mo, Reg o, i32 off) { ASMS; REX8(o,0); ASM1(0xff); MRMo(o,0,off); ASME; }
ASMI(DEC4mo, Reg o, i32 off) { ASMS; REX4(o,0); ASM1(0xff); MRMo(o,1,off); ASME; }
ASMI(DEC8mo, Reg o, i32 off) { ASMS; REX8(o,0); ASM1(0xff); MRMo(o,1,off); ASME; }
ASMI(MOV4, Reg o, Reg i) { ASMS; REX4(o,i); ASM1(0x89); MRMr(o,i); ASME; }
ASMI(MOV , Reg o, Reg i) { ASMS; REX8(o,i); ASM1(0x89); MRMr(o,i); ASME; }
ASMI(MOVi, Reg o, i64 i) {
if (i==0) { XOR4(o,o); return; }
ASMS;
if (i==(i32)i) { REX4(o,0); MRM1(o,0xB8); ASM4(i); }
else { REX8(o,0); MRM1(o,0xB8); ASM8(i); }
ASME;
}
#define IMM(A,B) MOVi(A,(u64)(B))
ASMI(MOVi1l, Reg o, i8 imm) { ASMS; if (o>=4) ASM1(o>=8?0x41:0x40); MRM1(o,0xb0); ASM1(imm); ASME; }
ASMI(MOV4moi, Reg o, i32 off, i32 imm) { ASMS; REX4(o,0); ASM1(0xc7); MRMo(o,0,off); ASM4(imm); ASME; }
ASMI(MOV4mr, Reg o, Reg i) { ASMS; REX4(o,i); ASM1(0x89); MRM(o,i); ASME; } ASMI(MOV4mro, Reg o, Reg i, i32 off) { ASMS; REX4(o,i); ASM1(0x89); MRMo(o,i, off); ASME; }
ASMI(MOV4rm, Reg i, Reg o) { ASMS; REX4(o,i); ASM1(0x8B); MRM(o,i); ASME; } ASMI(MOV4rmo, Reg i, Reg o, i32 off) { ASMS; REX4(o,i); ASM1(0x8B); MRMo(o,i, off); ASME; }
ASMI(MOV8mr, Reg o, Reg i) { ASMS; REX8(o,i); ASM1(0x89); MRM(o,i); ASME; } ASMI(MOV8mro, Reg o, Reg i, i32 off) { ASMS; REX8(o,i); ASM1(0x89); MRMo(o,i, off); ASME; }
ASMI(MOV8rm, Reg i, Reg o) { ASMS; REX8(o,i); ASM1(0x8B); MRM(o,i); ASME; } ASMI(MOV8rmo, Reg i, Reg o, i32 off) { ASMS; REX8(o,i); ASM1(0x8B); MRMo(o,i, off); ASME; }
ASMI(MOV8pr_i, u64 pos, Reg i) { ASMS; REX8(0,i); ASM1(0x89); MRMp(pos-4,i); ASME; }
ASMI(MOV8rp_i, u64 pos, Reg i) { ASMS; REX8(0,i); ASM1(0x8B); MRMp(pos-4,i); ASME; }
#define MOV8pr(POS,I) { MOV8pr_i((u64)(POS),I); asm_addRel(ASM_SIZE-4); }
#define MOV8rp(I,POS) { MOV8rp_i((u64)(POS),I); asm_addRel(ASM_SIZE-4); }
ASMI(LEAi, Reg o, Reg i, i32 imm) { if(imm==0) MOV(o,i); else { ASMS; REX8(i,o); ASM1(0x8D); MRMo(i,o,imm); ASME; } }
ASMI(BZHI, Reg o, Reg i, Reg n) { ASMS; ASM1(0xC4); ASM1(0x42+(i<8)*0x20 + (o<8)*0x80); ASM1(0xf8-n*8); ASM1(0xF5); MRMr(i, o); ASME; }
ASMI(iPUSH, Reg o) { ASMS; REX4(o,0); MRM1(o,0x50); ASME; }
ASMI(iPOP, Reg o) { ASMS; REX4(o,0); MRM1(o,0x58); ASME; }
ASMI(RET) { ASMS; ASM1(0xC3); ASME; }
ASMI(CALL, Reg i) { ASMS; REX4(i,0); ASM1(0xFF); MRM1(i,0xd0); ASME; }
ASMI(CALLi_i, u64 pos) { ASMS; ASM1(0xE8); ASM4(pos-4); ASME; }
#define CALLi(POS) { CALLi_i((u64)(POS)); asm_addRel(ASM_SIZE-4); } // POS must be 32-bit