diff --git a/src/flamenco/vm/jit/Local.mk b/src/flamenco/vm/jit/Local.mk deleted file mode 100644 index 8ddd330701..0000000000 --- a/src/flamenco/vm/jit/Local.mk +++ /dev/null @@ -1,10 +0,0 @@ -ifdef FD_HAS_INT128 -ifdef FD_HAS_HOSTED -ifdef FD_HAS_X86 -$(call add-hdrs,fd_jit.h) -$(call add-objs,fd_jit fd_jit_compiler,fd_flamenco) -$(call make-unit-test,test_jit_dasm,test_jit_dasm,fd_flamenco fd_funk fd_ballet fd_util) -$(call run-unit-test,test_jit_dasm) -endif -endif -endif diff --git a/src/flamenco/vm/jit/Makefile b/src/flamenco/vm/jit/Makefile deleted file mode 100644 index a3c24fe766..0000000000 --- a/src/flamenco/vm/jit/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# Run 'make vendor' to refresh dynasm headers from the latest LuaJIT checkout. - -LUA ?= lua-5.1 -LUAJIT ?= ../../../../opt/git/luajit - -.PHONY: generate -generate: fd_jit_compiler.c - -fd_jit_compiler.c: fd_jit_compiler.dasc - $(LUA) $(LUAJIT)/dynasm/dynasm.lua -o $@ $< - -.PHONY: vendor -vendor: - rm -v dasm_{proto,x86}.h - cp $(LUAJIT)/dynasm/dasm_{proto,x86}.h . diff --git a/src/flamenco/vm/jit/dasm_proto.h b/src/flamenco/vm/jit/dasm_proto.h deleted file mode 100644 index 3f50f5028f..0000000000 --- a/src/flamenco/vm/jit/dasm_proto.h +++ /dev/null @@ -1,83 +0,0 @@ -/* -** DynASM encoding engine prototypes. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. -** Released under the MIT license. See dynasm.lua for full copyright notice. -*/ - -#ifndef _DASM_PROTO_H -#define _DASM_PROTO_H - -#include -#include - -#define DASM_IDENT "DynASM 1.5.0" -#define DASM_VERSION 10500 /* 1.5.0 */ - -#ifndef Dst_DECL -#define Dst_DECL dasm_State **Dst -#endif - -#ifndef Dst_REF -#define Dst_REF (*Dst) -#endif - -#ifndef DASM_FDEF -#define DASM_FDEF extern -#endif - -#ifndef DASM_M_GROW -#define DASM_M_GROW(ctx, t, p, sz, need) \ - do { \ - size_t _sz = (sz), _need = (need); \ - if (_sz < _need) { \ - if (_sz < 16) _sz = 16; \ - while (_sz < _need) _sz += _sz; \ - (p) = (t *)realloc((p), _sz); \ - if ((p) == NULL) exit(1); \ - (sz) = _sz; \ - } \ - } while(0) -#endif - -#ifndef DASM_M_FREE -#define DASM_M_FREE(ctx, p, sz) free(p) -#endif - -/* Internal DynASM encoder state. */ -typedef struct dasm_State dasm_State; - - -/* Initialize and free DynASM state. */ -DASM_FDEF void dasm_init(Dst_DECL, int maxsection); -DASM_FDEF void dasm_free(Dst_DECL); - -/* Setup global array. Must be called before dasm_setup(). */ -DASM_FDEF void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl); - -/* Grow PC label array. Can be called after dasm_setup(), too. */ -DASM_FDEF void dasm_growpc(Dst_DECL, unsigned int maxpc); - -/* Setup encoder. */ -DASM_FDEF void dasm_setup(Dst_DECL, const void *actionlist); - -/* Feed encoder with actions. Calls are generated by pre-processor. */ -DASM_FDEF void dasm_put(Dst_DECL, int start, ...); - -/* Link sections and return the resulting size. */ -DASM_FDEF int dasm_link(Dst_DECL, size_t *szp); - -/* Encode sections into buffer. */ -DASM_FDEF int dasm_encode(Dst_DECL, void *buffer); - -/* Get PC label offset. */ -DASM_FDEF int dasm_getpclabel(Dst_DECL, unsigned int pc); - -#ifdef DASM_CHECKS -/* Optional sanity checker to call between isolated encoding steps. */ -DASM_FDEF int dasm_checkstep(Dst_DECL, int secmatch); -#else -#define dasm_checkstep(a, b) 0 -#endif - - -#endif /* _DASM_PROTO_H */ diff --git a/src/flamenco/vm/jit/dasm_x86.h b/src/flamenco/vm/jit/dasm_x86.h deleted file mode 100644 index e76e36fb43..0000000000 --- a/src/flamenco/vm/jit/dasm_x86.h +++ /dev/null @@ -1,529 +0,0 @@ -/* -** DynASM x86 encoding engine. -** Copyright (C) 2005-2023 Mike Pall. All rights reserved. -** Released under the MIT license. See dynasm.lua for full copyright notice. -*/ - -#include -#include -#include -#include - -#define DASM_ARCH "x86" - -#ifndef DASM_EXTERN -#define DASM_EXTERN(a,b,c,d) 0 -#endif - -/* Action definitions. DASM_STOP must be 255. */ -enum { - DASM_DISP = 233, - DASM_IMM_S, DASM_IMM_B, DASM_IMM_W, DASM_IMM_D, DASM_IMM_WB, DASM_IMM_DB, - DASM_VREG, DASM_SPACE, DASM_SETLABEL, DASM_REL_A, DASM_REL_LG, DASM_REL_PC, - DASM_IMM_LG, DASM_IMM_PC, DASM_LABEL_LG, DASM_LABEL_PC, DASM_ALIGN, - DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP -}; - -/* Maximum number of section buffer positions for a single dasm_put() call. */ -#define DASM_MAXSECPOS 25 - -/* DynASM encoder status codes. Action list offset or number are or'ed in. */ -#define DASM_S_OK 0x00000000 -#define DASM_S_NOMEM 0x01000000 -#define DASM_S_PHASE 0x02000000 -#define DASM_S_MATCH_SEC 0x03000000 -#define DASM_S_RANGE_I 0x11000000 -#define DASM_S_RANGE_SEC 0x12000000 -#define DASM_S_RANGE_LG 0x13000000 -#define DASM_S_RANGE_PC 0x14000000 -#define DASM_S_RANGE_VREG 0x15000000 -#define DASM_S_UNDEF_L 0x21000000 -#define DASM_S_UNDEF_PC 0x22000000 - -/* Macros to convert positions (8 bit section + 24 bit index). */ -#define DASM_POS2IDX(pos) ((pos)&0x00ffffff) -#define DASM_POS2BIAS(pos) ((pos)&0xff000000) -#define DASM_SEC2POS(sec) ((sec)<<24) -#define DASM_POS2SEC(pos) ((pos)>>24) -#define DASM_POS2PTR(D, pos) (D->sections[DASM_POS2SEC(pos)].rbuf + (pos)) - -/* Action list type. */ -typedef const unsigned char *dasm_ActList; - -/* Per-section structure. */ -typedef struct dasm_Section { - int *rbuf; /* Biased buffer pointer (negative section bias). */ - int *buf; /* True buffer pointer. */ - size_t bsize; /* Buffer size in bytes. */ - int pos; /* Biased buffer position. */ - int epos; /* End of biased buffer position - max single put. */ - int ofs; /* Byte offset into section. */ -} dasm_Section; - -/* Core structure holding the DynASM encoding state. */ -struct dasm_State { - size_t psize; /* Allocated size of this structure. */ - dasm_ActList actionlist; /* Current actionlist pointer. */ - int *lglabels; /* Local/global chain/pos ptrs. */ - size_t lgsize; - int *pclabels; /* PC label chains/pos ptrs. */ - size_t pcsize; - void **globals; /* Array of globals. */ - dasm_Section *section; /* Pointer to active section. */ - size_t codesize; /* Total size of all code sections. */ - int maxsection; /* 0 <= sectionidx < maxsection. */ - int status; /* Status code. */ - dasm_Section sections[1]; /* All sections. Alloc-extended. */ -}; - -/* The size of the core structure depends on the max. number of sections. */ -#define DASM_PSZ(ms) (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section)) - -#ifndef FD_DASM_HEADER_ONLY -#define FD_DASM_HEADER_ONLY 0 -#endif - -#if !FD_DASM_HEADER_ONLY - -/* Initialize DynASM state. */ -void dasm_init(Dst_DECL, int maxsection) -{ - dasm_State *D; - size_t psz = 0; - Dst_REF = NULL; - DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); - D = Dst_REF; - D->psize = psz; - D->lglabels = NULL; - D->lgsize = 0; - D->pclabels = NULL; - D->pcsize = 0; - D->globals = NULL; - D->maxsection = maxsection; - memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section)); -} - -/* Free DynASM state. */ -void dasm_free(Dst_DECL) -{ - dasm_State *D = Dst_REF; - int i; - for (i = 0; i < D->maxsection; i++) - if (D->sections[i].buf) - DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize); - if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize); - if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize); - DASM_M_FREE(Dst, D, D->psize); -} - -/* Setup global label array. Must be called before dasm_setup(). */ -void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) -{ - dasm_State *D = Dst_REF; - D->globals = gl; - DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); -} - -/* Grow PC label array. Can be called after dasm_setup(), too. */ -void dasm_growpc(Dst_DECL, unsigned int maxpc) -{ - dasm_State *D = Dst_REF; - size_t osz = D->pcsize; - DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int)); - memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz); -} - -/* Setup encoder. */ -void dasm_setup(Dst_DECL, const void *actionlist) -{ - dasm_State *D = Dst_REF; - int i; - D->actionlist = (dasm_ActList)actionlist; - D->status = DASM_S_OK; - D->section = &D->sections[0]; - memset((void *)D->lglabels, 0, D->lgsize); - if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); - for (i = 0; i < D->maxsection; i++) { - D->sections[i].pos = DASM_SEC2POS(i); - D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos; - D->sections[i].ofs = 0; - } -} - - -#ifdef DASM_CHECKS -#define CK(x, st) \ - do { if (!(x)) { \ - D->status = DASM_S_##st|(int)(p-D->actionlist-1); return; } } while (0) -#define CKPL(kind, st) \ - do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ - D->status=DASM_S_RANGE_##st|(int)(p-D->actionlist-1); return; } } while (0) -#else -#define CK(x, st) ((void)0) -#define CKPL(kind, st) ((void)0) -#endif - -/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */ -void dasm_put(Dst_DECL, int start, ...) -{ - va_list ap; - dasm_State *D = Dst_REF; - dasm_ActList p = D->actionlist + start; - dasm_Section *sec = D->section; - int pos = sec->pos, ofs = sec->ofs, mrm = -1; - int *b; - - if (pos >= sec->epos) { - DASM_M_GROW(Dst, int, sec->buf, sec->bsize, - sec->bsize + 2*DASM_MAXSECPOS*sizeof(int)); - sec->rbuf = sec->buf - DASM_POS2BIAS(pos); - sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos); - } - - b = sec->rbuf; - b[pos++] = start; - - va_start(ap, start); - while (1) { - int action = *p++; - if (action < DASM_DISP) { - ofs++; - } else if (action <= DASM_REL_A) { - int n = va_arg(ap, int); - b[pos++] = n; - switch (action) { - case DASM_DISP: - if (n == 0) { if (mrm < 0) mrm = p[-2]; if ((mrm&7) != 5) break; } - /* fallthrough */ - case DASM_IMM_DB: if (((n+128)&-256) == 0) goto ob; /* fallthrough */ - case DASM_REL_A: /* Assumes ptrdiff_t is int. !x64 */ - case DASM_IMM_D: ofs += 4; break; - case DASM_IMM_S: CK(((n+128)&-256) == 0, RANGE_I); goto ob; - case DASM_IMM_B: CK((n&-256) == 0, RANGE_I); ob: ofs++; break; - case DASM_IMM_WB: if (((n+128)&-256) == 0) goto ob; /* fallthrough */ - case DASM_IMM_W: CK((n&-65536) == 0, RANGE_I); ofs += 2; break; - case DASM_SPACE: p++; ofs += n; break; - case DASM_SETLABEL: b[pos-2] = -0x40000000; break; /* Neg. label ofs. */ - case DASM_VREG: CK((n&-16) == 0 && (n != 4 || (*p>>5) != 2), RANGE_VREG); - if (*p < 0x40 && p[1] == DASM_DISP) mrm = n; - if (*p < 0x20 && (n&7) == 4) ofs++; - switch ((*p++ >> 3) & 3) { - case 3: n |= b[pos-3]; /* fallthrough */ - case 2: n |= b[pos-2]; /* fallthrough */ - case 1: if (n <= 7) { b[pos-1] |= 0x10; ofs--; } - } - continue; - } - mrm = -1; - } else { - int *pl, n; - switch (action) { - case DASM_REL_LG: - case DASM_IMM_LG: - n = *p++; pl = D->lglabels + n; - /* Bkwd rel or global. */ - if (n <= 246) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; } - pl -= 246; n = *pl; - if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */ - goto linkrel; - case DASM_REL_PC: - case DASM_IMM_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC); - putrel: - n = *pl; - if (n < 0) { /* Label exists. Get label pos and store it. */ - b[pos] = -n; - } else { - linkrel: - b[pos] = n; /* Else link to rel chain, anchored at label. */ - *pl = pos; - } - pos++; - ofs += 4; /* Maximum offset needed. */ - if (action == DASM_REL_LG || action == DASM_REL_PC) { - b[pos++] = ofs; /* Store pass1 offset estimate. */ - } else if (sizeof(ptrdiff_t) == 8) { - ofs += 4; - } - break; - case DASM_LABEL_LG: pl = D->lglabels + *p++; CKPL(lg, LG); goto putlabel; - case DASM_LABEL_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC); - putlabel: - n = *pl; /* n > 0: Collapse rel chain and replace with label pos. */ - while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos; } - *pl = -pos; /* Label exists now. */ - b[pos++] = ofs; /* Store pass1 offset estimate. */ - break; - case DASM_ALIGN: - ofs += *p++; /* Maximum alignment needed (arg is 2**n-1). */ - b[pos++] = ofs; /* Store pass1 offset estimate. */ - break; - case DASM_EXTERN: p += 2; ofs += 4; break; - case DASM_ESC: p++; ofs++; break; - case DASM_MARK: mrm = p[-2]; break; - case DASM_SECTION: - n = *p; CK(n < D->maxsection, RANGE_SEC); D->section = &D->sections[n]; - case DASM_STOP: goto stop; - } - } - } -stop: - va_end(ap); - sec->pos = pos; - sec->ofs = ofs; -} -#undef CK - -/* Pass 2: Link sections, shrink branches/aligns, fix label offsets. */ -int dasm_link(Dst_DECL, size_t *szp) -{ - dasm_State *D = Dst_REF; - int secnum; - int ofs = 0; - -#ifdef DASM_CHECKS - *szp = 0; - if (D->status != DASM_S_OK) return D->status; - { - int pc; - for (pc = 0; pc*sizeof(int) < D->pcsize; pc++) - if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc; - } -#endif - - { /* Handle globals not defined in this translation unit. */ - int idx; - for (idx = 10; idx*sizeof(int) < D->lgsize; idx++) { - int n = D->lglabels[idx]; - /* Undefined label: Collapse rel chain and replace with marker (< 0). */ - while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; } - } - } - - /* Combine all code sections. No support for data sections (yet). */ - for (secnum = 0; secnum < D->maxsection; secnum++) { - dasm_Section *sec = D->sections + secnum; - int *b = sec->rbuf; - int pos = DASM_SEC2POS(secnum); - int lastpos = sec->pos; - - while (pos != lastpos) { - dasm_ActList p = D->actionlist + b[pos++]; - int op = 0; - while (1) { - int action = *p++; - switch (action) { - case DASM_REL_LG: p++; - /* fallthrough */ - case DASM_REL_PC: { - int shrink = op == 0xe9 ? 3 : ((op&0xf0) == 0x80 ? 4 : 0); - if (shrink) { /* Shrinkable branch opcode? */ - int lofs, lpos = b[pos]; - if (lpos < 0) goto noshrink; /* Ext global? */ - lofs = *DASM_POS2PTR(D, lpos); - if (lpos > pos) { /* Fwd label: add cumulative section offsets. */ - int i; - for (i = secnum; i < DASM_POS2SEC(lpos); i++) - lofs += D->sections[i].ofs; - } else { - lofs -= ofs; /* Bkwd label: unfix offset. */ - } - lofs -= b[pos+1]; /* Short branch ok? */ - if (lofs >= -128-shrink && lofs <= 127) ofs -= shrink; /* Yes. */ - else { noshrink: shrink = 0; } /* No, cannot shrink op. */ - } - b[pos+1] = shrink; - pos += 2; - break; - } - /* fallthrough */ - case DASM_SPACE: case DASM_IMM_LG: case DASM_VREG: p++; - /* fallthrough */ - case DASM_DISP: case DASM_IMM_S: case DASM_IMM_B: case DASM_IMM_W: - case DASM_IMM_D: case DASM_IMM_WB: case DASM_IMM_DB: - case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: pos++; break; - case DASM_LABEL_LG: p++; - /* fallthrough */ - case DASM_LABEL_PC: b[pos++] += ofs; break; /* Fix label offset. */ - case DASM_ALIGN: ofs -= (b[pos++]+ofs)&*p++; break; /* Adjust ofs. */ - case DASM_EXTERN: p += 2; break; - case DASM_ESC: op = *p++; break; - case DASM_MARK: break; - case DASM_SECTION: case DASM_STOP: goto stop; - default: op = action; break; - } - } - stop: (void)0; - } - ofs += sec->ofs; /* Next section starts right after current section. */ - } - - D->codesize = ofs; /* Total size of all code sections */ - *szp = ofs; - return DASM_S_OK; -} - -#define dasmb(x) *cp++ = (unsigned char)(x) -#ifndef DASM_ALIGNED_WRITES -#define dasmw(x) \ - do { *((unsigned short *)cp) = (unsigned short)(x); cp+=2; } while (0) -#define dasmd(x) \ - do { *((unsigned int *)cp) = (unsigned int)(x); cp+=4; } while (0) -#define dasmq(x) \ - do { *((unsigned long long *)cp) = (unsigned long long)(x); cp+=8; } while (0) -#else -#define dasmw(x) do { dasmb(x); dasmb((x)>>8); } while (0) -#define dasmd(x) do { dasmw(x); dasmw((x)>>16); } while (0) -#define dasmq(x) do { dasmd(x); dasmd((x)>>32); } while (0) -#endif -static unsigned char *dasma_(unsigned char *cp, ptrdiff_t x) -{ - if (sizeof(ptrdiff_t) == 8) - dasmq((unsigned long long)x); - else - dasmd((unsigned int)x); - return cp; -} -#define dasma(x) (cp = dasma_(cp, (x))) - -/* Pass 3: Encode sections. */ -int dasm_encode(Dst_DECL, void *buffer) -{ - dasm_State *D = Dst_REF; - unsigned char *base = (unsigned char *)buffer; - unsigned char *cp = base; - int secnum; - - /* Encode all code sections. No support for data sections (yet). */ - for (secnum = 0; secnum < D->maxsection; secnum++) { - dasm_Section *sec = D->sections + secnum; - int *b = sec->buf; - int *endb = sec->rbuf + sec->pos; - - while (b != endb) { - dasm_ActList p = D->actionlist + *b++; - unsigned char *mark = NULL; - while (1) { - int action = *p++; - int n = (action >= DASM_DISP && action <= DASM_ALIGN) ? *b++ : 0; - switch (action) { - case DASM_DISP: if (!mark) mark = cp; { - unsigned char *mm = mark; - if (*p != DASM_IMM_DB && *p != DASM_IMM_WB) mark = NULL; - if (n == 0) { int mrm = mm[-1]&7; if (mrm == 4) mrm = mm[0]&7; - if (mrm != 5) { mm[-1] -= 0x80; break; } } - if (((n+128) & -256) != 0) goto wd; else mm[-1] -= 0x40; - } - /* fallthrough */ - case DASM_IMM_S: case DASM_IMM_B: wb: dasmb(n); break; - case DASM_IMM_DB: if (((n+128)&-256) == 0) { - db: if (!mark) mark = cp; mark[-2] += 2; mark = NULL; goto wb; - } else mark = NULL; - /* fallthrough */ - case DASM_IMM_D: wd: dasmd(n); break; - case DASM_IMM_WB: if (((n+128)&-256) == 0) goto db; else mark = NULL; - /* fallthrough */ - case DASM_IMM_W: dasmw(n); break; - case DASM_VREG: { - int t = *p++; - unsigned char *ex = cp - (t&7); - if ((n & 8) && t < 0xa0) { - if (*ex & 0x80) ex[1] ^= 0x20 << (t>>6); else *ex ^= 1 << (t>>6); - n &= 7; - } else if (n & 0x10) { - if (*ex & 0x80) { - *ex = 0xc5; ex[1] = (ex[1] & 0x80) | ex[2]; ex += 2; - } - while (++ex < cp) ex[-1] = *ex; - if (mark) mark--; - cp--; - n &= 7; - } - if (t >= 0xc0) n <<= 4; - else if (t >= 0x40) n <<= 3; - else if (n == 4 && t < 0x20) { cp[-1] ^= n; *cp++ = 0x20; } - cp[-1] ^= n; - break; - } - case DASM_REL_LG: p++; if (n >= 0) goto rel_pc; - b++; n = (int)(ptrdiff_t)D->globals[-n-10]; - /* fallthrough */ - case DASM_REL_A: rel_a: - n -= (unsigned int)(ptrdiff_t)(cp+4); goto wd; /* !x64 */ - case DASM_REL_PC: rel_pc: { - int shrink = *b++; - int *pb = DASM_POS2PTR(D, n); if (*pb < 0) { n = pb[1]; goto rel_a; } - n = *pb - ((int)(cp-base) + 4-shrink); - if (shrink == 0) goto wd; - if (shrink == 4) { cp--; cp[-1] = *cp-0x10; } else cp[-1] = 0xeb; - goto wb; - } - case DASM_IMM_LG: - p++; - if (n < 0) { dasma((ptrdiff_t)D->globals[-n-10]); break; } - /* fallthrough */ - case DASM_IMM_PC: { - int *pb = DASM_POS2PTR(D, n); - dasma(*pb < 0 ? (ptrdiff_t)pb[1] : (*pb + (ptrdiff_t)base)); - break; - } - case DASM_LABEL_LG: { - int idx = *p++; - if (idx >= 10) - D->globals[idx-10] = (void *)(base + (*p == DASM_SETLABEL ? *b : n)); - break; - } - case DASM_LABEL_PC: case DASM_SETLABEL: break; - case DASM_SPACE: { int fill = *p++; while (n--) *cp++ = fill; break; } - case DASM_ALIGN: - n = *p++; - while (((cp-base) & n)) *cp++ = 0x90; /* nop */ - break; - case DASM_EXTERN: n = DASM_EXTERN(Dst, cp, p[1], *p); p += 2; goto wd; - case DASM_MARK: mark = cp; break; - case DASM_ESC: action = *p++; - /* fallthrough */ - default: *cp++ = action; break; - case DASM_SECTION: case DASM_STOP: goto stop; - } - } - stop: (void)0; - } - } - - if (base + D->codesize != cp) /* Check for phase errors. */ - return DASM_S_PHASE; - return DASM_S_OK; -} - -/* Get PC label offset. */ -int dasm_getpclabel(Dst_DECL, unsigned int pc) -{ - dasm_State *D = Dst_REF; - if (pc*sizeof(int) < D->pcsize) { - int pos = D->pclabels[pc]; - if (pos < 0) return *DASM_POS2PTR(D, -pos); - if (pos > 0) return -1; /* Undefined. */ - } - return -2; /* Unused or out of range. */ -} - -#ifdef DASM_CHECKS -/* Optional sanity checker to call between isolated encoding steps. */ -int dasm_checkstep(Dst_DECL, int secmatch) -{ - dasm_State *D = Dst_REF; - if (D->status == DASM_S_OK) { - int i; - for (i = 1; i <= 9; i++) { - if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_L|i; break; } - D->lglabels[i] = 0; - } - } - if (D->status == DASM_S_OK && secmatch >= 0 && - D->section != &D->sections[secmatch]) - D->status = DASM_S_MATCH_SEC|(int)(D->section-D->sections); - return D->status; -} -#endif - -#endif /* !FD_DASM_HEADER_ONLY */ diff --git a/src/flamenco/vm/jit/fd_jit.c b/src/flamenco/vm/jit/fd_jit.c deleted file mode 100644 index 70b91479b5..0000000000 --- a/src/flamenco/vm/jit/fd_jit.c +++ /dev/null @@ -1,171 +0,0 @@ -#include "fd_jit_private.h" -#include - -FD_TL fd_vm_t * fd_jit_vm; /* current VM being executed */ -FD_TL fd_sbpf_syscalls_t const * fd_jit_syscalls; /* current syscall table */ - -FD_TL uint fd_jit_segment_cnt; -FD_TL uint fd_jit_mem_sz [ 2*FD_VM_JIT_SEGMENT_MAX ]; -FD_TL ulong fd_jit_mem_haddr[ FD_VM_JIT_SEGMENT_MAX ]; -FD_TL ulong fd_jit_jmp_buf[8]; -FD_TL ulong fd_jit_segfault_vaddr; -FD_TL ulong fd_jit_segfault_rip; - -FD_TL jmp_buf fd_jit_compile_abort; - -FD_TL void * fd_jit_code_section_base; -FD_TL ulong fd_jit_code_section_sz; - -ulong -fd_jit_est_code_sz( ulong bpf_sz ) { - if( FD_UNLIKELY( bpf_sz > (1UL<<24) ) ) return 0UL; /* float32 representation limit */ - return FD_JIT_BLOAT_BASE + (ulong)ceilf( (float)bpf_sz * FD_JIT_BLOAT_MAX ); -} - -ulong -fd_jit_est_scratch_sz( ulong bpf_sz ) { - fd_jit_scratch_layout_t layout[1]; - if( FD_UNLIKELY( !fd_jit_scratch_layout( layout, bpf_sz ) ) ) return 0UL; - return layout->sz; -} - -fd_jit_prog_t * -fd_jit_prog_new( fd_jit_prog_t * jit_prog, - fd_sbpf_program_t const * prog, - fd_sbpf_syscalls_t const * syscalls, - void * code_buf, - ulong code_bufsz, - void * scratch, - ulong scratch_sz, - int * out_err ) { - - memset( jit_prog, 0, sizeof(fd_jit_prog_t) ); - *out_err = FD_VM_ERR_INVAL; - - if( FD_UNLIKELY( setjmp( fd_jit_compile_abort ) ) ) { - /* DASM_M_GROW longjmp() here in case of alloc failure */ - *out_err = FD_VM_ERR_FULL; - return NULL; - } - fd_jit_code_section_base = (void *)1; /* an invalid non-NULL pointer */ - fd_jit_code_section_sz = 0UL; - - uint text_cnt = (uint)prog->text_cnt; - ulong bpf_sz = text_cnt * 8UL; - - /* Prepare custom scratch allocator for DynASM. - Constructors provided by dasm_x86.h heavily rely on realloc() like - semantics. The code below replaces these with pre-allocated - regions out of code_buf and uses the redefined DASM_M_GROW to - detect out-of-memory conditions. */ - - fd_jit_scratch_layout_t layout[1]; - if( FD_UNLIKELY( !fd_jit_scratch_layout( layout, bpf_sz ) ) ) { - *out_err = FD_VM_ERR_FULL; - return NULL; - } - if( FD_UNLIKELY( layout->sz > scratch_sz ) ) { - *out_err = FD_VM_ERR_FULL; - return NULL; - } - - dasm_State * d = fd_jit_prepare( scratch, layout ); - - fd_jit_compile( &d, prog, syscalls ); - /* above longjmp()'s to fd_jit_compile_abort on failure */ - - ulong code_sz; - dasm_link( &d, &code_sz ); - if( FD_UNLIKELY( code_sz > code_bufsz ) ) { - *out_err = FD_VM_ERR_FULL; - return NULL; - } - - dasm_encode( &d, code_buf ); - jit_prog->entrypoint = fd_jit_get_entrypoint(); - jit_prog->first_rip = (ulong)code_buf + (ulong)dasm_getpclabel( &d, (uint)prog->entry_pc ); - jit_prog->code_buf = code_buf; - jit_prog->code_sz = code_sz; - - /* Would ordinarily call dasm_free here, but no need, since all - memory was allocated in scratch and is released on function return. */ - //dasm_free( &d ); - - FD_COMPILER_MFENCE(); - jit_prog->magic = FD_JIT_PROG_MAGIC; - FD_COMPILER_MFENCE(); - - *out_err = FD_VM_SUCCESS; - return jit_prog; -} - -void * -fd_jit_prog_delete( fd_jit_prog_t * prog ) { - if( FD_UNLIKELY( prog->magic != FD_JIT_PROG_MAGIC ) ) { - FD_LOG_WARNING(( "invalid magic!" )); - } - FD_COMPILER_MFENCE(); - prog->magic = 0UL; - FD_COMPILER_MFENCE(); - return NULL; -} - -int -fd_jit_vm_attach( fd_vm_t * vm ) { - fd_jit_vm = vm; - fd_jit_syscalls = vm->syscalls; - - /* ABIv1 has 6 segments only */ - fd_jit_segment_cnt = 6UL; - for( ulong j=0UL; jregion_haddr[ j ]; - fd_jit_mem_sz [ j*2 ] = vm->region_st_sz[ j ]; - fd_jit_mem_sz [ j*2+1 ] = vm->region_ld_sz[ j ]; - } - - fd_jit_segfault_vaddr = 0UL; - fd_jit_segfault_rip = 0UL; - - return FD_VM_SUCCESS; -} - -void -fd_jit_vm_detach( void ) { - fd_jit_segment_cnt = 0U; - fd_jit_segfault_vaddr = 0UL; - fd_jit_segfault_rip = 0UL; -} - -FD_FN_PURE int -fd_jit_vm_compatible( fd_vm_t const * vm ) { - if( vm->input_mem_regions_cnt != 1 ) return FD_VM_ERR_UNSUP; - if( vm->input_mem_regions[0].vaddr_offset != 0 ) return FD_VM_ERR_UNSUP; - return FD_VM_SUCCESS; -} - -int -fd_jit_exec( fd_jit_prog_t * jit_prog, - fd_vm_t * vm ) { - int err = fd_jit_vm_compatible( vm ); - if( FD_UNLIKELY( err!=FD_VM_SUCCESS ) ) return err; - fd_jit_vm_attach( vm ); - err = jit_prog->entrypoint( jit_prog->first_rip ); - fd_jit_vm_detach(); - return err; -} - -/* fd_dasm_grow_check gets called when DynASM tries to grow a buffer - using realloc(). We stubbed out realloc(), so we just check if the - requested buffer is sufficiently large. If it's not, we abort via - longjmp(). */ - -void -fd_dasm_grow_check( void * ptr, - ulong min_sz ) { - if( FD_UNLIKELY( ptr!=fd_jit_code_section_base ) ) goto fail; - if( FD_UNLIKELY( min_sz>fd_jit_code_section_sz ) ) goto fail; - return; -fail: - FD_LOG_WARNING(( "Aborting JIT compile" )); - longjmp( fd_jit_compile_abort, 1 ); -} diff --git a/src/flamenco/vm/jit/fd_jit.h b/src/flamenco/vm/jit/fd_jit.h deleted file mode 100644 index af26e02729..0000000000 --- a/src/flamenco/vm/jit/fd_jit.h +++ /dev/null @@ -1,125 +0,0 @@ -#ifndef HEADER_fd_src_flamenco_vm_jit_fd_jit_h -#define HEADER_fd_src_flamenco_vm_jit_fd_jit_h - -/* fd_vm_jit.h provides an API to transpile sBPF programs to native - machine code. Currently only supports x86 with threads. - - FIXME This is not a "JIT" compiler. Fix the naming ... - - WARNING: WORK IN PROGRESS! - This is an experimental version of the Firedancer JIT compiler. - It is disabled in production. There are known security issues in - this code. It is not covered by the Firedancer bug bounty program. */ - -#include "../fd_vm.h" -#include "../../../ballet/sbpf/fd_sbpf_loader.h" - -/* fd_jit_entrypoint is the entrypoint function of JIT compiled code. - first_rip is a pointer to the x86 instruction in the host address - space that corresponds to the BPF entrypoint. */ - -typedef int (* fd_jit_entrypoint_t)( ulong first_rip ); - -/* fd_jit_prog_t is an instance of a JIT compiled program. It is not - relocatable since the generated code effectively contains absolute - addresses. - - fd_jit_prog_t references an external "code" buffer (the memory region - storing executable code). During the lifetime of a fd_jit_prog_t - object, the backing "code" buffer may not be moved, freed, or written - to */ - -struct fd_jit_prog { - ulong magic; - void * code_buf; - ulong code_sz; - fd_jit_entrypoint_t entrypoint; - ulong first_rip; -}; - -typedef struct fd_jit_prog fd_jit_prog_t; - -#define FD_JIT_PROG_MAGIC (0x2c19d91e7ca38c6bUL) - -#if FD_HAS_X86 && FD_HAS_THREADS - -/* fd_jit_est_code_sz returns the estimated x86 code footprint for a - BPF program with bpf_sz .text size. Returns zero if bpf_sz is too - large. - - fd_jit_est_scratch_sz returns the estimated scratch memory size - required to JIT compile a bpf program with bpf_sz .text size. - Returns zero if bpf_sz is too large. - - This estimation is a "best guess", some programs will exceed this - size requirement. */ - -ulong -fd_jit_est_code_sz( ulong bpf_sz ); - -/* fd_jit_est_scratch_sz returns the estimated scratch buffer size - required to call fd_jit_prog_new. This buffer is used to store - intermediate results during compilation. */ - -ulong -fd_jit_est_scratch_sz( ulong bpf_sz ); - -/* fd_jit_prog_new transpiles an sBPF program into native machine code. - - @param jit_prog Output JIT program -- Fully initialized on success - @param prog sBPF program to compile - @param syscalls fd_map_dynamic of enabled syscalls - @param code_buf Buffer to place machine code in - @param code_bufsz Size of code_buf buffer - @param scratch Temporary buffer (released on return) - @param scratch_sz Size of scratch buffer - @param out_err *out_err set to FD_VM_{SUCCESS,ERR_{...}} - - Writes up to scratch_sz bytes for arbitrary use to scratch. This - memory is always released back to the caller on return. - - Writes up to code_bufsz bytes of x86 bytecode to code_buf. The - number of bytes written is set to jit_prog->code_sz. A read interest - remains for the lifetime of the jit_prog object. - - On success, initializes and returns jit_prog. - - On failure, returns NULL and releases any memory back to the caller. - Reasons include FD_VM_ERR_{...}: - - FULL Not enough output or scratch space to compile - FIXME introduce separate error codes for scratch and code sz - UNSUP JIT compiler does not support the given executable. */ - -fd_jit_prog_t * -fd_jit_prog_new( fd_jit_prog_t * jit_prog, - fd_sbpf_program_t const * prog, - fd_sbpf_syscalls_t const * syscalls, - void * code_buf, - ulong code_bufsz, - void * scratch, - ulong scratch_sz, - int * out_err ); - -/* fd_jit_prog_delete releases the memory backing prog and code_buf back - to the caller. */ - -void * -fd_jit_prog_delete( fd_jit_prog_t * prog ); - -/* fd_jit_exec executes a compiled program from entrypoint to halt or - fault. - - On failure, returns NULL and releases any memory back to the caller. - Reasons include FD_VM_ERR_{...}: - - UNSUP JIT compiler does not support the given execution context - (e.g. complex memory map) */ - -int -fd_jit_exec( fd_jit_prog_t * jit_prog, - fd_vm_t * vm ); - -#endif /* FD_HAS_X86 && FD_HAS_THREADS */ - -#endif /* HEADER_fd_src_flamenco_vm_jit_fd_jit_h */ diff --git a/src/flamenco/vm/jit/fd_jit_compiler.c b/src/flamenco/vm/jit/fd_jit_compiler.c deleted file mode 100644 index 32c2da97be..0000000000 --- a/src/flamenco/vm/jit/fd_jit_compiler.c +++ /dev/null @@ -1,1555 +0,0 @@ -/* -** This file has been pre-processed with DynASM. -** https://luajit.org/dynasm.html -** DynASM version 1.5.0, DynASM x64 version 1.5.0 -** DO NOT EDIT! The original file is in "fd_jit_compiler.dasc". -*/ - -#line 1 "fd_jit_compiler.dasc" -/* WARNING: WORK IN PROGRESS! - This is an experimental version of the Firedancer JIT compiler. - It is disabled in production. There are known security issues in - this code. It is not covered by the Firedancer bug bounty policy. - - ### fd_vm_interp compatibility - - fd_jit aims for exact compatibility with fd_vm_interp unless - otherwise noted. - - ### Optimizations - - The fd_jit transpiler does not implement any optimizations. - - ### Error Handling - - VM faults (e.g. segfault, divide by zero, CU overrun, invalid call - destination, program EOF, syscall error) are detected reliably and - deterministically. - - However, after a fault happens, any VM state becomes unreliable - (non-deterministic or undefined). Unreliable fault state include CU - left, current PC, and the error code. - - A VM fault usually triggers ->longjmp, transitioning back into the - fd_jit_execute call frame. */ - -#include "../../../util/fd_util_base.h" - -/* Include dynasm headers. These fail to compile when some strict - checks are enabled. */ - -void fd_dasm_grow_check( void * ptr, ulong min_sz ); -#define DASM_M_GROW(ctx, t, p, sz, need) (fd_dasm_grow_check( (p), (need) )) -#define DASM_M_FREE(ctx, p, sz) do{}while(0) - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wsign-conversion" -#pragma GCC diagnostic ignored "-Wimplicit-fallthrough" -#include "dasm_proto.h" -#include "dasm_x86.h" -#pragma GCC diagnostic pop - -#include "fd_jit_private.h" -#include "../fd_vm_private.h" - -//| .arch x64 -#if DASM_VERSION != 10500 -#error "Version mismatch between DynASM and included encoding engine" -#endif -#line 49 "fd_jit_compiler.dasc" -//| .actionlist actions -static const unsigned char actions[1657] = { - 254,0,248,10,100,72,139,4,37,237,72,137,176,233,76,137,152,233,76,137,160, - 233,76,137,168,233,76,137,176,233,76,137,184,233,72,137,152,233,72,137,136, - 233,76,137,128,233,76,137,136,233,76,137,144,233,195,255,248,11,100,72,139, - 4,37,237,72,139,176,233,76,139,152,233,76,139,160,233,76,139,168,233,76,139, - 176,233,76,139,184,233,72,139,152,233,72,139,136,233,76,139,128,233,76,139, - 136,233,76,139,144,233,195,255,248,12,137,252,250,72,193,252,239,32,209,231, - 252,233,244,13,255,248,14,137,252,250,72,193,252,239,32,141,60,125,1,0,0, - 0,255,248,13,255,252,233,244,15,255,252,233,244,16,255,248,15,137,252,250, - 72,193,252,239,32,100,59,60,37,237,15,131,244,17,1,213,15,130,244,17,100, - 59,20,189,237,15,131,244,17,49,252,237,131,252,255,2,184,0,16,0,0,15,68,232, - 33,252,234,15,131,244,17,100,72,3,20,252,253,237,195,255,248,16,100,59,60, - 37,237,15,131,244,17,1,213,15,130,244,17,100,59,20,189,237,15,131,244,17, - 100,72,3,20,252,253,237,195,255,248,17,72,193,231,32,72,9,215,100,72,137, - 60,37,237,72,139,60,36,100,72,137,60,37,237,252,233,244,18,255,248,19,73, - 187,237,237,73,137,155,233,73,137,171,233,77,137,99,16,77,137,107,24,77,137, - 115,32,77,137,123,40,72,141,84,36,8,73,137,83,48,72,139,20,36,73,137,83,56, - 49,192,49,210,195,255,248,20,72,137,252,248,186,1,0,0,0,72,191,237,237,72, - 139,159,233,72,139,175,233,76,139,103,16,76,139,111,24,76,139,119,32,76,139, - 127,40,72,139,103,48,252,255,119,56,195,255,248,21,232,244,10,232,244,10, - 72,137,229,72,131,228,252,240,72,131,252,236,16,100,76,139,28,37,237,100, - 76,139,151,233,76,137,223,72,139,176,233,72,139,144,233,72,139,136,233,76, - 139,128,233,76,139,136,233,76,141,152,233,65,83,65,252,255,210,133,252,255, - 15,133,244,18,72,137,252,236,232,244,11,195,255,248,22,100,72,139,60,37,237, - 72,252,255,135,233,95,83,81,65,80,65,81,73,129,194,0,32,0,0,252,255,231,255, - 248,23,100,72,139,60,37,237,72,252,255,143,233,95,65,89,65,88,89,91,73,129, - 252,234,0,32,0,0,252,255,231,255,248,18,191,237,252,233,244,20,255,248,24, - 255,232,244,19,133,210,15,133,244,247,255,232,244,11,72,131,252,236,32,252, - 255,215,72,137,252,247,232,244,20,248,1,195,255,249,255,64,129,192,240,43, - 239,255,252,233,245,255,72,129,192,240,35,239,255,64,1,192,240,131,240,51, - 255,72,1,192,240,131,240,35,255,64,129,232,240,43,239,255,72,129,252,248, - 240,35,239,255,15,132,245,255,72,129,232,240,35,239,255,64,49,192,240,131, - 240,51,255,72,199,192,240,35,237,255,64,41,192,240,131,240,51,255,72,57,192, - 240,131,240,35,15,132,245,255,72,41,192,240,131,240,35,255,64,105,192,240, - 131,240,51,239,255,72,129,252,248,240,35,239,15,135,245,255,72,105,192,240, - 131,240,35,239,255,64,15,175,192,240,132,240,52,255,72,57,192,240,131,240, - 35,15,135,245,255,72,15,175,192,240,132,240,36,255,64,144,240,42,49,210,191, - 237,252,247,252,247,64,144,240,42,255,72,129,252,248,240,35,239,15,131,245, - 255,72,144,240,34,49,210,72,199,199,237,72,252,247,252,247,72,144,240,34, - 255,64,133,192,240,131,240,51,15,132,244,18,255,64,184,240,42,1,0,0,0,255, - 64,144,240,42,49,210,64,252,247,252,240,240,43,64,144,240,42,255,72,57,192, - 240,131,240,35,15,131,245,255,72,133,192,240,131,240,35,15,132,244,18,255, - 72,144,240,34,49,210,72,252,247,252,240,240,35,72,144,240,34,255,64,129,200, - 240,43,239,255,72,252,247,192,240,35,237,15,133,245,255,72,129,200,240,35, - 239,255,64,9,192,240,131,240,51,255,72,133,192,240,131,240,35,15,133,245, - 255,72,9,192,240,131,240,35,255,64,129,224,240,43,239,255,72,129,252,248, - 240,35,239,15,133,245,255,72,129,224,240,35,239,255,64,33,192,240,131,240, - 51,255,72,57,192,240,131,240,35,15,133,245,255,72,33,192,240,131,240,35,255, - 72,141,184,253,240,3,233,189,3,0,0,0,232,244,12,64,139,2,240,139,255,72,141, - 184,253,240,3,233,189,3,0,0,0,232,244,14,199,2,237,255,72,141,184,253,240, - 3,233,189,3,0,0,0,232,244,14,64,137,2,240,139,255,64,193,224,240,43,235,255, - 72,129,252,248,240,35,239,15,143,245,255,72,193,224,240,35,235,255,72,141, - 184,253,240,3,233,189,1,0,0,0,232,244,12,64,49,192,240,131,240,51,102,64, - 139,2,240,139,255,72,141,184,253,240,3,233,189,1,0,0,0,232,244,14,102,199, - 2,236,255,72,141,184,253,240,3,233,189,1,0,0,0,232,244,14,64,137,2,240,139, - 255,64,136,193,240,131,64,211,224,240,43,255,72,57,192,240,131,240,35,15, - 143,245,255,64,136,193,240,131,72,211,224,240,35,255,72,141,184,253,240,3, - 233,49,252,237,232,244,12,255,64,49,192,240,131,240,51,64,138,2,240,131,255, - 72,141,184,253,240,3,233,49,252,237,232,244,14,198,2,235,255,72,141,184,253, - 240,3,233,49,252,237,232,244,14,64,136,2,240,131,255,64,193,232,240,43,235, - 255,72,129,252,248,240,35,239,15,141,245,255,72,193,232,240,35,235,255,72, - 141,184,253,240,3,233,189,7,0,0,0,232,244,12,72,139,2,240,131,255,72,141, - 184,253,240,3,233,189,7,0,0,0,232,244,14,199,2,237,199,66,4,0,0,0,0,255,72, - 141,184,253,240,3,233,189,7,0,0,0,232,244,14,72,137,2,240,131,255,64,136, - 193,240,131,64,211,232,240,43,255,72,57,192,240,131,240,35,15,141,245,255, - 64,136,193,240,131,72,211,232,240,35,255,64,252,247,216,240,43,255,232,244, - 22,232,245,255,72,199,199,237,232,244,21,255,72,252,247,216,240,35,255,64, - 144,240,42,49,210,191,237,252,247,252,247,64,135,208,240,43,255,232,244,23, - 195,255,64,184,240,42,0,0,0,0,255,64,144,240,42,49,210,64,252,247,252,240, - 240,43,64,135,208,240,43,255,72,144,240,34,49,210,72,252,247,252,240,240, - 35,72,135,208,240,35,255,64,129,252,240,240,43,239,255,72,129,252,248,240, - 35,239,15,130,245,255,72,129,252,240,240,35,239,255,72,57,192,240,131,240, - 35,15,130,245,255,72,49,192,240,131,240,35,255,64,184,240,42,237,255,72,129, - 252,248,240,35,239,15,134,245,255,64,137,192,240,131,240,51,255,72,57,192, - 240,131,240,35,15,134,245,255,72,137,192,240,131,240,35,255,64,193,252,248, - 240,43,235,255,72,129,252,248,240,35,239,15,140,245,255,72,193,252,248,240, - 35,235,255,64,136,193,240,131,64,211,252,248,240,43,255,72,57,192,240,131, - 240,35,15,140,245,255,64,136,193,240,131,72,211,252,248,240,35,255,72,129, - 252,248,240,35,239,15,142,245,255,64,15,183,192,240,132,240,52,102,64,193, - 200,240,43,8,255,64,15,200,240,43,255,72,15,200,240,35,255,72,57,192,240, - 131,240,35,15,142,245,255,248,25,252,233,244,18,255 -}; - -#line 50 "fd_jit_compiler.dasc" -//| .globals fd_jit_lbl_ -enum { - fd_jit_lbl_save_regs, - fd_jit_lbl_restore_regs, - fd_jit_lbl_fd_jit_vm_translate_ro, - fd_jit_lbl_fd_jit_vm_translate, - fd_jit_lbl_fd_jit_vm_translate_rw, - fd_jit_lbl_fd_jit_vm_translate_abiv1, - fd_jit_lbl_fd_jit_vm_translate_abiv2, - fd_jit_lbl_translate_fail, - fd_jit_lbl_vm_fault, - fd_jit_lbl_setjmp, - fd_jit_lbl_longjmp, - fd_jit_lbl_emulate_syscall, - fd_jit_lbl_call_stack_push, - fd_jit_lbl_call_stack_pop, - fd_jit_lbl_entrypoint, - fd_jit_lbl_overrun, - fd_jit_lbl__MAX -}; -#line 51 "fd_jit_compiler.dasc" -//| .section code -#define DASM_SECTION_CODE 0 -#define DASM_MAXSECTION 1 -#line 52 "fd_jit_compiler.dasc" - -fd_jit_scratch_layout_t * -fd_jit_scratch_layout( fd_jit_scratch_layout_t * layout, - ulong bpf_sz ) { - - if( FD_UNLIKELY( bpf_sz > (1UL<<24) ) ) return NULL; - ulong text_cnt = bpf_sz / 8UL; - - /* These "magic" values are taken from dasm_x86.h */ - - ulong dasm_sz = DASM_PSZ( DASM_MAXSECTION ); /* dasm_x86.h(89) */ - ulong lglabels_sz = (10+fd_jit_lbl__MAX)*sizeof(int); /* dasm_x86.h(119) */ - ulong pclabels_sz = text_cnt*sizeof(int); /* dasm_x86.h(127) */ - ulong code_sz = fd_jit_est_code_sz( bpf_sz ); - - memset( layout, 0, sizeof(fd_jit_scratch_layout_t) ); - FD_SCRATCH_ALLOC_INIT( l, 0 ); - layout->dasm_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, dasm_sz ); - layout->dasm_sz = dasm_sz; - layout->lglabels_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, lglabels_sz ); - layout->lglabels_sz = lglabels_sz; - layout->pclabels_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, pclabels_sz ); - layout->pclabels_sz = pclabels_sz; - layout->code_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, code_sz ); - layout->code_sz = code_sz; - layout->sz = (ulong)FD_SCRATCH_ALLOC_FINI( l, 16UL ); - return layout; -} - -dasm_State * -fd_jit_prepare( void * scratch, - fd_jit_scratch_layout_t const * layout ) { - - /* Custom dasm_init */ - dasm_State * d = (void *)( (ulong)scratch + layout->dasm_off ); - fd_memset( d, 0, layout->dasm_sz ); - d->psize = layout->dasm_sz; - d->maxsection = DASM_MAXSECTION; - - /* Custom dasm_setupglobal */ - d->globals = fd_jit_labels; - d->lglabels = (void *)( (ulong)scratch + layout->lglabels_off ); - d->lgsize = layout->lglabels_sz; - - /* Custom dasm_growpc */ - d->pcsize = layout->pclabels_sz; - d->pclabels = (void *)( (ulong)scratch + layout->pclabels_off ); - - /* Setup encoder. Zeros lglabels and pclabels. */ - dasm_setup( &d, actions ); - - /* Preallocate space for .code section - See dasm_x86.h(172) */ - dasm_Section * code = d->sections + 0; - code->buf = (void *)( (ulong)scratch + layout->code_off ); - code->bsize = layout->code_sz; - code->pos = 0; - code->rbuf = code->buf; - code->epos = (int)( ((ulong)code->bsize / sizeof(int)) - DASM_MAXSECPOS ); - code->ofs = 0; - - return d; -} - -/* Compile time thread locals */ - -FD_TL void * fd_jit_labels[ FD_JIT_LABEL_CNT ]; -FD_STATIC_ASSERT( sizeof(fd_jit_labels)==fd_jit_lbl__MAX*8, label_cnt ); - -/* Mapping between sBPF registers and x86_64 registers ***************** - - This mapping is valid just before a translated sBPF instruction is - about to be executed. (At the `=>next_label` token in the code gen - loop) - - BPF | r0 | r1 | r2 | r3 | r4 | r5 | r6 | r7 | r8 | r9 | r10 - X86 | rsi | r11 | r12 | r13 | r14 | r15 | rbx | rcx | r8 | r9 | r10 - - x86_64 GPRs rax, rdi, rdx, rbp do not map to sBPF registers. Those can - be used as scratch registers for complex opcodes. - - Note that this mapping cannot be trivially changed. Certain x86 - instructions (like div) have hardcoded register accesses which the - JIT code works around. - - dynasm macros bpf_r{...} resolve to 64-bit register names. - - reg_bpf2x86 is indexed by sBPF register numbers and resolves to the - x86_64 dynasm register index. */ - -static uchar const reg_bpf2x86[11] = { - [ 0] = FD_DASM_RSI, - //| .define bpf_r0, rsi - [ 1] = FD_DASM_R11, - //| .define bpf_r1, r11 - [ 2] = FD_DASM_R12, - //| .define bpf_r2, r12 - [ 3] = FD_DASM_R13, - //| .define bpf_r3, r13 - [ 4] = FD_DASM_R14, - //| .define bpf_r4, r14 - [ 5] = FD_DASM_R15, - //| .define bpf_r5, r15 - [ 6] = FD_DASM_RBX, - //| .define bpf_r6, rbx - [ 7] = FD_DASM_RCX, - //| .define bpf_r7, rcx - [ 8] = FD_DASM_R8, - //| .define bpf_r8, r8 - [ 9] = FD_DASM_R9, - //| .define bpf_r9, r9 - [10] = FD_DASM_R10 - //| .define bpf_r10, r10 -}; - - -/* JIT compiler *******************************************************/ - -void -fd_jit_compile( struct dasm_State ** Dst, - fd_sbpf_program_t const * prog, - fd_sbpf_syscalls_t const * syscalls ) { - - int const abiv2 = 0; /* For now, only support ABIv1 */ - - //| .code - dasm_put(Dst, 0); -#line 178 "fd_jit_compiler.dasc" - - /* Derive offsets of thread locals in FS "segment" */ - -# if defined(__FSGSBASE__) - ulong fs_base = __builtin_ia32_rdfsbase64(); -# else - ulong fs_base; __asm__( "mov %%fs:0, %0" : "=r"(fs_base) ); -# endif -# define FS_RELATIVE(ptr) ((uint)( (ulong)(ptr) - fs_base )) - uint fd_jit_vm_tpoff = FS_RELATIVE( &fd_jit_vm ); - uint fd_jit_syscalls_tpoff = FS_RELATIVE( &fd_jit_syscalls ); - uint fd_jit_segment_cnt_tpoff = FS_RELATIVE( &fd_jit_segment_cnt ); - uint fd_jit_mem_sz_tpoff = FS_RELATIVE( fd_jit_mem_sz ); - uint fd_jit_mem_haddr_tpoff = FS_RELATIVE( fd_jit_mem_haddr ); - uint fd_jit_segfault_vaddr_tpoff = FS_RELATIVE( &fd_jit_segfault_vaddr ); - uint fd_jit_segfault_rip_tpoff = FS_RELATIVE( &fd_jit_segfault_rip ); -# undef FD_RELATIVE - - //|->save_regs: - //| fs - //| mov rax, [fd_jit_vm_tpoff] - //| mov [rax + offsetof(fd_vm_t, reg[ 0])], bpf_r0 - //| mov [rax + offsetof(fd_vm_t, reg[ 1])], bpf_r1 - //| mov [rax + offsetof(fd_vm_t, reg[ 2])], bpf_r2 - //| mov [rax + offsetof(fd_vm_t, reg[ 3])], bpf_r3 - //| mov [rax + offsetof(fd_vm_t, reg[ 4])], bpf_r4 - //| mov [rax + offsetof(fd_vm_t, reg[ 5])], bpf_r5 - //| mov [rax + offsetof(fd_vm_t, reg[ 6])], bpf_r6 - //| mov [rax + offsetof(fd_vm_t, reg[ 7])], bpf_r7 - //| mov [rax + offsetof(fd_vm_t, reg[ 8])], bpf_r8 - //| mov [rax + offsetof(fd_vm_t, reg[ 9])], bpf_r9 - //| mov [rax + offsetof(fd_vm_t, reg[10])], bpf_r10 - //| ret - dasm_put(Dst, 2, fd_jit_vm_tpoff, offsetof(fd_vm_t, reg[ 0]), offsetof(fd_vm_t, reg[ 1]), offsetof(fd_vm_t, reg[ 2]), offsetof(fd_vm_t, reg[ 3]), offsetof(fd_vm_t, reg[ 4]), offsetof(fd_vm_t, reg[ 5]), offsetof(fd_vm_t, reg[ 6]), offsetof(fd_vm_t, reg[ 7]), offsetof(fd_vm_t, reg[ 8]), offsetof(fd_vm_t, reg[ 9]), offsetof(fd_vm_t, reg[10])); -#line 211 "fd_jit_compiler.dasc" - - //|->restore_regs: - //| fs - //| mov rax, [fd_jit_vm_tpoff] - //| mov bpf_r0, [rax + offsetof(fd_vm_t, reg[ 0])] - //| mov bpf_r1, [rax + offsetof(fd_vm_t, reg[ 1])] - //| mov bpf_r2, [rax + offsetof(fd_vm_t, reg[ 2])] - //| mov bpf_r3, [rax + offsetof(fd_vm_t, reg[ 3])] - //| mov bpf_r4, [rax + offsetof(fd_vm_t, reg[ 4])] - //| mov bpf_r5, [rax + offsetof(fd_vm_t, reg[ 5])] - //| mov bpf_r6, [rax + offsetof(fd_vm_t, reg[ 6])] - //| mov bpf_r7, [rax + offsetof(fd_vm_t, reg[ 7])] - //| mov bpf_r8, [rax + offsetof(fd_vm_t, reg[ 8])] - //| mov bpf_r9, [rax + offsetof(fd_vm_t, reg[ 9])] - //| mov bpf_r10, [rax + offsetof(fd_vm_t, reg[10])] - //| ret - dasm_put(Dst, 56, fd_jit_vm_tpoff, offsetof(fd_vm_t, reg[ 0]), offsetof(fd_vm_t, reg[ 1]), offsetof(fd_vm_t, reg[ 2]), offsetof(fd_vm_t, reg[ 3]), offsetof(fd_vm_t, reg[ 4]), offsetof(fd_vm_t, reg[ 5]), offsetof(fd_vm_t, reg[ 6]), offsetof(fd_vm_t, reg[ 7]), offsetof(fd_vm_t, reg[ 8]), offsetof(fd_vm_t, reg[ 9]), offsetof(fd_vm_t, reg[10])); -#line 227 "fd_jit_compiler.dasc" - - /* Helper macros for BPF-to-x86 function calls */ - - //|.macro enter_x86_frame - //| call ->save_regs - //| mov rbp, rsp - //| and rsp, -16 - //|.endmacro - - //|.macro leave_x86_frame - //| mov rsp, rbp - //| call ->restore_regs - //|.endmacro - - /* Address translation macros - - The translate_{rw,ro}_{1,2,4,8} macros perform address translation - and access permission checks for {read-write,read-only} accesses of - {1,2,4,8} bytes. The compiler may inline this macro for each - translated sBPF instruction, so these should be optimized for small - size. - - Prior to the macro, rdi holds an address in the virtual address - space (untrusted in [0,2^64)). If translation and permission - checks succeed, rdx holds the translated address in the host - address space. On failure jumps to sigsegv. Reasons for failure - include access to out-of-bounds memory, unaligned address, access - permission error. */ - - //| .define translate_in, rdi - //| .define translate_out, rdx - /* FIXME make output argument rdi instead of rdx */ - - //|->fd_jit_vm_translate_ro: - //| mov edx, edi // segment offset - //| shr rdi, 32 // segment index - //| shl edi, 1 - //| jmp ->fd_jit_vm_translate - dasm_put(Dst, 110); -#line 265 "fd_jit_compiler.dasc" - - //|->fd_jit_vm_translate_rw: - //| mov edx, edi // segment offset - //| shr rdi, 32 // segment index - //| lea edi, [edi*2+1] - dasm_put(Dst, 127); -#line 270 "fd_jit_compiler.dasc" - /* fallthrough */ - - //|->fd_jit_vm_translate: - dasm_put(Dst, 145); -#line 273 "fd_jit_compiler.dasc" - if( abiv2 ) { - //| jmp ->fd_jit_vm_translate_abiv1 - dasm_put(Dst, 148); -#line 275 "fd_jit_compiler.dasc" - } else { - //| jmp ->fd_jit_vm_translate_abiv2 - dasm_put(Dst, 153); -#line 277 "fd_jit_compiler.dasc" - } - - /* ABIv1 virtual memory overview - - - There are 6 pages indexed in [0,6) - - The number of readable bytes in a page is variable in [0,2^31) - - The number of writable bytes in a page is equal to the number of - readable bytes or zero - - The first byte of a page in virtual memory is at (index<<32) - - Page 0 is always empty (all bytes are unaddressable) - - Page 2 is 'striped': Virtual addresses with bit 12 set are not - addressable */ - - //|->fd_jit_vm_translate_abiv1: - //| // rdi := virtual address - //| // ebp := size of the access minus 1 - //| - //| // edx := segment offset - //| mov edx, edi - //| - //| // edi := segment index - //| shr rdi, 32 - //| - //| // segment index in bounds? - //| fs - //| cmp edi, [fd_jit_segment_cnt_tpoff] - //| jae ->translate_fail - //| - //| // no multi segment overlap? - //| add ebp, edx - //| jc ->translate_fail - //| - //| // segment offset in bounds? - //| fs - //| cmp edx, [rdi*4 + fd_jit_mem_sz_tpoff] - //| jae ->translate_fail - //| - //| // stack gap? - //| xor ebp, ebp - //| cmp edi, 2 - //| mov eax, 0x1000 - //| cmove ebp, eax - //| and edx, ebp - //| jae ->translate_fail - //| - //| // rdx := host address - //| fs - //| add rdx, [rdi*8 + fd_jit_mem_haddr_tpoff] - //| ret - dasm_put(Dst, 158, fd_jit_segment_cnt_tpoff, fd_jit_mem_sz_tpoff, fd_jit_mem_haddr_tpoff); -#line 326 "fd_jit_compiler.dasc" - - /* ABIv2 virtual memory overview: - - - Virtual memory is described by a page table - - There can be up to 2^16 pages - - Each page is 4 GiB (2^32 byte) aligned in virtual memory - - The number of readable bytes in a page is variable in [0,2^31) - - The number of writable bytes in a page is less or equal to the - number of readable bytes */ - - //|->fd_jit_vm_translate_abiv2: - //| // rdi := virtual address - //| // ebp := size of the access minus 1 - //| - //| // segment index in bounds? - //| fs - //| cmp edi, [fd_jit_segment_cnt_tpoff] - //| jae ->translate_fail - //| - //| //// aligned access? - //| // FIXME do we need an alignment check? - //| //mov eax, edx - //| //and eax, ebp - //| //test eax, eax - //| //jnz ->translate_fail - //| - //| // no multi segment overlap? - //| add ebp, edx - //| jc ->translate_fail - //| - //| // segment offset in bounds? - //| fs - //| cmp edx, [rdi*4 + fd_jit_mem_sz_tpoff] - //| jae ->translate_fail - //| - //| // rdx := host address - //| fs - //| add rdx, [rdi*8 + fd_jit_mem_haddr_tpoff] - //| ret - dasm_put(Dst, 223, fd_jit_segment_cnt_tpoff, fd_jit_mem_sz_tpoff, fd_jit_mem_haddr_tpoff); -#line 365 "fd_jit_compiler.dasc" - - //|->translate_fail: - //| shl rdi, 32 - //| or rdi, rdx - //| fs - //| mov [fd_jit_segfault_vaddr_tpoff], rdi - //| mov rdi, [rsp] - //| fs - //| mov [fd_jit_segfault_rip_tpoff], rdi - //| jmp ->vm_fault - dasm_put(Dst, 258, fd_jit_segfault_vaddr_tpoff, fd_jit_segfault_rip_tpoff); -#line 375 "fd_jit_compiler.dasc" - - //|.macro translate_rw_1 - //| xor ebp, ebp - //| call ->fd_jit_vm_translate_rw - //|.endmacro - - //|.macro translate_rw_2 - //| mov ebp, 1 - //| call ->fd_jit_vm_translate_rw - //|.endmacro - - //|.macro translate_rw_4 - //| mov ebp, 3 - //| call ->fd_jit_vm_translate_rw - //|.endmacro - - //|.macro translate_rw_8 - //| mov ebp, 7 - //| call ->fd_jit_vm_translate_rw - //|.endmacro - - //|.macro translate_ro_1 - //| xor ebp, ebp - //| call ->fd_jit_vm_translate_ro - //|.endmacro - - //|.macro translate_ro_2 - //| mov ebp, 1 - //| call ->fd_jit_vm_translate_ro - //|.endmacro - - //|.macro translate_ro_4 - //| mov ebp, 3 - //| call ->fd_jit_vm_translate_ro - //|.endmacro - - //|.macro translate_ro_8 - //| mov ebp, 7 - //| call ->fd_jit_vm_translate_ro - //|.endmacro - - /* Generate setjmp/longjmp subroutines. These can be called from any - execution state with a valid stack. The JIT uses them to restore a - sane SystemV-ABI context when exiting JIT code. - - These are based on musl libc's setjmp/longjmp implementation. - Copyright 2011-2012 Nicholas J. Kain, licensed under standard MIT license - - setjmp takes no arguments. longjmp takes a 64-bit value in rdi. - When setjmp returns from setjmp, sets rax=0 and rdx=0. When setjmp - returns from longjmp, sets rax to the rdi argument of longjmp, and - sets rdx=1. setjmp preserves rdi. */ - - //|->setjmp: - //| mov64 r11, (ulong)fd_jit_jmp_buf - //| mov [r11+ 0], rbx - //| mov [r11+ 8], rbp - //| mov [r11+16], r12 - //| mov [r11+24], r13 - //| mov [r11+32], r14 - //| mov [r11+40], r15 - //| // save callee's stack pointer - //| // derived by removing our 8 byte stack frame (only return address) - //| lea rdx, [rsp+8] - //| mov [r11+48], rdx - //| // save return address - //| mov rdx, [rsp] - //| mov [r11+56], rdx - //| // normal return - //| xor eax, eax - //| xor edx, edx - //| ret - dasm_put(Dst, 288, (unsigned int)((ulong)fd_jit_jmp_buf), (unsigned int)(((ulong)fd_jit_jmp_buf)>>32), 0, 8); -#line 447 "fd_jit_compiler.dasc" - - //|->longjmp: - //| mov rax, rdi // move first argument to first output register - //| mov edx, 1 // set second output register to 1 - //| mov64 rdi, (ulong)fd_jit_jmp_buf - //| // restore execution state to callee of setjmp - //| mov rbx, [rdi+ 0] - //| mov rbp, [rdi+ 8] - //| mov r12, [rdi+16] - //| mov r13, [rdi+24] - //| mov r14, [rdi+32] - //| mov r15, [rdi+40] - //| mov rsp, [rdi+48] - //| push qword [rdi+56] - //| ret // retpoline - dasm_put(Dst, 341, (unsigned int)((ulong)fd_jit_jmp_buf), (unsigned int)(((ulong)fd_jit_jmp_buf)>>32), 0, 8); -#line 462 "fd_jit_compiler.dasc" - - /* The emulate_syscall function switches from a JIT to an interpreter (C) - execution context and invokes a syscall handler. Register edi is - assumed to hold the byte offset into the vm->syscalls table of the - fd_sbpf_syscalls_t entry to invoke. - On syscall return, switches back to the JIT execution context and - resumes execution after the syscall instruction. */ - - //|->emulate_syscall: - //| call ->save_regs - //| enter_x86_frame - //| sub rsp, 16 - //| fs - //| mov r11, [fd_jit_vm_tpoff] - //| fs - //| mov r10, [rdi + fd_jit_syscalls_tpoff + offsetof(fd_sbpf_syscalls_t, func)] - //| mov rdi, r11 - //| // load BPF r1 through r5 into function arguments - //| // FIXME could avoid spill to memory by shuffling registers - //| mov rsi, [rax + offsetof(fd_vm_t, reg[1])] - //| mov rdx, [rax + offsetof(fd_vm_t, reg[2])] - //| mov rcx, [rax + offsetof(fd_vm_t, reg[3])] - //| mov r8, [rax + offsetof(fd_vm_t, reg[4])] - //| mov r9, [rax + offsetof(fd_vm_t, reg[5])] - //| lea r11, [rax + offsetof(fd_vm_t, reg[0])] - //| push r11 - //| call r10 - //| test edi, edi - //| jnz ->vm_fault - //| leave_x86_frame - //| ret - dasm_put(Dst, 390, fd_jit_vm_tpoff, fd_jit_syscalls_tpoff + offsetof(fd_sbpf_syscalls_t, func), offsetof(fd_vm_t, reg[1]), offsetof(fd_vm_t, reg[2]), offsetof(fd_vm_t, reg[3]), offsetof(fd_vm_t, reg[4]), offsetof(fd_vm_t, reg[5]), offsetof(fd_vm_t, reg[0])); -#line 493 "fd_jit_compiler.dasc" - - /* The call_stack_push function pushes the current program counter and - eBPF registers r6, r7, r8, r9 to the shadow stack. The frame register - (r10) grows upwards. FIXME implement shadow stack overflow. */ - -# define REG(n) (offsetof(fd_vm_t, shadow[0].r##n)) - - //|->call_stack_push: - //| fs - //| mov rdi, [fd_jit_vm_tpoff] - //| // vm->frame_cnt++ - //| inc qword [rdi + offsetof(fd_vm_t, frame_cnt)] - //| // save registers - //| pop rdi - //| push bpf_r6 - //| push bpf_r7 - //| push bpf_r8 - //| push bpf_r9 - //| add bpf_r10, 0x2000 - //| jmp rdi - dasm_put(Dst, 471, fd_jit_vm_tpoff, offsetof(fd_vm_t, frame_cnt)); -#line 513 "fd_jit_compiler.dasc" - - /* The call_stack_pop function undoes the effects of call_stack_push. */ - - //|->call_stack_pop: - //| fs - //| mov rdi, [fd_jit_vm_tpoff] - //| // vm->frame_cnt-- - //| dec qword [rdi + offsetof(fd_vm_t, frame_cnt)] - //| // restore registers - //| pop rdi - //| pop bpf_r9 - //| pop bpf_r8 - //| pop bpf_r7 - //| pop bpf_r6 - //| sub bpf_r10, 0x2000 - //| jmp rdi - dasm_put(Dst, 502, fd_jit_vm_tpoff, offsetof(fd_vm_t, frame_cnt)); -#line 529 "fd_jit_compiler.dasc" - -# undef REG - /* Exception handlers */ - - //|->vm_fault: - //| mov edi, FD_VM_ERR_ABORT - //| jmp ->longjmp - dasm_put(Dst, 534, FD_VM_ERR_SIGABORT); -#line 536 "fd_jit_compiler.dasc" - - /* JIT entrypoint from C code */ - - //|->entrypoint: - dasm_put(Dst, 543); -#line 540 "fd_jit_compiler.dasc" - - /* Create setjmp anchor used to return from JIT */ - - //| call ->setjmp // preserves rdi - //| test edx, edx - //| jnz >1 - dasm_put(Dst, 546); -#line 546 "fd_jit_compiler.dasc" - - /* Enter JIT execution context */ - - //| call ->restore_regs - //| sub rsp, 0x20 // balance call_stack_push - //| call rdi - //| mov rdi, bpf_r0 - //| call ->longjmp - //|1: - //| ret - dasm_put(Dst, 556); -#line 556 "fd_jit_compiler.dasc" - - /* Start translating user code */ - - ulong * const text_start = prog->text; - ulong * text_end = prog->text + prog->text_cnt; - - for( ulong * cur=text_start; cur=text_start && jmpnext_label: - dasm_put(Dst, 578, next_label); -#line 597 "fd_jit_compiler.dasc" - - /* Translate instruction */ - - switch( opcode ) { - - /* 0x00 - 0x0f ******************************************************/ - - case 0x04: /* FD_SBPF_OP_ADD_IMM */ - //| add dst32, imm - dasm_put(Dst, 580, (x86_dst), imm); -#line 606 "fd_jit_compiler.dasc" - break; - - case 0x05: /* FD_SBPF_OP_JA */ - //| jmp =>jmp_dst_lbl - dasm_put(Dst, 587, jmp_dst_lbl); -#line 610 "fd_jit_compiler.dasc" - break; - - case 0x07: /* FD_SBPF_OP_ADD64_IMM */ - //| add dst64, imm - dasm_put(Dst, 591, (x86_dst), imm); -#line 614 "fd_jit_compiler.dasc" - break; - - case 0x0c: /* FD_SBPF_OP_ADD_REG */ - //| add dst32, src32 - dasm_put(Dst, 598, (x86_src), (x86_dst)); -#line 618 "fd_jit_compiler.dasc" - break; - - case 0x0f: /* FD_SBPF_OP_ADD64_REG */ - //| add dst64, src64 - dasm_put(Dst, 606, (x86_src), (x86_dst)); -#line 622 "fd_jit_compiler.dasc" - break; - - /* 0x10 - 0x1f ******************************************************/ - - case 0x14: /* FD_SBPF_OP_SUB_IMM */ - //| sub dst32, imm - dasm_put(Dst, 614, (x86_dst), imm); -#line 628 "fd_jit_compiler.dasc" - break; - - case 0x15: /* FD_SBPF_OP_JEQ_IMM */ - //| cmp dst64, imm - dasm_put(Dst, 621, (x86_dst), imm); -#line 632 "fd_jit_compiler.dasc" - /* pre branch check here ... branchless cu update? */ - //| je =>jmp_dst_lbl - dasm_put(Dst, 629, jmp_dst_lbl); -#line 634 "fd_jit_compiler.dasc" - break; - - case 0x17: /* FD_SBPF_OP_SUB64_IMM */ - //| sub dst64, imm - dasm_put(Dst, 633, (x86_dst), imm); -#line 638 "fd_jit_compiler.dasc" - break; - - case 0x18: /* FD_SBPF_OP_LDQ */ - cur++; { - ulong imm64 = (ulong)imm | ( (ulong)fd_vm_instr_imm( *cur ) << 32 ); - if( imm64==0 ) { - //| xor dst32, dst32 - dasm_put(Dst, 640, (x86_dst), (x86_dst)); -#line 645 "fd_jit_compiler.dasc" - } else { - //| mov dst64, imm64 - dasm_put(Dst, 648, (x86_dst), imm64); -#line 647 "fd_jit_compiler.dasc" - } - break; - } - - case 0x1c: /* FD_SBPF_OP_SUB_REG */ - //| sub dst32, src32 - dasm_put(Dst, 655, (x86_src), (x86_dst)); -#line 653 "fd_jit_compiler.dasc" - break; - - case 0x1d: /* FD_SBPF_OP_JEQ_REG */ - //| cmp dst64, src64 - //| je =>jmp_dst_lbl - dasm_put(Dst, 663, (x86_src), (x86_dst), jmp_dst_lbl); -#line 658 "fd_jit_compiler.dasc" - break; - - case 0x1f: /* FD_SBPF_OP_SUB64_REG */ - //| sub dst64, src64 - dasm_put(Dst, 674, (x86_src), (x86_dst)); -#line 662 "fd_jit_compiler.dasc" - break; - - /* 0x20 - 0x2f ******************************************************/ - - case 0x24: /* FD_SBPF_OP_MUL_IMM */ - /* TODO strength reduction? */ - //| imul dst32, imm - dasm_put(Dst, 682, (x86_dst), (x86_dst), imm); -#line 669 "fd_jit_compiler.dasc" - break; - - case 0x25: /* FD_SBPF_OP_JGT_IMM */ - //| cmp dst64, imm - //| ja =>jmp_dst_lbl - dasm_put(Dst, 691, (x86_dst), imm, jmp_dst_lbl); -#line 674 "fd_jit_compiler.dasc" - break; - - case 0x27: /* FD_SBPF_OP_MUL64_IMM */ - /* TODO strength reduction? */ - //| imul dst64, imm - dasm_put(Dst, 702, (x86_dst), (x86_dst), imm); -#line 679 "fd_jit_compiler.dasc" - break; - - case 0x2c: /* FD_SBPF_OP_MUL_REG */ - //| imul dst32, src32 - dasm_put(Dst, 711, (x86_dst), (x86_src)); -#line 683 "fd_jit_compiler.dasc" - break; - - case 0x2d: /* FD_SBPF_OP_JGT_REG */ - //| cmp dst64, src64 - //| ja =>jmp_dst_lbl - dasm_put(Dst, 720, (x86_src), (x86_dst), jmp_dst_lbl); -#line 688 "fd_jit_compiler.dasc" - break; - - case 0x2f: /* FD_SBPF_OP_MUL64_REG */ - //| imul dst64, src64 - dasm_put(Dst, 731, (x86_dst), (x86_src)); -#line 692 "fd_jit_compiler.dasc" - break; - - /* 0x30 - 0x3f ******************************************************/ - - case 0x34: /* FD_SBPF_OP_DIV_IMM */ - if( FD_UNLIKELY( imm==0 ) ) { - //| jmp ->vm_fault - dasm_put(Dst, 283); -#line 699 "fd_jit_compiler.dasc" - break; - } - //| xchg eax, dst32 - //| xor edx, edx - //| mov edi, imm - //| div edi - //| xchg eax, dst32 - dasm_put(Dst, 740, (x86_dst), imm, (x86_dst)); -#line 706 "fd_jit_compiler.dasc" - break; - - case 0x35: /* FD_SBPF_OP_JGE_IMM */ - //| cmp dst64, imm - //| jae =>jmp_dst_lbl - dasm_put(Dst, 757, (x86_dst), imm, jmp_dst_lbl); -#line 711 "fd_jit_compiler.dasc" - break; - - case 0x37: /* FD_SBPF_OP_DIV64_IMM */ - if( FD_UNLIKELY( imm==0 ) ) { - //| jmp ->vm_fault - dasm_put(Dst, 283); -#line 716 "fd_jit_compiler.dasc" - break; - } - //| xchg rax, dst64 - //| xor edx, edx - //| mov rdi, imm - //| div rdi - //| xchg rax, dst64 - dasm_put(Dst, 768, (x86_dst), imm, (x86_dst)); -#line 723 "fd_jit_compiler.dasc" - break; - - case 0x3c: /* FD_SBPF_OP_DIV_REG */ - //| test src32, src32 - //| jz ->vm_fault - dasm_put(Dst, 788, (x86_src), (x86_src)); -#line 728 "fd_jit_compiler.dasc" - if( x86_dst==x86_src ) { - //| mov dst32, 1 - dasm_put(Dst, 800, (x86_dst)); -#line 730 "fd_jit_compiler.dasc" - break; - } - //| xchg eax, dst32 - //| xor edx, edx - //| div src32 - //| xchg eax, dst32 - dasm_put(Dst, 809, (x86_dst), (x86_src), (x86_dst)); -#line 736 "fd_jit_compiler.dasc" - break; - - case 0x3d: /* FD_SBPF_OP_JGE_REG */ - //| cmp dst64, src64 - //| jae =>jmp_dst_lbl - dasm_put(Dst, 827, (x86_src), (x86_dst), jmp_dst_lbl); -#line 741 "fd_jit_compiler.dasc" - break; - - case 0x3f: /* FD_SBPF_OP_DIV64_REG */ - //| test src64, src64 - //| jz ->vm_fault - dasm_put(Dst, 838, (x86_src), (x86_src)); -#line 746 "fd_jit_compiler.dasc" - if( x86_dst==x86_src ) { - //| mov dst32, 1 - dasm_put(Dst, 800, (x86_dst)); -#line 748 "fd_jit_compiler.dasc" - break; - } - //| xchg rax, dst64 - //| xor edx, edx - //| div src64 - //| xchg rax, dst64 - dasm_put(Dst, 850, (x86_dst), (x86_src), (x86_dst)); -#line 754 "fd_jit_compiler.dasc" - break; - - /* 0x40 - 0x4f ******************************************************/ - - case 0x44: /* FD_SBPF_OP_OR_IMM */ - //| or dst32, imm - dasm_put(Dst, 868, (x86_dst), imm); -#line 760 "fd_jit_compiler.dasc" - break; - - case 0x45: /* FD_SBPF_OP_JSET_IMM */ - //| test dst64, imm - //| jnz =>jmp_dst_lbl - dasm_put(Dst, 875, (x86_dst), imm, jmp_dst_lbl); -#line 765 "fd_jit_compiler.dasc" - break; - - case 0x47: /* FD_SBPF_OP_OR64_IMM */ - //| or dst64, imm - dasm_put(Dst, 886, (x86_dst), imm); -#line 769 "fd_jit_compiler.dasc" - break; - - case 0x4c: /* FD_SBPF_OP_OR_REG */ - //| or dst32, src32 - dasm_put(Dst, 893, (x86_src), (x86_dst)); -#line 773 "fd_jit_compiler.dasc" - break; - - case 0x4d: /* FD_SBPF_OP_JSET_REG */ - //| test dst64, src64 - //| jnz =>jmp_dst_lbl - dasm_put(Dst, 901, (x86_src), (x86_dst), jmp_dst_lbl); -#line 778 "fd_jit_compiler.dasc" - break; - - case 0x4f: /* FD_SBPF_OP_OR64_REG */ - //| or dst64, src64 - dasm_put(Dst, 912, (x86_src), (x86_dst)); -#line 782 "fd_jit_compiler.dasc" - break; - - /* 0x50 - 0x5f ******************************************************/ - - case 0x54: /* FD_SBPF_OP_AND_IMM */ - //| and dst32, imm - dasm_put(Dst, 920, (x86_dst), imm); -#line 788 "fd_jit_compiler.dasc" - break; - - case 0x55: /* FD_SBPF_OP_JNE_IMM */ - //| cmp dst64, imm - //| jne =>jmp_dst_lbl - dasm_put(Dst, 927, (x86_dst), imm, jmp_dst_lbl); -#line 793 "fd_jit_compiler.dasc" - break; - - case 0x57: /* FD_SBPF_OP_AND64_IMM */ - //| and dst64, imm - dasm_put(Dst, 938, (x86_dst), imm); -#line 797 "fd_jit_compiler.dasc" - break; - - case 0x5c: /* FD_SBPF_OP_AND_REG */ - //| and dst32, src32 - dasm_put(Dst, 945, (x86_src), (x86_dst)); -#line 801 "fd_jit_compiler.dasc" - break; - - case 0x5d: /* FD_SBPF_OP_JNE_REG */ - //| cmp dst64, src64 - //| jne =>jmp_dst_lbl - dasm_put(Dst, 953, (x86_src), (x86_dst), jmp_dst_lbl); -#line 806 "fd_jit_compiler.dasc" - break; - - case 0x5f: /* FD_SBPF_OP_AND64_REG */ - //| and dst64, src64 - dasm_put(Dst, 964, (x86_src), (x86_dst)); -#line 810 "fd_jit_compiler.dasc" - break; - - /* 0x60 - 0x6f ******************************************************/ - - case 0x61: /* FD_SBPF_OP_LDXW */ - //| lea translate_in, [src64+offset] - //| translate_ro_4 - //| mov dst32, [translate_out] - dasm_put(Dst, 972, (x86_src), offset, (x86_dst)); -#line 818 "fd_jit_compiler.dasc" - break; - - case 0x62: /* FD_SBPF_OP_STW */ - //| lea translate_in, [dst64+offset] - //| translate_rw_4 - //| mov dword [translate_out], imm - dasm_put(Dst, 993, (x86_dst), offset, imm); -#line 824 "fd_jit_compiler.dasc" - break; - - case 0x63: /* FD_SBPF_OP_STXW */ - //| lea translate_in, [dst64+offset] - //| translate_rw_4 - //| mov [translate_out], src32 - dasm_put(Dst, 1012, (x86_dst), offset, (x86_src)); -#line 830 "fd_jit_compiler.dasc" - break; - - case 0x64: /* FD_SBPF_OP_LSH_IMM */ - //| shl dst32, imm - dasm_put(Dst, 1033, (x86_dst), imm); -#line 834 "fd_jit_compiler.dasc" - break; - - case 0x65: /* FD_SBPF_OP_JSGT_IMM */ - //| cmp dst64, imm - //| jg =>jmp_dst_lbl - dasm_put(Dst, 1040, (x86_dst), imm, jmp_dst_lbl); -#line 839 "fd_jit_compiler.dasc" - break; - - case 0x67: /* FD_SBPF_OP_LSH64_IMM */ - //| shl dst64, imm - dasm_put(Dst, 1051, (x86_dst), imm); -#line 843 "fd_jit_compiler.dasc" - break; - - case 0x69: /* FD_SBPF_OP_LDXH */ - //| lea translate_in, [src64+offset] - //| translate_ro_2 - //| xor dst32, dst32 - //| mov Rw(x86_dst), [translate_out] - dasm_put(Dst, 1058, (x86_src), offset, (x86_dst), (x86_dst), (x86_dst)); -#line 850 "fd_jit_compiler.dasc" - break; - - case 0x6a: /* FD_SBPF_OP_STH */ - //| lea translate_in, [dst64+offset] - //| translate_rw_2 - //| mov word [translate_out], imm - dasm_put(Dst, 1087, (x86_dst), offset, imm); -#line 856 "fd_jit_compiler.dasc" - break; - - case 0x6b: /* FD_SBPF_OP_STXH */ - //| lea translate_in, [dst64+offset] - //| translate_rw_2 - //| mov [translate_out], src32 - dasm_put(Dst, 1107, (x86_dst), offset, (x86_src)); -#line 862 "fd_jit_compiler.dasc" - break; - - case 0x6c: /* FD_SBPF_OP_LSH_REG */ - //| mov cl, src8 - //| shl dst32, cl - dasm_put(Dst, 1128, (x86_src), (x86_dst)); -#line 867 "fd_jit_compiler.dasc" - break; - - case 0x6d: /* FD_SBPF_OP_JSGT_REG */ - //| cmp dst64, src64 - //| jg =>jmp_dst_lbl - dasm_put(Dst, 1139, (x86_src), (x86_dst), jmp_dst_lbl); -#line 872 "fd_jit_compiler.dasc" - break; - - case 0x6f: /* FD_SBPF_OP_LSH64_REG */ - //| mov cl, src8 - //| shl dst64, cl - dasm_put(Dst, 1150, (x86_src), (x86_dst)); -#line 877 "fd_jit_compiler.dasc" - break; - - /* 0x70 - 0x7f ******************************************************/ - - case 0x71: /* FD_SBPF_OP_LDXB */ - //| lea translate_in, [src64+offset] - //| translate_ro_1 - dasm_put(Dst, 1161, (x86_src), offset); -#line 884 "fd_jit_compiler.dasc" - /* TODO is there a better way to zero upper and mov byte? */ - //| xor dst32, dst32 - //| mov Rb(x86_dst), [translate_out] - dasm_put(Dst, 1175, (x86_dst), (x86_dst), (x86_dst)); -#line 887 "fd_jit_compiler.dasc" - break; - - case 0x72: /* FD_SBPF_OP_STB */ - //| lea translate_in, [src64+offset] - //| translate_rw_1 - //| mov byte [translate_out], imm - dasm_put(Dst, 1188, (x86_src), offset, imm); -#line 893 "fd_jit_compiler.dasc" - break; - - case 0x73: /* FD_SBPF_OP_STXB */ - //| lea translate_in, [dst64+offset] - //| translate_rw_1 - //| mov byte [translate_out], Rb(x86_src) - dasm_put(Dst, 1205, (x86_dst), offset, (x86_src)); -#line 899 "fd_jit_compiler.dasc" - break; - - case 0x74: /* FD_SBPF_OP_RSH_IMM */ - //| shr dst32, imm - dasm_put(Dst, 1224, (x86_dst), imm); -#line 903 "fd_jit_compiler.dasc" - break; - - case 0x75: /* FD_SBPF_OP_JSGE_IMM */ - //| cmp dst64, imm - //| jge =>jmp_dst_lbl - dasm_put(Dst, 1231, (x86_dst), imm, jmp_dst_lbl); -#line 908 "fd_jit_compiler.dasc" - break; - - case 0x77: /* FD_SBPF_OP_RSH64_IMM */ - //| shr dst64, imm - dasm_put(Dst, 1242, (x86_dst), imm); -#line 912 "fd_jit_compiler.dasc" - break; - - case 0x79: /* FD_SBPF_OP_LDXQ */ - //| lea translate_in, [src64+offset] - //| translate_ro_8 - //| mov dst64, [translate_out] - dasm_put(Dst, 1249, (x86_src), offset, (x86_dst)); -#line 918 "fd_jit_compiler.dasc" - break; - - case 0x7a: /* FD_SBPF_OP_STQ */ - //| lea translate_in, [dst64+offset] - //| translate_rw_8 - //| mov dword [translate_out], imm - //| mov dword [translate_out+4], 0 - dasm_put(Dst, 1270, (x86_dst), offset, imm); -#line 925 "fd_jit_compiler.dasc" - break; - - case 0x7b: /* FD_SBPF_OP_STXQ */ - //| lea translate_in, [dst64+offset] - //| translate_rw_8 - //| mov [translate_out], src64 - dasm_put(Dst, 1296, (x86_dst), offset, (x86_src)); -#line 931 "fd_jit_compiler.dasc" - break; - - case 0x7c: /* FD_SBPF_OP_RSH_REG */ - //| mov cl, src8 - //| shr dst32, cl - dasm_put(Dst, 1317, (x86_src), (x86_dst)); -#line 936 "fd_jit_compiler.dasc" - break; - - case 0x7d: /* FD_SBPF_OP_JSGE_REG */ - //| cmp dst64, src64 - //| jge =>jmp_dst_lbl - dasm_put(Dst, 1328, (x86_src), (x86_dst), jmp_dst_lbl); -#line 941 "fd_jit_compiler.dasc" - break; - - case 0x7f: /* FD_SBPF_OP_RSH64_REG */ - //| mov cl, src8 - //| shr dst64, cl - dasm_put(Dst, 1339, (x86_src), (x86_dst)); -#line 946 "fd_jit_compiler.dasc" - break; - - /* 0x80-0x8f ********************************************************/ - - case 0x84: /* FD_SBPF_OP_NEG */ - //| neg dst32 - dasm_put(Dst, 1350, (x86_dst)); -#line 952 "fd_jit_compiler.dasc" - break; - - case 0x85: { /* FD_SBPF_OP_CALL_IMM */ - fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, imm, NULL ); - if( !syscall ) { - ulong target_pc = (ulong)fd_pchash_inverse( imm ); - //| call ->call_stack_push - //| call =>target_pc - dasm_put(Dst, 1357, target_pc); -#line 960 "fd_jit_compiler.dasc" - } else { - /* Optimize for code footprint: Generate an offset into the - syscall table (32-bit) instead of the syscall address (64-bit) */ - //| mov rdi, (uint)( (ulong)syscall - (ulong)syscalls ); - //| call ->emulate_syscall - dasm_put(Dst, 1363, (uint)( (ulong)syscall - (ulong)syscalls )); -#line 965 "fd_jit_compiler.dasc" - } - break; - } - - case 0x87: /* FD_SBPF_OP_NEG64 */ - //| neg dst64 - dasm_put(Dst, 1371, (x86_dst)); -#line 971 "fd_jit_compiler.dasc" - break; - - case 0x8d: /* FD_SBPF_OP_CALL_REG */ - FD_LOG_WARNING(( "TODO: CALLX" )); - break; - - /* 0x90 - 0x9f ******************************************************/ - - case 0x94: /* FD_SBPF_OP_MOD_IMM */ - if( FD_UNLIKELY( imm==0 ) ) { - //| jmp ->vm_fault - dasm_put(Dst, 283); -#line 982 "fd_jit_compiler.dasc" - break; - } - //| xchg eax, dst32 - //| xor edx, edx - //| mov edi, imm - //| div edi - //| xchg edx, dst32 - dasm_put(Dst, 1378, (x86_dst), imm, (x86_dst)); -#line 989 "fd_jit_compiler.dasc" - break; - - case 0x95: /* FD_SBPF_OP_EXIT */ - //| call ->call_stack_pop - //| ret - dasm_put(Dst, 1396); -#line 994 "fd_jit_compiler.dasc" - break; - - case 0x97: /* FD_SBPF_OP_MOD64_IMM */ - if( FD_UNLIKELY( imm==0 ) ) { - //| jmp ->vm_fault - dasm_put(Dst, 283); -#line 999 "fd_jit_compiler.dasc" - break; - } - //| xchg rax, dst64 - //| xor edx, edx - //| mov rdi, imm - //| div rdi - //| xchg rax, dst64 - dasm_put(Dst, 768, (x86_dst), imm, (x86_dst)); -#line 1006 "fd_jit_compiler.dasc" - break; - - case 0x9c: /* FD_SBPF_OP_MOD_REG */ - //| test src32, src32 - //| jz ->vm_fault - dasm_put(Dst, 788, (x86_src), (x86_src)); -#line 1011 "fd_jit_compiler.dasc" - if( x86_dst==x86_src ) { - //| mov dst32, 0 - dasm_put(Dst, 1401, (x86_dst)); -#line 1013 "fd_jit_compiler.dasc" - break; - } - //| xchg eax, dst32 - //| xor edx, edx - //| div src32 - //| xchg edx, dst32 - dasm_put(Dst, 1410, (x86_dst), (x86_src), (x86_dst)); -#line 1019 "fd_jit_compiler.dasc" - break; - - case 0x9f: /* FD_SBPF_OP_MOD64_REG */ - //| test src64, src64 - //| jz ->vm_fault - dasm_put(Dst, 838, (x86_src), (x86_src)); -#line 1024 "fd_jit_compiler.dasc" - if( x86_dst==x86_src ) { - //| mov dst32, 0 - dasm_put(Dst, 1401, (x86_dst)); -#line 1026 "fd_jit_compiler.dasc" - break; - } - //| xchg rax, dst64 - //| xor edx, edx - //| div src64 - //| xchg rdx, dst64 - dasm_put(Dst, 1429, (x86_dst), (x86_src), (x86_dst)); -#line 1032 "fd_jit_compiler.dasc" - break; - - /* 0xa0 - 0xaf ******************************************************/ - - case 0xa4: /* FD_SBPF_OP_XOR_IMM */ - //| xor dst32, imm - dasm_put(Dst, 1448, (x86_dst), imm); -#line 1038 "fd_jit_compiler.dasc" - break; - - case 0xa5: /* FD_SBPF_OP_JLT_IMM */ - //| cmp dst64, imm - //| jb =>jmp_dst_lbl - dasm_put(Dst, 1456, (x86_dst), imm, jmp_dst_lbl); -#line 1043 "fd_jit_compiler.dasc" - break; - - case 0xa7: /* FD_SBPF_OP_XOR64_IMM */ - // TODO sign extension - //| xor dst64, imm - dasm_put(Dst, 1467, (x86_dst), imm); -#line 1048 "fd_jit_compiler.dasc" - break; - - case 0xac: /* FD_SBPF_OP_XOR_REG */ - //| xor dst32, src32 - dasm_put(Dst, 640, (x86_src), (x86_dst)); -#line 1052 "fd_jit_compiler.dasc" - break; - - case 0xad: /* FD_SBPF_OP_JLT_REG */ - //| cmp dst64, src64 - //| jb =>jmp_dst_lbl - dasm_put(Dst, 1475, (x86_src), (x86_dst), jmp_dst_lbl); -#line 1057 "fd_jit_compiler.dasc" - break; - - case 0xaf: /* FD_SBPF_OP_XOR64_REG */ - //| xor dst64, src64 - dasm_put(Dst, 1486, (x86_src), (x86_dst)); -#line 1061 "fd_jit_compiler.dasc" - break; - - /* 0xb0 - 0xbf ******************************************************/ - - case 0xb4: /* FD_SBPF_OP_MOV_IMM */ - //| mov dst32, imm - dasm_put(Dst, 1494, (x86_dst), imm); -#line 1067 "fd_jit_compiler.dasc" - break; - - case 0xb5: /* FD_SBPF_OP_JLE_IMM */ - //| cmp dst64, imm - //| jbe =>jmp_dst_lbl - dasm_put(Dst, 1500, (x86_dst), imm, jmp_dst_lbl); -#line 1072 "fd_jit_compiler.dasc" - break; - - case 0xb7: /* FD_SBPF_OP_MOV64_IMM */ - if( imm==0 ) { - //| xor dst32, dst32 - dasm_put(Dst, 640, (x86_dst), (x86_dst)); -#line 1077 "fd_jit_compiler.dasc" - } else { - //| mov dst64, imm - dasm_put(Dst, 648, (x86_dst), imm); -#line 1079 "fd_jit_compiler.dasc" - } - break; - - case 0xbc: /* FD_SBPF_OP_MOV_REG */ - //| mov dst32, src32 - dasm_put(Dst, 1511, (x86_src), (x86_dst)); -#line 1084 "fd_jit_compiler.dasc" - break; - - case 0xbd: /* FD_SBPF_OP_JLE_REG */ - //| cmp dst64, src64 - //| jbe =>jmp_dst_lbl - dasm_put(Dst, 1519, (x86_src), (x86_dst), jmp_dst_lbl); -#line 1089 "fd_jit_compiler.dasc" - break; - - case 0xbf: /* FD_SBPF_OP_MOV64_REG */ - //| mov dst64, src64 - dasm_put(Dst, 1530, (x86_src), (x86_dst)); -#line 1093 "fd_jit_compiler.dasc" - break; - - /* 0xc0 - 0xcf ******************************************************/ - - case 0xc4: /* FD_SBPF_OP_ARSH_IMM */ - //| sar dst32, imm - dasm_put(Dst, 1538, (x86_dst), imm); -#line 1099 "fd_jit_compiler.dasc" - break; - - case 0xc5: /* FD_SBPF_OP_JSLT_IMM */ - //| cmp dst64, imm - //| jl =>jmp_dst_lbl - dasm_put(Dst, 1546, (x86_dst), imm, jmp_dst_lbl); -#line 1104 "fd_jit_compiler.dasc" - break; - - case 0xc7: /* FD_SBPF_OP_ARSH64_IMM */ - //| sar dst64, imm - dasm_put(Dst, 1557, (x86_dst), imm); -#line 1108 "fd_jit_compiler.dasc" - break; - - case 0xcc: /* FD_SBPF_OP_ARSH_REG */ - //| mov cl, src8 - //| sar dst32, cl - dasm_put(Dst, 1565, (x86_src), (x86_dst)); -#line 1113 "fd_jit_compiler.dasc" - break; - - case 0xcd: /* FD_SBPF_OP_JSLT_REG */ - //| cmp dst64, src64 - //| jl =>jmp_dst_lbl - dasm_put(Dst, 1577, (x86_src), (x86_dst), jmp_dst_lbl); -#line 1118 "fd_jit_compiler.dasc" - break; - - case 0xcf: /* FD_SBPF_OP_ARSH64_REG */ - //| mov cl, src8 - //| sar dst64, cl - dasm_put(Dst, 1588, (x86_src), (x86_dst)); -#line 1123 "fd_jit_compiler.dasc" - break; - - /* 0xd0 - 0xdf ******************************************************/ - - case 0xd4: /* FD_SBPF_OP_END_LE */ - /* nop */ - break; - - case 0xd5: /* FD_SBPF_OP_JSLE_IMM */ - //| cmp dst64, imm - //| jle =>jmp_dst_lbl - dasm_put(Dst, 1600, (x86_dst), imm, jmp_dst_lbl); -#line 1134 "fd_jit_compiler.dasc" - break; - - case 0xdc: /* FD_SBPF_OP_END_BE */ - switch( imm ) { - case 16U: - //| movzx dst32, Rw(x86_dst) - //| ror Rw(x86_dst), 8 - dasm_put(Dst, 1611, (x86_dst), (x86_dst), (x86_dst)); -#line 1141 "fd_jit_compiler.dasc" - break; - case 32U: - //| bswap dst32 - dasm_put(Dst, 1627, (x86_dst)); -#line 1144 "fd_jit_compiler.dasc" - break; - case 64U: - //| bswap dst64 - dasm_put(Dst, 1633, (x86_dst)); -#line 1147 "fd_jit_compiler.dasc" - break; - default: - break; - // TODO sigill - } - break; - - case 0xdd: /* FD_SBPF_OP_JSLE_REG */ - //| cmp dst64, src64 - //| jle =>jmp_dst_lbl - dasm_put(Dst, 1639, (x86_src), (x86_dst), jmp_dst_lbl); -#line 1157 "fd_jit_compiler.dasc" - break; - - default: - FD_LOG_WARNING(( "Unsupported opcode %lx", opcode )); - cur = text_end; - break; - - } - - } - - //|->overrun: - //| jmp ->vm_fault - dasm_put(Dst, 1650); -#line 1170 "fd_jit_compiler.dasc" - -} - -fd_jit_entrypoint_t -fd_jit_get_entrypoint( void ) { - return (fd_jit_entrypoint_t)(ulong)fd_jit_labels[ fd_jit_lbl_entrypoint ]; -} diff --git a/src/flamenco/vm/jit/fd_jit_compiler.dasc b/src/flamenco/vm/jit/fd_jit_compiler.dasc deleted file mode 100644 index 49a783f3be..0000000000 --- a/src/flamenco/vm/jit/fd_jit_compiler.dasc +++ /dev/null @@ -1,1176 +0,0 @@ -/* WARNING: WORK IN PROGRESS! - This is an experimental version of the Firedancer JIT compiler. - It is disabled in production. There are known security issues in - this code. It is not covered by the Firedancer bug bounty policy. - - ### fd_vm_interp compatibility - - fd_jit aims for exact compatibility with fd_vm_interp unless - otherwise noted. - - ### Optimizations - - The fd_jit transpiler does not implement any optimizations. - - ### Error Handling - - VM faults (e.g. segfault, divide by zero, CU overrun, invalid call - destination, program EOF, syscall error) are detected reliably and - deterministically. - - However, after a fault happens, any VM state becomes unreliable - (non-deterministic or undefined). Unreliable fault state include CU - left, current PC, and the error code. - - A VM fault usually triggers ->longjmp, transitioning back into the - fd_jit_execute call frame. */ - -#include "../../../util/fd_util_base.h" - -/* Include dynasm headers. These fail to compile when some strict - checks are enabled. */ - -void fd_dasm_grow_check( void * ptr, ulong min_sz ); -#define DASM_M_GROW(ctx, t, p, sz, need) (fd_dasm_grow_check( (p), (need) )) -#define DASM_M_FREE(ctx, p, sz) do{}while(0) - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wsign-conversion" -#pragma GCC diagnostic ignored "-Wimplicit-fallthrough" -#include "dasm_proto.h" -#include "dasm_x86.h" -#pragma GCC diagnostic pop - -#include "fd_jit_private.h" -#include "../fd_vm_private.h" - -| .arch x64 -| .actionlist actions -| .globals fd_jit_lbl_ -| .section code - -fd_jit_scratch_layout_t * -fd_jit_scratch_layout( fd_jit_scratch_layout_t * layout, - ulong bpf_sz ) { - - if( FD_UNLIKELY( bpf_sz > (1UL<<24) ) ) return NULL; - ulong text_cnt = bpf_sz / 8UL; - - /* These "magic" values are taken from dasm_x86.h */ - - ulong dasm_sz = DASM_PSZ( DASM_MAXSECTION ); /* dasm_x86.h(89) */ - ulong lglabels_sz = (10+fd_jit_lbl__MAX)*sizeof(int); /* dasm_x86.h(119) */ - ulong pclabels_sz = text_cnt*sizeof(int); /* dasm_x86.h(127) */ - ulong code_sz = fd_jit_est_code_sz( bpf_sz ); - - memset( layout, 0, sizeof(fd_jit_scratch_layout_t) ); - FD_SCRATCH_ALLOC_INIT( l, 0 ); - layout->dasm_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, dasm_sz ); - layout->dasm_sz = dasm_sz; - layout->lglabels_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, lglabels_sz ); - layout->lglabels_sz = lglabels_sz; - layout->pclabels_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, pclabels_sz ); - layout->pclabels_sz = pclabels_sz; - layout->code_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, code_sz ); - layout->code_sz = code_sz; - layout->sz = (ulong)FD_SCRATCH_ALLOC_FINI( l, 16UL ); - return layout; -} - -dasm_State * -fd_jit_prepare( void * scratch, - fd_jit_scratch_layout_t const * layout ) { - - /* Custom dasm_init */ - dasm_State * d = (void *)( (ulong)scratch + layout->dasm_off ); - fd_memset( d, 0, layout->dasm_sz ); - d->psize = layout->dasm_sz; - d->maxsection = DASM_MAXSECTION; - - /* Custom dasm_setupglobal */ - d->globals = fd_jit_labels; - d->lglabels = (void *)( (ulong)scratch + layout->lglabels_off ); - d->lgsize = layout->lglabels_sz; - - /* Custom dasm_growpc */ - d->pcsize = layout->pclabels_sz; - d->pclabels = (void *)( (ulong)scratch + layout->pclabels_off ); - - /* Setup encoder. Zeros lglabels and pclabels. */ - dasm_setup( &d, actions ); - - /* Preallocate space for .code section - See dasm_x86.h(172) */ - dasm_Section * code = d->sections + 0; - code->buf = (void *)( (ulong)scratch + layout->code_off ); - code->bsize = layout->code_sz; - code->pos = 0; - code->rbuf = code->buf; - code->epos = (int)( ((ulong)code->bsize / sizeof(int)) - DASM_MAXSECPOS ); - code->ofs = 0; - - return d; -} - -/* Compile time thread locals */ - -FD_TL void * fd_jit_labels[ FD_JIT_LABEL_CNT ]; -FD_STATIC_ASSERT( sizeof(fd_jit_labels)==fd_jit_lbl__MAX*8, label_cnt ); - -/* Mapping between sBPF registers and x86_64 registers ***************** - - This mapping is valid just before a translated sBPF instruction is - about to be executed. (At the `=>next_label` token in the code gen - loop) - - BPF | r0 | r1 | r2 | r3 | r4 | r5 | r6 | r7 | r8 | r9 | r10 - X86 | rsi | r11 | r12 | r13 | r14 | r15 | rbx | rcx | r8 | r9 | r10 - - x86_64 GPRs rax, rdi, rdx, rbp do not map to sBPF registers. Those can - be used as scratch registers for complex opcodes. - - Note that this mapping cannot be trivially changed. Certain x86 - instructions (like div) have hardcoded register accesses which the - JIT code works around. - - dynasm macros bpf_r{...} resolve to 64-bit register names. - - reg_bpf2x86 is indexed by sBPF register numbers and resolves to the - x86_64 dynasm register index. */ - -static uchar const reg_bpf2x86[11] = { - [ 0] = FD_DASM_RSI, - | .define bpf_r0, rsi - [ 1] = FD_DASM_R11, - | .define bpf_r1, r11 - [ 2] = FD_DASM_R12, - | .define bpf_r2, r12 - [ 3] = FD_DASM_R13, - | .define bpf_r3, r13 - [ 4] = FD_DASM_R14, - | .define bpf_r4, r14 - [ 5] = FD_DASM_R15, - | .define bpf_r5, r15 - [ 6] = FD_DASM_RBX, - | .define bpf_r6, rbx - [ 7] = FD_DASM_RCX, - | .define bpf_r7, rcx - [ 8] = FD_DASM_R8, - | .define bpf_r8, r8 - [ 9] = FD_DASM_R9, - | .define bpf_r9, r9 - [10] = FD_DASM_R10 - | .define bpf_r10, r10 -}; - - -/* JIT compiler *******************************************************/ - -void -fd_jit_compile( struct dasm_State ** Dst, - fd_sbpf_program_t const * prog, - fd_sbpf_syscalls_t const * syscalls ) { - - int const abiv2 = 0; /* For now, only support ABIv1 */ - - | .code - - /* Derive offsets of thread locals in FS "segment" */ - -# if defined(__FSGSBASE__) - ulong fs_base = __builtin_ia32_rdfsbase64(); -# else - ulong fs_base; __asm__( "mov %%fs:0, %0" : "=r"(fs_base) ); -# endif -# define FS_RELATIVE(ptr) ((uint)( (ulong)(ptr) - fs_base )) - uint fd_jit_vm_tpoff = FS_RELATIVE( &fd_jit_vm ); - uint fd_jit_syscalls_tpoff = FS_RELATIVE( &fd_jit_syscalls ); - uint fd_jit_segment_cnt_tpoff = FS_RELATIVE( &fd_jit_segment_cnt ); - uint fd_jit_mem_sz_tpoff = FS_RELATIVE( fd_jit_mem_sz ); - uint fd_jit_mem_haddr_tpoff = FS_RELATIVE( fd_jit_mem_haddr ); - uint fd_jit_segfault_vaddr_tpoff = FS_RELATIVE( &fd_jit_segfault_vaddr ); - uint fd_jit_segfault_rip_tpoff = FS_RELATIVE( &fd_jit_segfault_rip ); -# undef FD_RELATIVE - - |->save_regs: - | fs - | mov rax, [fd_jit_vm_tpoff] - | mov [rax + offsetof(fd_vm_t, reg[ 0])], bpf_r0 - | mov [rax + offsetof(fd_vm_t, reg[ 1])], bpf_r1 - | mov [rax + offsetof(fd_vm_t, reg[ 2])], bpf_r2 - | mov [rax + offsetof(fd_vm_t, reg[ 3])], bpf_r3 - | mov [rax + offsetof(fd_vm_t, reg[ 4])], bpf_r4 - | mov [rax + offsetof(fd_vm_t, reg[ 5])], bpf_r5 - | mov [rax + offsetof(fd_vm_t, reg[ 6])], bpf_r6 - | mov [rax + offsetof(fd_vm_t, reg[ 7])], bpf_r7 - | mov [rax + offsetof(fd_vm_t, reg[ 8])], bpf_r8 - | mov [rax + offsetof(fd_vm_t, reg[ 9])], bpf_r9 - | mov [rax + offsetof(fd_vm_t, reg[10])], bpf_r10 - | ret - - |->restore_regs: - | fs - | mov rax, [fd_jit_vm_tpoff] - | mov bpf_r0, [rax + offsetof(fd_vm_t, reg[ 0])] - | mov bpf_r1, [rax + offsetof(fd_vm_t, reg[ 1])] - | mov bpf_r2, [rax + offsetof(fd_vm_t, reg[ 2])] - | mov bpf_r3, [rax + offsetof(fd_vm_t, reg[ 3])] - | mov bpf_r4, [rax + offsetof(fd_vm_t, reg[ 4])] - | mov bpf_r5, [rax + offsetof(fd_vm_t, reg[ 5])] - | mov bpf_r6, [rax + offsetof(fd_vm_t, reg[ 6])] - | mov bpf_r7, [rax + offsetof(fd_vm_t, reg[ 7])] - | mov bpf_r8, [rax + offsetof(fd_vm_t, reg[ 8])] - | mov bpf_r9, [rax + offsetof(fd_vm_t, reg[ 9])] - | mov bpf_r10, [rax + offsetof(fd_vm_t, reg[10])] - | ret - - /* Helper macros for BPF-to-x86 function calls */ - - |.macro enter_x86_frame - | call ->save_regs - | mov rbp, rsp - | and rsp, -16 - |.endmacro - - |.macro leave_x86_frame - | mov rsp, rbp - | call ->restore_regs - |.endmacro - - /* Address translation macros - - The translate_{rw,ro}_{1,2,4,8} macros perform address translation - and access permission checks for {read-write,read-only} accesses of - {1,2,4,8} bytes. The compiler may inline this macro for each - translated sBPF instruction, so these should be optimized for small - size. - - Prior to the macro, rdi holds an address in the virtual address - space (untrusted in [0,2^64)). If translation and permission - checks succeed, rdx holds the translated address in the host - address space. On failure jumps to sigsegv. Reasons for failure - include access to out-of-bounds memory, unaligned address, access - permission error. */ - - | .define translate_in, rdi - | .define translate_out, rdx - /* FIXME make output argument rdi instead of rdx */ - - |->fd_jit_vm_translate_ro: - | mov edx, edi // segment offset - | shr rdi, 32 // segment index - | shl edi, 1 - | jmp ->fd_jit_vm_translate - - |->fd_jit_vm_translate_rw: - | mov edx, edi // segment offset - | shr rdi, 32 // segment index - | lea edi, [edi*2+1] - /* fallthrough */ - - |->fd_jit_vm_translate: - if( abiv2 ) { - | jmp ->fd_jit_vm_translate_abiv1 - } else { - | jmp ->fd_jit_vm_translate_abiv2 - } - - /* ABIv1 virtual memory overview - - - There are 6 pages indexed in [0,6) - - The number of readable bytes in a page is variable in [0,2^31) - - The number of writable bytes in a page is equal to the number of - readable bytes or zero - - The first byte of a page in virtual memory is at (index<<32) - - Page 0 is always empty (all bytes are unaddressable) - - Page 2 is 'striped': Virtual addresses with bit 12 set are not - addressable */ - - |->fd_jit_vm_translate_abiv1: - | // rdi := virtual address - | // ebp := size of the access minus 1 - | - | // edx := segment offset - | mov edx, edi - | - | // edi := segment index - | shr rdi, 32 - | - | // segment index in bounds? - | fs - | cmp edi, [fd_jit_segment_cnt_tpoff] - | jae ->translate_fail - | - | // no multi segment overlap? - | add ebp, edx - | jc ->translate_fail - | - | // segment offset in bounds? - | fs - | cmp edx, [rdi*4 + fd_jit_mem_sz_tpoff] - | jae ->translate_fail - | - | // stack gap? - | xor ebp, ebp - | cmp edi, 2 - | mov eax, 0x1000 - | cmove ebp, eax - | and edx, ebp - | jae ->translate_fail - | - | // rdx := host address - | fs - | add rdx, [rdi*8 + fd_jit_mem_haddr_tpoff] - | ret - - /* ABIv2 virtual memory overview: - - - Virtual memory is described by a page table - - There can be up to 2^16 pages - - Each page is 4 GiB (2^32 byte) aligned in virtual memory - - The number of readable bytes in a page is variable in [0,2^31) - - The number of writable bytes in a page is less or equal to the - number of readable bytes */ - - |->fd_jit_vm_translate_abiv2: - | // rdi := virtual address - | // ebp := size of the access minus 1 - | - | // segment index in bounds? - | fs - | cmp edi, [fd_jit_segment_cnt_tpoff] - | jae ->translate_fail - | - | //// aligned access? - | // FIXME do we need an alignment check? - | //mov eax, edx - | //and eax, ebp - | //test eax, eax - | //jnz ->translate_fail - | - | // no multi segment overlap? - | add ebp, edx - | jc ->translate_fail - | - | // segment offset in bounds? - | fs - | cmp edx, [rdi*4 + fd_jit_mem_sz_tpoff] - | jae ->translate_fail - | - | // rdx := host address - | fs - | add rdx, [rdi*8 + fd_jit_mem_haddr_tpoff] - | ret - - |->translate_fail: - | shl rdi, 32 - | or rdi, rdx - | fs - | mov [fd_jit_segfault_vaddr_tpoff], rdi - | mov rdi, [rsp] - | fs - | mov [fd_jit_segfault_rip_tpoff], rdi - | jmp ->vm_fault - - |.macro translate_rw_1 - | xor ebp, ebp - | call ->fd_jit_vm_translate_rw - |.endmacro - - |.macro translate_rw_2 - | mov ebp, 1 - | call ->fd_jit_vm_translate_rw - |.endmacro - - |.macro translate_rw_4 - | mov ebp, 3 - | call ->fd_jit_vm_translate_rw - |.endmacro - - |.macro translate_rw_8 - | mov ebp, 7 - | call ->fd_jit_vm_translate_rw - |.endmacro - - |.macro translate_ro_1 - | xor ebp, ebp - | call ->fd_jit_vm_translate_ro - |.endmacro - - |.macro translate_ro_2 - | mov ebp, 1 - | call ->fd_jit_vm_translate_ro - |.endmacro - - |.macro translate_ro_4 - | mov ebp, 3 - | call ->fd_jit_vm_translate_ro - |.endmacro - - |.macro translate_ro_8 - | mov ebp, 7 - | call ->fd_jit_vm_translate_ro - |.endmacro - - /* Generate setjmp/longjmp subroutines. These can be called from any - execution state with a valid stack. The JIT uses them to restore a - sane SystemV-ABI context when exiting JIT code. - - These are based on musl libc's setjmp/longjmp implementation. - Copyright 2011-2012 Nicholas J. Kain, licensed under standard MIT license - - setjmp takes no arguments. longjmp takes a 64-bit value in rdi. - When setjmp returns from setjmp, sets rax=0 and rdx=0. When setjmp - returns from longjmp, sets rax to the rdi argument of longjmp, and - sets rdx=1. setjmp preserves rdi. */ - - |->setjmp: - | mov64 r11, (ulong)fd_jit_jmp_buf - | mov [r11+ 0], rbx - | mov [r11+ 8], rbp - | mov [r11+16], r12 - | mov [r11+24], r13 - | mov [r11+32], r14 - | mov [r11+40], r15 - | // save callee's stack pointer - | // derived by removing our 8 byte stack frame (only return address) - | lea rdx, [rsp+8] - | mov [r11+48], rdx - | // save return address - | mov rdx, [rsp] - | mov [r11+56], rdx - | // normal return - | xor eax, eax - | xor edx, edx - | ret - - |->longjmp: - | mov rax, rdi // move first argument to first output register - | mov edx, 1 // set second output register to 1 - | mov64 rdi, (ulong)fd_jit_jmp_buf - | // restore execution state to callee of setjmp - | mov rbx, [rdi+ 0] - | mov rbp, [rdi+ 8] - | mov r12, [rdi+16] - | mov r13, [rdi+24] - | mov r14, [rdi+32] - | mov r15, [rdi+40] - | mov rsp, [rdi+48] - | push qword [rdi+56] - | ret // retpoline - - /* The emulate_syscall function switches from a JIT to an interpreter (C) - execution context and invokes a syscall handler. Register edi is - assumed to hold the byte offset into the vm->syscalls table of the - fd_sbpf_syscalls_t entry to invoke. - On syscall return, switches back to the JIT execution context and - resumes execution after the syscall instruction. */ - - |->emulate_syscall: - | call ->save_regs - | enter_x86_frame - | sub rsp, 16 - | fs - | mov r11, [fd_jit_vm_tpoff] - | fs - | mov r10, [rdi + fd_jit_syscalls_tpoff + offsetof(fd_sbpf_syscalls_t, func)] - | mov rdi, r11 - | // load BPF r1 through r5 into function arguments - | // FIXME could avoid spill to memory by shuffling registers - | mov rsi, [rax + offsetof(fd_vm_t, reg[1])] - | mov rdx, [rax + offsetof(fd_vm_t, reg[2])] - | mov rcx, [rax + offsetof(fd_vm_t, reg[3])] - | mov r8, [rax + offsetof(fd_vm_t, reg[4])] - | mov r9, [rax + offsetof(fd_vm_t, reg[5])] - | lea r11, [rax + offsetof(fd_vm_t, reg[0])] - | push r11 - | call r10 - | test edi, edi - | jnz ->vm_fault - | leave_x86_frame - | ret - - /* The call_stack_push function pushes the current program counter and - eBPF registers r6, r7, r8, r9 to the shadow stack. The frame register - (r10) grows upwards. FIXME implement shadow stack overflow. */ - -# define REG(n) (offsetof(fd_vm_t, shadow[0].r##n)) - - |->call_stack_push: - | fs - | mov rdi, [fd_jit_vm_tpoff] - | // vm->frame_cnt++ - | inc qword [rdi + offsetof(fd_vm_t, frame_cnt)] - | // save registers - | pop rdi - | push bpf_r6 - | push bpf_r7 - | push bpf_r8 - | push bpf_r9 - | add bpf_r10, 0x2000 - | jmp rdi - - /* The call_stack_pop function undoes the effects of call_stack_push. */ - - |->call_stack_pop: - | fs - | mov rdi, [fd_jit_vm_tpoff] - | // vm->frame_cnt-- - | dec qword [rdi + offsetof(fd_vm_t, frame_cnt)] - | // restore registers - | pop rdi - | pop bpf_r9 - | pop bpf_r8 - | pop bpf_r7 - | pop bpf_r6 - | sub bpf_r10, 0x2000 - | jmp rdi - -# undef REG - /* Exception handlers */ - - |->vm_fault: - | mov edi, FD_VM_ERR_SIGABORT - | jmp ->longjmp - - /* JIT entrypoint from C code */ - - |->entrypoint: - - /* Create setjmp anchor used to return from JIT */ - - | call ->setjmp // preserves rdi - | test edx, edx - | jnz >1 - - /* Enter JIT execution context */ - - | call ->restore_regs - | sub rsp, 0x20 // balance call_stack_push - | call rdi - | mov rdi, bpf_r0 - | call ->longjmp - |1: - | ret - - /* Start translating user code */ - - ulong * const text_start = prog->text; - ulong * text_end = prog->text + prog->text_cnt; - - for( ulong * cur=text_start; cur=text_start && jmpnext_label: - - /* Translate instruction */ - - switch( opcode ) { - - /* 0x00 - 0x0f ******************************************************/ - - case 0x04: /* FD_SBPF_OP_ADD_IMM */ - | add dst32, imm - break; - - case 0x05: /* FD_SBPF_OP_JA */ - | jmp =>jmp_dst_lbl - break; - - case 0x07: /* FD_SBPF_OP_ADD64_IMM */ - | add dst64, imm - break; - - case 0x0c: /* FD_SBPF_OP_ADD_REG */ - | add dst32, src32 - break; - - case 0x0f: /* FD_SBPF_OP_ADD64_REG */ - | add dst64, src64 - break; - - /* 0x10 - 0x1f ******************************************************/ - - case 0x14: /* FD_SBPF_OP_SUB_IMM */ - | sub dst32, imm - break; - - case 0x15: /* FD_SBPF_OP_JEQ_IMM */ - | cmp dst64, imm - /* pre branch check here ... branchless cu update? */ - | je =>jmp_dst_lbl - break; - - case 0x17: /* FD_SBPF_OP_SUB64_IMM */ - | sub dst64, imm - break; - - case 0x18: /* FD_SBPF_OP_LDQ */ - cur++; { - ulong imm64 = (ulong)imm | ( (ulong)fd_vm_instr_imm( *cur ) << 32 ); - if( imm64==0 ) { - | xor dst32, dst32 - } else { - | mov dst64, imm64 - } - break; - } - - case 0x1c: /* FD_SBPF_OP_SUB_REG */ - | sub dst32, src32 - break; - - case 0x1d: /* FD_SBPF_OP_JEQ_REG */ - | cmp dst64, src64 - | je =>jmp_dst_lbl - break; - - case 0x1f: /* FD_SBPF_OP_SUB64_REG */ - | sub dst64, src64 - break; - - /* 0x20 - 0x2f ******************************************************/ - - case 0x24: /* FD_SBPF_OP_MUL_IMM */ - /* TODO strength reduction? */ - | imul dst32, imm - break; - - case 0x25: /* FD_SBPF_OP_JGT_IMM */ - | cmp dst64, imm - | ja =>jmp_dst_lbl - break; - - case 0x27: /* FD_SBPF_OP_MUL64_IMM */ - /* TODO strength reduction? */ - | imul dst64, imm - break; - - case 0x2c: /* FD_SBPF_OP_MUL_REG */ - | imul dst32, src32 - break; - - case 0x2d: /* FD_SBPF_OP_JGT_REG */ - | cmp dst64, src64 - | ja =>jmp_dst_lbl - break; - - case 0x2f: /* FD_SBPF_OP_MUL64_REG */ - | imul dst64, src64 - break; - - /* 0x30 - 0x3f ******************************************************/ - - case 0x34: /* FD_SBPF_OP_DIV_IMM */ - if( FD_UNLIKELY( imm==0 ) ) { - | jmp ->vm_fault - break; - } - | xchg eax, dst32 - | xor edx, edx - | mov edi, imm - | div edi - | xchg eax, dst32 - break; - - case 0x35: /* FD_SBPF_OP_JGE_IMM */ - | cmp dst64, imm - | jae =>jmp_dst_lbl - break; - - case 0x37: /* FD_SBPF_OP_DIV64_IMM */ - if( FD_UNLIKELY( imm==0 ) ) { - | jmp ->vm_fault - break; - } - | xchg rax, dst64 - | xor edx, edx - | mov rdi, imm - | div rdi - | xchg rax, dst64 - break; - - case 0x3c: /* FD_SBPF_OP_DIV_REG */ - | test src32, src32 - | jz ->vm_fault - if( x86_dst==x86_src ) { - | mov dst32, 1 - break; - } - | xchg eax, dst32 - | xor edx, edx - | div src32 - | xchg eax, dst32 - break; - - case 0x3d: /* FD_SBPF_OP_JGE_REG */ - | cmp dst64, src64 - | jae =>jmp_dst_lbl - break; - - case 0x3f: /* FD_SBPF_OP_DIV64_REG */ - | test src64, src64 - | jz ->vm_fault - if( x86_dst==x86_src ) { - | mov dst32, 1 - break; - } - | xchg rax, dst64 - | xor edx, edx - | div src64 - | xchg rax, dst64 - break; - - /* 0x40 - 0x4f ******************************************************/ - - case 0x44: /* FD_SBPF_OP_OR_IMM */ - | or dst32, imm - break; - - case 0x45: /* FD_SBPF_OP_JSET_IMM */ - | test dst64, imm - | jnz =>jmp_dst_lbl - break; - - case 0x47: /* FD_SBPF_OP_OR64_IMM */ - | or dst64, imm - break; - - case 0x4c: /* FD_SBPF_OP_OR_REG */ - | or dst32, src32 - break; - - case 0x4d: /* FD_SBPF_OP_JSET_REG */ - | test dst64, src64 - | jnz =>jmp_dst_lbl - break; - - case 0x4f: /* FD_SBPF_OP_OR64_REG */ - | or dst64, src64 - break; - - /* 0x50 - 0x5f ******************************************************/ - - case 0x54: /* FD_SBPF_OP_AND_IMM */ - | and dst32, imm - break; - - case 0x55: /* FD_SBPF_OP_JNE_IMM */ - | cmp dst64, imm - | jne =>jmp_dst_lbl - break; - - case 0x57: /* FD_SBPF_OP_AND64_IMM */ - | and dst64, imm - break; - - case 0x5c: /* FD_SBPF_OP_AND_REG */ - | and dst32, src32 - break; - - case 0x5d: /* FD_SBPF_OP_JNE_REG */ - | cmp dst64, src64 - | jne =>jmp_dst_lbl - break; - - case 0x5f: /* FD_SBPF_OP_AND64_REG */ - | and dst64, src64 - break; - - /* 0x60 - 0x6f ******************************************************/ - - case 0x61: /* FD_SBPF_OP_LDXW */ - | lea translate_in, [src64+offset] - | translate_ro_4 - | mov dst32, [translate_out] - break; - - case 0x62: /* FD_SBPF_OP_STW */ - | lea translate_in, [dst64+offset] - | translate_rw_4 - | mov dword [translate_out], imm - break; - - case 0x63: /* FD_SBPF_OP_STXW */ - | lea translate_in, [dst64+offset] - | translate_rw_4 - | mov [translate_out], src32 - break; - - case 0x64: /* FD_SBPF_OP_LSH_IMM */ - | shl dst32, imm - break; - - case 0x65: /* FD_SBPF_OP_JSGT_IMM */ - | cmp dst64, imm - | jg =>jmp_dst_lbl - break; - - case 0x67: /* FD_SBPF_OP_LSH64_IMM */ - | shl dst64, imm - break; - - case 0x69: /* FD_SBPF_OP_LDXH */ - | lea translate_in, [src64+offset] - | translate_ro_2 - | xor dst32, dst32 - | mov Rw(x86_dst), [translate_out] - break; - - case 0x6a: /* FD_SBPF_OP_STH */ - | lea translate_in, [dst64+offset] - | translate_rw_2 - | mov word [translate_out], imm - break; - - case 0x6b: /* FD_SBPF_OP_STXH */ - | lea translate_in, [dst64+offset] - | translate_rw_2 - | mov [translate_out], src32 - break; - - case 0x6c: /* FD_SBPF_OP_LSH_REG */ - | mov cl, src8 - | shl dst32, cl - break; - - case 0x6d: /* FD_SBPF_OP_JSGT_REG */ - | cmp dst64, src64 - | jg =>jmp_dst_lbl - break; - - case 0x6f: /* FD_SBPF_OP_LSH64_REG */ - | mov cl, src8 - | shl dst64, cl - break; - - /* 0x70 - 0x7f ******************************************************/ - - case 0x71: /* FD_SBPF_OP_LDXB */ - | lea translate_in, [src64+offset] - | translate_ro_1 - /* TODO is there a better way to zero upper and mov byte? */ - | xor dst32, dst32 - | mov Rb(x86_dst), [translate_out] - break; - - case 0x72: /* FD_SBPF_OP_STB */ - | lea translate_in, [src64+offset] - | translate_rw_1 - | mov byte [translate_out], imm - break; - - case 0x73: /* FD_SBPF_OP_STXB */ - | lea translate_in, [dst64+offset] - | translate_rw_1 - | mov byte [translate_out], Rb(x86_src) - break; - - case 0x74: /* FD_SBPF_OP_RSH_IMM */ - | shr dst32, imm - break; - - case 0x75: /* FD_SBPF_OP_JSGE_IMM */ - | cmp dst64, imm - | jge =>jmp_dst_lbl - break; - - case 0x77: /* FD_SBPF_OP_RSH64_IMM */ - | shr dst64, imm - break; - - case 0x79: /* FD_SBPF_OP_LDXQ */ - | lea translate_in, [src64+offset] - | translate_ro_8 - | mov dst64, [translate_out] - break; - - case 0x7a: /* FD_SBPF_OP_STQ */ - | lea translate_in, [dst64+offset] - | translate_rw_8 - | mov dword [translate_out], imm - | mov dword [translate_out+4], 0 - break; - - case 0x7b: /* FD_SBPF_OP_STXQ */ - | lea translate_in, [dst64+offset] - | translate_rw_8 - | mov [translate_out], src64 - break; - - case 0x7c: /* FD_SBPF_OP_RSH_REG */ - | mov cl, src8 - | shr dst32, cl - break; - - case 0x7d: /* FD_SBPF_OP_JSGE_REG */ - | cmp dst64, src64 - | jge =>jmp_dst_lbl - break; - - case 0x7f: /* FD_SBPF_OP_RSH64_REG */ - | mov cl, src8 - | shr dst64, cl - break; - - /* 0x80-0x8f ********************************************************/ - - case 0x84: /* FD_SBPF_OP_NEG */ - | neg dst32 - break; - - case 0x85: { /* FD_SBPF_OP_CALL_IMM */ - fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, imm, NULL ); - if( !syscall ) { - ulong target_pc = (ulong)fd_pchash_inverse( imm ); - | call ->call_stack_push - | call =>target_pc - } else { - /* Optimize for code footprint: Generate an offset into the - syscall table (32-bit) instead of the syscall address (64-bit) */ - | mov rdi, (uint)( (ulong)syscall - (ulong)syscalls ); - | call ->emulate_syscall - } - break; - } - - case 0x87: /* FD_SBPF_OP_NEG64 */ - | neg dst64 - break; - - case 0x8d: /* FD_SBPF_OP_CALL_REG */ - FD_LOG_WARNING(( "TODO: CALLX" )); - break; - - /* 0x90 - 0x9f ******************************************************/ - - case 0x94: /* FD_SBPF_OP_MOD_IMM */ - if( FD_UNLIKELY( imm==0 ) ) { - | jmp ->vm_fault - break; - } - | xchg eax, dst32 - | xor edx, edx - | mov edi, imm - | div edi - | xchg edx, dst32 - break; - - case 0x95: /* FD_SBPF_OP_EXIT */ - | call ->call_stack_pop - | ret - break; - - case 0x97: /* FD_SBPF_OP_MOD64_IMM */ - if( FD_UNLIKELY( imm==0 ) ) { - | jmp ->vm_fault - break; - } - | xchg rax, dst64 - | xor edx, edx - | mov rdi, imm - | div rdi - | xchg rax, dst64 - break; - - case 0x9c: /* FD_SBPF_OP_MOD_REG */ - | test src32, src32 - | jz ->vm_fault - if( x86_dst==x86_src ) { - | mov dst32, 0 - break; - } - | xchg eax, dst32 - | xor edx, edx - | div src32 - | xchg edx, dst32 - break; - - case 0x9f: /* FD_SBPF_OP_MOD64_REG */ - | test src64, src64 - | jz ->vm_fault - if( x86_dst==x86_src ) { - | mov dst32, 0 - break; - } - | xchg rax, dst64 - | xor edx, edx - | div src64 - | xchg rdx, dst64 - break; - - /* 0xa0 - 0xaf ******************************************************/ - - case 0xa4: /* FD_SBPF_OP_XOR_IMM */ - | xor dst32, imm - break; - - case 0xa5: /* FD_SBPF_OP_JLT_IMM */ - | cmp dst64, imm - | jb =>jmp_dst_lbl - break; - - case 0xa7: /* FD_SBPF_OP_XOR64_IMM */ - // TODO sign extension - | xor dst64, imm - break; - - case 0xac: /* FD_SBPF_OP_XOR_REG */ - | xor dst32, src32 - break; - - case 0xad: /* FD_SBPF_OP_JLT_REG */ - | cmp dst64, src64 - | jb =>jmp_dst_lbl - break; - - case 0xaf: /* FD_SBPF_OP_XOR64_REG */ - | xor dst64, src64 - break; - - /* 0xb0 - 0xbf ******************************************************/ - - case 0xb4: /* FD_SBPF_OP_MOV_IMM */ - | mov dst32, imm - break; - - case 0xb5: /* FD_SBPF_OP_JLE_IMM */ - | cmp dst64, imm - | jbe =>jmp_dst_lbl - break; - - case 0xb7: /* FD_SBPF_OP_MOV64_IMM */ - if( imm==0 ) { - | xor dst32, dst32 - } else { - | mov dst64, imm - } - break; - - case 0xbc: /* FD_SBPF_OP_MOV_REG */ - | mov dst32, src32 - break; - - case 0xbd: /* FD_SBPF_OP_JLE_REG */ - | cmp dst64, src64 - | jbe =>jmp_dst_lbl - break; - - case 0xbf: /* FD_SBPF_OP_MOV64_REG */ - | mov dst64, src64 - break; - - /* 0xc0 - 0xcf ******************************************************/ - - case 0xc4: /* FD_SBPF_OP_ARSH_IMM */ - | sar dst32, imm - break; - - case 0xc5: /* FD_SBPF_OP_JSLT_IMM */ - | cmp dst64, imm - | jl =>jmp_dst_lbl - break; - - case 0xc7: /* FD_SBPF_OP_ARSH64_IMM */ - | sar dst64, imm - break; - - case 0xcc: /* FD_SBPF_OP_ARSH_REG */ - | mov cl, src8 - | sar dst32, cl - break; - - case 0xcd: /* FD_SBPF_OP_JSLT_REG */ - | cmp dst64, src64 - | jl =>jmp_dst_lbl - break; - - case 0xcf: /* FD_SBPF_OP_ARSH64_REG */ - | mov cl, src8 - | sar dst64, cl - break; - - /* 0xd0 - 0xdf ******************************************************/ - - case 0xd4: /* FD_SBPF_OP_END_LE */ - /* nop */ - break; - - case 0xd5: /* FD_SBPF_OP_JSLE_IMM */ - | cmp dst64, imm - | jle =>jmp_dst_lbl - break; - - case 0xdc: /* FD_SBPF_OP_END_BE */ - switch( imm ) { - case 16U: - | movzx dst32, Rw(x86_dst) - | ror Rw(x86_dst), 8 - break; - case 32U: - | bswap dst32 - break; - case 64U: - | bswap dst64 - break; - default: - break; - // TODO sigill - } - break; - - case 0xdd: /* FD_SBPF_OP_JSLE_REG */ - | cmp dst64, src64 - | jle =>jmp_dst_lbl - break; - - default: - FD_LOG_WARNING(( "Unsupported opcode %lx", opcode )); - cur = text_end; - break; - - } - - } - - |->overrun: - | jmp ->vm_fault - -} - -fd_jit_entrypoint_t -fd_jit_get_entrypoint( void ) { - return (fd_jit_entrypoint_t)(ulong)fd_jit_labels[ fd_jit_lbl_entrypoint ]; -} diff --git a/src/flamenco/vm/jit/fd_jit_private.h b/src/flamenco/vm/jit/fd_jit_private.h deleted file mode 100644 index d3a55cf364..0000000000 --- a/src/flamenco/vm/jit/fd_jit_private.h +++ /dev/null @@ -1,170 +0,0 @@ -#ifndef HEADER_fd_src_flamenco_vm_jit_fd_jit_private_h -#define HEADER_fd_src_flamenco_vm_jit_fd_jit_private_h - -#include "fd_jit.h" -#include - -/* DynASM code uses realloc. fd_jit can estimate the number of bytes - needed, so we can do better and allocate once on startup. - - DynASM calls realloc in 4 places: On initialization in dasm_init, - dasm_setupglobal, and dasm_growpc. Then, for every block of code - in dasm_put. - - Checks that enough memory was allocated on initialization are moved - to test_vm_jit. Checks during code generation (dasm_put) are moved - to the fd_dasm_grow_check function call. Calls to 'realloc' and - 'free' are removed. */ - -#include "dasm_proto.h" - -/* FD_DASM_R{...} specify the dynasm register index of x86_64 registers. */ - -#define FD_DASM_RAX (0) -#define FD_DASM_RCX (1) -#define FD_DASM_RDX (2) -#define FD_DASM_RBX (3) -#define FD_DASM_RSP (4) -#define FD_DASM_RBP (5) -#define FD_DASM_RSI (6) -#define FD_DASM_RDI (7) -#define FD_DASM_R8 (8) -#define FD_DASM_R9 (9) -#define FD_DASM_R10 (10) -#define FD_DASM_R11 (11) -#define FD_DASM_R12 (12) -#define FD_DASM_R13 (13) -#define FD_DASM_R14 (14) -#define FD_DASM_R15 (15) - -/* FD_VM_JIT_SEGMENT_MAX is the max number of segments. */ - -#define FD_VM_JIT_SEGMENT_MAX (64) - -/* Thread-local storage ************************************************ - - For now, these are assumed to be absolute-addressed using the fs - segment selector. Practically, this means that fd_jit only supports - targets with FD_HAS_THREADS. (Other targets might use absolute - addressing without a segment selector or rip-relative) */ - -FD_PROTOTYPES_BEGIN - -extern FD_TL fd_vm_t * fd_jit_vm; /* current VM being executed */ -extern FD_TL fd_sbpf_syscalls_t const * fd_jit_syscalls; /* current syscall table */ - -/* Thread-local storage for address translation - - fd_jit_segment_cnt is number of memory regions mapped in by the VM. - For each region i, fd_jit_mem_sz[2*i] is the number of readable bytes - and fd_jit_mem_sz[2*i+1] is the number of writable bytes. - fd_jit_mem_haddr points to the first byte of a region in host address - space. */ - -extern FD_TL uint fd_jit_segment_cnt; -extern FD_TL uint fd_jit_mem_sz [ 2*FD_VM_JIT_SEGMENT_MAX ]; -extern FD_TL ulong fd_jit_mem_haddr[ FD_VM_JIT_SEGMENT_MAX ]; - -/* Thread-local storage for fast return to JIT entrypoint - These are a setjmp()-like anchor for quickly exiting out of a VM - execution, e.g. in case of a VM fault. - Slots: 0=rbx 1=rbp 2=r12 3=r13 4=r14 5=r15 6=rsp 7=rip */ - -extern FD_TL ulong fd_jit_jmp_buf[8]; - -/* Thread-local storage for exception handling */ - -extern FD_TL ulong fd_jit_segfault_vaddr; -extern FD_TL ulong fd_jit_segfault_rip; - -/* Thread-local storage for compile time */ - -/* fd_jit_compile_abort is a setjmp buffer to quickly abort a JIT - compile operation without unwinding. */ - -extern FD_TL jmp_buf fd_jit_compile_abort; - -/* fd_jit_code_section points to the code buffer managed by - dasm_Section. This thread-local is used to detect when this - area is about to run out of space. */ - -extern FD_TL void * fd_jit_code_section_base; -extern FD_TL ulong fd_jit_code_section_sz; - -/* fd_jit_labels is a table of function pointers to 'static' labels in the - JIT code. They are indexed by fd_jit_lbl_{...}. Only used at - compile time. */ - -#define FD_JIT_LABEL_CNT 16 -extern FD_TL void * fd_jit_labels[ FD_JIT_LABEL_CNT ]; - -/* FD_JIT_BLOAT_BASE approximates the number of code bytes that the JIT - compiler generates regardless of the BPF instruction count. */ - -#define FD_JIT_BLOAT_BASE (10000UL) - -/* FD_JIT_BLOAT_MAX is the max acceptable JIT code bloat factor - (Ratio jit_code_sz / bpf_sz) */ - -#define FD_JIT_BLOAT_MAX (3.0f) /* FIXME choose value based on mainnet contracts */ - -FD_PROTOTYPES_END - -/* fd_jit_scratch_layout_t describes the layout of the scratch region. - The scratch region contains preallocated objects used by DynASM. */ - -struct fd_jit_scratch_layout { - ulong dasm_off; - ulong dasm_sz; - - ulong lglabels_off; - ulong lglabels_sz; - - ulong pclabels_off; - ulong pclabels_sz; - - ulong code_off; - ulong code_sz; - - ulong sz; -}; - -typedef struct fd_jit_scratch_layout fd_jit_scratch_layout_t; - -FD_PROTOTYPES_BEGIN - -/* fd_jit_scratch_layout proposes a memory layout for - - The size of the scratch region is assumed to be - fd_jit_est_scratch_sz. */ - -fd_jit_scratch_layout_t * -fd_jit_scratch_layout( fd_jit_scratch_layout_t * scratch, - ulong bpf_sz ); - -/* fd_jit_prepare constructs a dasm_State object in the given scratch - memory region. Calls dasm_init and dasm_setup. */ - -dasm_State * -fd_jit_prepare( void * scratch, - fd_jit_scratch_layout_t const * layout ); - -/* FIXME documentation for fd_jit_compile. */ - -void -fd_jit_compile( struct dasm_State ** Dst, - fd_sbpf_program_t const * prog, - fd_sbpf_syscalls_t const * syscalls ); - -/* fd_jit_vm_compatible checks whether a VM instance is compatible with - fd_jit. Returns FD_VM_SUCCESS or FD_VM_ERR_UNSUP. */ - -FD_FN_PURE int -fd_jit_vm_compatible( fd_vm_t const * vm ); - -fd_jit_entrypoint_t -fd_jit_get_entrypoint( void ); - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_flamenco_vm_jit_fd_jit_private_h */ diff --git a/src/flamenco/vm/jit/test_jit_dasm.c b/src/flamenco/vm/jit/test_jit_dasm.c deleted file mode 100644 index b1d2c40ce0..0000000000 --- a/src/flamenco/vm/jit/test_jit_dasm.c +++ /dev/null @@ -1,39 +0,0 @@ -/* test_jit_dasm tests fd_jit's integration with DynASM, particularly - the custom memory layout of the dasm_State object. */ - -#include "fd_jit_private.h" - -#define FD_DASM_HEADER_ONLY 1 -#include "dasm_x86.h" - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - static uchar scratch[ 16384 ]; - ulong scratch_bufsz = fd_jit_est_scratch_sz( 8UL ); - FD_TEST( scratch_bufsz<=sizeof(scratch) ); - - fd_jit_scratch_layout_t layout[1]; - fd_jit_scratch_layout( layout, 8UL ); - /* TODO CBMC proof that scratch layout regions are in bounds for - every program size up to 24MiB. */ - - dasm_State * state = fd_jit_prepare( scratch, layout ); - FD_TEST( state ); - -# define PTR_WITHIN( ptr, base, len ) \ - ( (ulong)(ptr) >= (ulong)(base) && (ulong)(ptr) < ( (ulong)(base) + (ulong)(len) ) ) - - FD_TEST( state->maxsection == 1 ); - FD_TEST( state->psize == layout->dasm_sz ); - FD_TEST( state->lgsize == layout->lglabels_sz ); - FD_TEST( state->pcsize == layout->pclabels_sz ); - FD_TEST( PTR_WITHIN( state->lglabels, scratch, scratch_bufsz ) ); - FD_TEST( PTR_WITHIN( state->pclabels, scratch, scratch_bufsz ) ); - FD_TEST( PTR_WITHIN( state->sections[0].rbuf, scratch, scratch_bufsz ) ); - - fd_halt(); - return 0; -}