| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| David S. Miller | 8218 | 95.17% | 11 | 36.67% |
| Daniel Borkmann | 181 | 2.10% | 7 | 23.33% |
| Jiong Wang | 160 | 1.85% | 1 | 3.33% |
| Brendan Jackman | 35 | 0.41% | 1 | 3.33% |
| Martin KaFai Lau | 15 | 0.17% | 2 | 6.67% |
| Jakub Kiciński | 14 | 0.16% | 1 | 3.33% |
| Alexei Starovoitov | 4 | 0.05% | 2 | 6.67% |
| Christophe Leroy | 4 | 0.05% | 1 | 3.33% |
| Tiezhu Yang | 1 | 0.01% | 1 | 3.33% |
| Christoph Hellwig | 1 | 0.01% | 1 | 3.33% |
| Greg Kroah-Hartman | 1 | 0.01% | 1 | 3.33% |
| Song Liu | 1 | 0.01% | 1 | 3.33% |
| Total | 8635 | 30 |
// SPDX-License-Identifier: GPL-2.0 #include <linux/moduleloader.h> #include <linux/workqueue.h> #include <linux/netdevice.h> #include <linux/filter.h> #include <linux/bpf.h> #include <linux/cache.h> #include <linux/if_vlan.h> #include <asm/cacheflush.h> #include <asm/ptrace.h> #include "bpf_jit_64.h" static inline bool is_simm13(unsigned int value) { return value + 0x1000 < 0x2000; } static inline bool is_simm10(unsigned int value) { return value + 0x200 < 0x400; } static inline bool is_simm5(unsigned int value) { return value + 0x10 < 0x20; } static inline bool is_sethi(unsigned int value) { return (value & ~0x3fffff) == 0; } static void bpf_flush_icache(void *start_, void *end_) { /* Cheetah's I-cache is fully coherent. */ if (tlb_type == spitfire) { unsigned long start = (unsigned long) start_; unsigned long end = (unsigned long) end_; start &= ~7UL; end = (end + 7UL) & ~7UL; while (start < end) { flushi(start); start += 32; } } } #define S13(X) ((X) & 0x1fff) #define S5(X) ((X) & 0x1f) #define IMMED 0x00002000 #define RD(X) ((X) << 25) #define RS1(X) ((X) << 14) #define RS2(X) ((X)) #define OP(X) ((X) << 30) #define OP2(X) ((X) << 22) #define OP3(X) ((X) << 19) #define COND(X) (((X) & 0xf) << 25) #define CBCOND(X) (((X) & 0x1f) << 25) #define F1(X) OP(X) #define F2(X, Y) (OP(X) | OP2(Y)) #define F3(X, Y) (OP(X) | OP3(Y)) #define ASI(X) (((X) & 0xff) << 5) #define CONDN COND(0x0) #define CONDE COND(0x1) #define CONDLE COND(0x2) #define CONDL COND(0x3) #define CONDLEU COND(0x4) #define CONDCS COND(0x5) #define CONDNEG COND(0x6) #define CONDVC COND(0x7) #define CONDA COND(0x8) #define CONDNE COND(0x9) #define CONDG COND(0xa) #define CONDGE COND(0xb) #define CONDGU COND(0xc) #define CONDCC COND(0xd) #define CONDPOS COND(0xe) #define CONDVS COND(0xf) #define CONDGEU CONDCC #define CONDLU CONDCS #define WDISP22(X) (((X) >> 2) & 0x3fffff) #define WDISP19(X) (((X) >> 2) & 0x7ffff) /* The 10-bit branch displacement for CBCOND is split into two fields */ static u32 WDISP10(u32 off) { u32 ret = ((off >> 2) & 0xff) << 5; ret |= ((off >> (2 + 8)) & 0x03) << 19; return ret; } #define CBCONDE CBCOND(0x09) #define CBCONDLE CBCOND(0x0a) #define CBCONDL CBCOND(0x0b) #define CBCONDLEU CBCOND(0x0c) #define CBCONDCS CBCOND(0x0d) #define CBCONDN CBCOND(0x0e) #define CBCONDVS CBCOND(0x0f) #define CBCONDNE CBCOND(0x19) #define CBCONDG CBCOND(0x1a) #define CBCONDGE CBCOND(0x1b) #define CBCONDGU CBCOND(0x1c) #define CBCONDCC CBCOND(0x1d) #define CBCONDPOS CBCOND(0x1e) #define CBCONDVC CBCOND(0x1f) #define CBCONDGEU CBCONDCC #define CBCONDLU CBCONDCS #define ANNUL (1 << 29) #define XCC (1 << 21) #define BRANCH (F2(0, 1) | XCC) #define CBCOND_OP (F2(0, 3) | XCC) #define BA (BRANCH | CONDA) #define BG (BRANCH | CONDG) #define BL (BRANCH | CONDL) #define BLE (BRANCH | CONDLE) #define BGU (BRANCH | CONDGU) #define BLEU (BRANCH | CONDLEU) #define BGE (BRANCH | CONDGE) #define BGEU (BRANCH | CONDGEU) #define BLU (BRANCH | CONDLU) #define BE (BRANCH | CONDE) #define BNE (BRANCH | CONDNE) #define SETHI(K, REG) \ (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) #define OR_LO(K, REG) \ (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG)) #define ADD F3(2, 0x00) #define AND F3(2, 0x01) #define ANDCC F3(2, 0x11) #define OR F3(2, 0x02) #define XOR F3(2, 0x03) #define SUB F3(2, 0x04) #define SUBCC F3(2, 0x14) #define MUL F3(2, 0x0a) #define MULX F3(2, 0x09) #define UDIVX F3(2, 0x0d) #define DIV F3(2, 0x0e) #define SLL F3(2, 0x25) #define SLLX (F3(2, 0x25)|(1<<12)) #define SRA F3(2, 0x27) #define SRAX (F3(2, 0x27)|(1<<12)) #define SRL F3(2, 0x26) #define SRLX (F3(2, 0x26)|(1<<12)) #define JMPL F3(2, 0x38) #define SAVE F3(2, 0x3c) #define RESTORE F3(2, 0x3d) #define CALL F1(1) #define BR F2(0, 0x01) #define RD_Y F3(2, 0x28) #define WR_Y F3(2, 0x30) #define LD32 F3(3, 0x00) #define LD8 F3(3, 0x01) #define LD16 F3(3, 0x02) #define LD64 F3(3, 0x0b) #define LD64A F3(3, 0x1b) #define ST8 F3(3, 0x05) #define ST16 F3(3, 0x06) #define ST32 F3(3, 0x04) #define ST64 F3(3, 0x0e) #define CAS F3(3, 0x3c) #define CASX F3(3, 0x3e) #define LDPTR LD64 #define BASE_STACKFRAME 176 #define LD32I (LD32 | IMMED) #define LD8I (LD8 | IMMED) #define LD16I (LD16 | IMMED) #define LD64I (LD64 | IMMED) #define LDPTRI (LDPTR | IMMED) #define ST32I (ST32 | IMMED) struct jit_ctx { struct bpf_prog *prog; unsigned int *offset; int idx; int epilogue_offset; bool tmp_1_used; bool tmp_2_used; bool tmp_3_used; bool saw_frame_pointer; bool saw_call; bool saw_tail_call; u32 *image; }; #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) #define TMP_REG_3 (MAX_BPF_JIT_REG + 2) /* Map BPF registers to SPARC registers */ static const int bpf2sparc[] = { /* return value from in-kernel function, and exit value from eBPF */ [BPF_REG_0] = O5, /* arguments from eBPF program to in-kernel function */ [BPF_REG_1] = O0, [BPF_REG_2] = O1, [BPF_REG_3] = O2, [BPF_REG_4] = O3, [BPF_REG_5] = O4, /* callee saved registers that in-kernel function will preserve */ [BPF_REG_6] = L0, [BPF_REG_7] = L1, [BPF_REG_8] = L2, [BPF_REG_9] = L3, /* read-only frame pointer to access stack */ [BPF_REG_FP] = L6, [BPF_REG_AX] = G7, /* temporary register for BPF JIT */ [TMP_REG_1] = G1, [TMP_REG_2] = G2, [TMP_REG_3] = G3, }; static void emit(const u32 insn, struct jit_ctx *ctx) { if (ctx->image != NULL) ctx->image[ctx->idx] = insn; ctx->idx++; } static void emit_call(u32 *func, struct jit_ctx *ctx) { if (ctx->image != NULL) { void *here = &ctx->image[ctx->idx]; unsigned int off; off = (void *)func - here; ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff); } ctx->idx++; } static void emit_nop(struct jit_ctx *ctx) { emit(SETHI(0, G0), ctx); } static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx) { emit(OR | RS1(G0) | RS2(from) | RD(to), ctx); } /* Emit 32-bit constant, zero extended. */ static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx) { emit(SETHI(K, reg), ctx); emit(OR_LO(K, reg), ctx); } /* Emit 32-bit constant, sign extended. */ static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx) { if (K >= 0) { emit(SETHI(K, reg), ctx); emit(OR_LO(K, reg), ctx); } else { u32 hbits = ~(u32) K; u32 lbits = -0x400 | (u32) K; emit(SETHI(hbits, reg), ctx); emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx); } } static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx) { emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx); } static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx) { emit(opcode | RS1(a) | RS2(b) | RD(c), ctx); } static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm, struct jit_ctx *ctx) { bool small_immed = is_simm13(imm); unsigned int insn = opcode; insn |= RS1(dst) | RD(dst); if (small_immed) { emit(insn | IMMED | S13(imm), ctx); } else { unsigned int tmp = bpf2sparc[TMP_REG_1]; ctx->tmp_1_used = true; emit_set_const_sext(imm, tmp, ctx); emit(insn | RS2(tmp), ctx); } } static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm, unsigned int dst, struct jit_ctx *ctx) { bool small_immed = is_simm13(imm); unsigned int insn = opcode; insn |= RS1(src) | RD(dst); if (small_immed) { emit(insn | IMMED | S13(imm), ctx); } else { unsigned int tmp = bpf2sparc[TMP_REG_1]; ctx->tmp_1_used = true; emit_set_const_sext(imm, tmp, ctx); emit(insn | RS2(tmp), ctx); } } static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx) { if (K >= 0 && is_simm13(K)) { /* or %g0, K, DEST */ emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); } else { emit_set_const(K, dest, ctx); } } static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx) { if (is_simm13(K)) { /* or %g0, K, DEST */ emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); } else { emit_set_const(K, dest, ctx); } } static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx) { if (is_simm13(K)) { /* or %g0, K, DEST */ emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); } else { emit_set_const_sext(K, dest, ctx); } } static void analyze_64bit_constant(u32 high_bits, u32 low_bits, int *hbsp, int *lbsp, int *abbasp) { int lowest_bit_set, highest_bit_set, all_bits_between_are_set; int i; lowest_bit_set = highest_bit_set = -1; i = 0; do { if ((lowest_bit_set == -1) && ((low_bits >> i) & 1)) lowest_bit_set = i; if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1)) highest_bit_set = (64 - i - 1); } while (++i < 32 && (highest_bit_set == -1 || lowest_bit_set == -1)); if (i == 32) { i = 0; do { if (lowest_bit_set == -1 && ((high_bits >> i) & 1)) lowest_bit_set = i + 32; if (highest_bit_set == -1 && ((low_bits >> (32 - i - 1)) & 1)) highest_bit_set = 32 - i - 1; } while (++i < 32 && (highest_bit_set == -1 || lowest_bit_set == -1)); } all_bits_between_are_set = 1; for (i = lowest_bit_set; i <= highest_bit_set; i++) { if (i < 32) { if ((low_bits & (1 << i)) != 0) continue; } else { if ((high_bits & (1 << (i - 32))) != 0) continue; } all_bits_between_are_set = 0; break; } *hbsp = highest_bit_set; *lbsp = lowest_bit_set; *abbasp = all_bits_between_are_set; } static unsigned long create_simple_focus_bits(unsigned long high_bits, unsigned long low_bits, int lowest_bit_set, int shift) { long hi, lo; if (lowest_bit_set < 32) { lo = (low_bits >> lowest_bit_set) << shift; hi = ((high_bits << (32 - lowest_bit_set)) << shift); } else { lo = 0; hi = ((high_bits >> (lowest_bit_set - 32)) << shift); } return hi | lo; } static bool const64_is_2insns(unsigned long high_bits, unsigned long low_bits) { int highest_bit_set, lowest_bit_set, all_bits_between_are_set; if (high_bits == 0 || high_bits == 0xffffffff) return true; analyze_64bit_constant(high_bits, low_bits, &highest_bit_set, &lowest_bit_set, &all_bits_between_are_set); if ((highest_bit_set == 63 || lowest_bit_set == 0) && all_bits_between_are_set != 0) return true; if (highest_bit_set - lowest_bit_set < 21) return true; return false; } static void sparc_emit_set_const64_quick2(unsigned long high_bits, unsigned long low_imm, unsigned int dest, int shift_count, struct jit_ctx *ctx) { emit_loadimm32(high_bits, dest, ctx); /* Now shift it up into place. */ emit_alu_K(SLLX, dest, shift_count, ctx); /* If there is a low immediate part piece, finish up by * putting that in as well. */ if (low_imm != 0) emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx); } static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx) { int all_bits_between_are_set, lowest_bit_set, highest_bit_set; unsigned int tmp = bpf2sparc[TMP_REG_1]; u32 low_bits = (K & 0xffffffff); u32 high_bits = (K >> 32); /* These two tests also take care of all of the one * instruction cases. */ if (high_bits == 0xffffffff && (low_bits & 0x80000000)) return emit_loadimm_sext(K, dest, ctx); if (high_bits == 0x00000000) return emit_loadimm32(K, dest, ctx); analyze_64bit_constant(high_bits, low_bits, &highest_bit_set, &lowest_bit_set, &all_bits_between_are_set); /* 1) mov -1, %reg * sllx %reg, shift, %reg * 2) mov -1, %reg * srlx %reg, shift, %reg * 3) mov some_small_const, %reg * sllx %reg, shift, %reg */ if (((highest_bit_set == 63 || lowest_bit_set == 0) && all_bits_between_are_set != 0) || ((highest_bit_set - lowest_bit_set) < 12)) { int shift = lowest_bit_set; long the_const = -1; if ((highest_bit_set != 63 && lowest_bit_set != 0) || all_bits_between_are_set == 0) { the_const = create_simple_focus_bits(high_bits, low_bits, lowest_bit_set, 0); } else if (lowest_bit_set == 0) shift = -(63 - highest_bit_set); emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx); if (shift > 0) emit_alu_K(SLLX, dest, shift, ctx); else if (shift < 0) emit_alu_K(SRLX, dest, -shift, ctx); return; } /* Now a range of 22 or less bits set somewhere. * 1) sethi %hi(focus_bits), %reg * sllx %reg, shift, %reg * 2) sethi %hi(focus_bits), %reg * srlx %reg, shift, %reg */ if ((highest_bit_set - lowest_bit_set) < 21) { unsigned long focus_bits = create_simple_focus_bits(high_bits, low_bits, lowest_bit_set, 10); emit(SETHI(focus_bits, dest), ctx); /* If lowest_bit_set == 10 then a sethi alone could * have done it. */ if (lowest_bit_set < 10) emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx); else if (lowest_bit_set > 10) emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx); return; } /* Ok, now 3 instruction sequences. */ if (low_bits == 0) { emit_loadimm32(high_bits, dest, ctx); emit_alu_K(SLLX, dest, 32, ctx); return; } /* We may be able to do something quick * when the constant is negated, so try that. */ if (const64_is_2insns((~high_bits) & 0xffffffff, (~low_bits) & 0xfffffc00)) { /* NOTE: The trailing bits get XOR'd so we need the * non-negated bits, not the negated ones. */ unsigned long trailing_bits = low_bits & 0x3ff; if ((((~high_bits) & 0xffffffff) == 0 && ((~low_bits) & 0x80000000) == 0) || (((~high_bits) & 0xffffffff) == 0xffffffff && ((~low_bits) & 0x80000000) != 0)) { unsigned long fast_int = (~low_bits & 0xffffffff); if ((is_sethi(fast_int) && (~high_bits & 0xffffffff) == 0)) { emit(SETHI(fast_int, dest), ctx); } else if (is_simm13(fast_int)) { emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx); } else { emit_loadimm64(fast_int, dest, ctx); } } else { u64 n = ((~low_bits) & 0xfffffc00) | (((unsigned long)((~high_bits) & 0xffffffff))<<32); emit_loadimm64(n, dest, ctx); } low_bits = -0x400 | trailing_bits; emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx); return; } /* 1) sethi %hi(xxx), %reg * or %reg, %lo(xxx), %reg * sllx %reg, yyy, %reg */ if ((highest_bit_set - lowest_bit_set) < 32) { unsigned long focus_bits = create_simple_focus_bits(high_bits, low_bits, lowest_bit_set, 0); /* So what we know is that the set bits straddle the * middle of the 64-bit word. */ sparc_emit_set_const64_quick2(focus_bits, 0, dest, lowest_bit_set, ctx); return; } /* 1) sethi %hi(high_bits), %reg * or %reg, %lo(high_bits), %reg * sllx %reg, 32, %reg * or %reg, low_bits, %reg */ if (is_simm13(low_bits) && ((int)low_bits > 0)) { sparc_emit_set_const64_quick2(high_bits, low_bits, dest, 32, ctx); return; } /* Oh well, we tried... Do a full 64-bit decomposition. */ ctx->tmp_1_used = true; emit_loadimm32(high_bits, tmp, ctx); emit_loadimm32(low_bits, dest, ctx); emit_alu_K(SLLX, tmp, 32, ctx); emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx); } static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx, struct jit_ctx *ctx) { unsigned int off = to_idx - from_idx; if (br_opc & XCC) emit(br_opc | WDISP19(off << 2), ctx); else emit(br_opc | WDISP22(off << 2), ctx); } static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, const u8 dst, const u8 src, struct jit_ctx *ctx) { unsigned int off = to_idx - from_idx; emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx); } static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, const u8 dst, s32 imm, struct jit_ctx *ctx) { unsigned int off = to_idx - from_idx; emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx); } #define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX) #define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX) #define emit_cmp(R1, R2, CTX) \ emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX) #define emit_cmpi(R1, IMM, CTX) \ emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) #define emit_btst(R1, R2, CTX) \ emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX) #define emit_btsti(R1, IMM, CTX) \ emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src, const s32 imm, bool is_imm, int branch_dst, struct jit_ctx *ctx) { bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0; const u8 tmp = bpf2sparc[TMP_REG_1]; branch_dst = ctx->offset[branch_dst]; if (!is_simm10(branch_dst - ctx->idx) || BPF_OP(code) == BPF_JSET) use_cbcond = false; if (is_imm) { bool fits = true; if (use_cbcond) { if (!is_simm5(imm)) fits = false; } else if (!is_simm13(imm)) { fits = false; } if (!fits) { ctx->tmp_1_used = true; emit_loadimm_sext(imm, tmp, ctx); src = tmp; is_imm = false; } } if (!use_cbcond) { u32 br_opcode; if (BPF_OP(code) == BPF_JSET) { if (is_imm) emit_btsti(dst, imm, ctx); else emit_btst(dst, src, ctx); } else { if (is_imm) emit_cmpi(dst, imm, ctx); else emit_cmp(dst, src, ctx); } switch (BPF_OP(code)) { case BPF_JEQ: br_opcode = BE; break; case BPF_JGT: br_opcode = BGU; break; case BPF_JLT: br_opcode = BLU; break; case BPF_JGE: br_opcode = BGEU; break; case BPF_JLE: br_opcode = BLEU; break; case BPF_JSET: case BPF_JNE: br_opcode = BNE; break; case BPF_JSGT: br_opcode = BG; break; case BPF_JSLT: br_opcode = BL; break; case BPF_JSGE: br_opcode = BGE; break; case BPF_JSLE: br_opcode = BLE; break; default: /* Make sure we dont leak kernel information to the * user. */ return -EFAULT; } emit_branch(br_opcode, ctx->idx, branch_dst, ctx); emit_nop(ctx); } else { u32 cbcond_opcode; switch (BPF_OP(code)) { case BPF_JEQ: cbcond_opcode = CBCONDE; break; case BPF_JGT: cbcond_opcode = CBCONDGU; break; case BPF_JLT: cbcond_opcode = CBCONDLU; break; case BPF_JGE: cbcond_opcode = CBCONDGEU; break; case BPF_JLE: cbcond_opcode = CBCONDLEU; break; case BPF_JNE: cbcond_opcode = CBCONDNE; break; case BPF_JSGT: cbcond_opcode = CBCONDG; break; case BPF_JSLT: cbcond_opcode = CBCONDL; break; case BPF_JSGE: cbcond_opcode = CBCONDGE; break; case BPF_JSLE: cbcond_opcode = CBCONDLE; break; default: /* Make sure we dont leak kernel information to the * user. */ return -EFAULT; } cbcond_opcode |= CBCOND_OP; if (is_imm) emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst, dst, imm, ctx); else emit_cbcond(cbcond_opcode, ctx->idx, branch_dst, dst, src, ctx); } return 0; } /* Just skip the save instruction and the ctx register move. */ #define BPF_TAILCALL_PROLOGUE_SKIP 32 #define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128) static void build_prologue(struct jit_ctx *ctx) { s32 stack_needed = BASE_STACKFRAME; if (ctx->saw_frame_pointer || ctx->saw_tail_call) { struct bpf_prog *prog = ctx->prog; u32 stack_depth; stack_depth = prog->aux->stack_depth; stack_needed += round_up(stack_depth, 16); } if (ctx->saw_tail_call) stack_needed += 8; /* save %sp, -176, %sp */ emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx); /* tail_call_cnt = 0 */ if (ctx->saw_tail_call) { u32 off = BPF_TAILCALL_CNT_SP_OFF; emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx); } else { emit_nop(ctx); } if (ctx->saw_frame_pointer) { const u8 vfp = bpf2sparc[BPF_REG_FP]; emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx); } else { emit_nop(ctx); } emit_reg_move(I0, O0, ctx); emit_reg_move(I1, O1, ctx); emit_reg_move(I2, O2, ctx); emit_reg_move(I3, O3, ctx); emit_reg_move(I4, O4, ctx); /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */ } static void build_epilogue(struct jit_ctx *ctx) { ctx->epilogue_offset = ctx->idx; /* ret (jmpl %i7 + 8, %g0) */ emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx); /* restore %i5, %g0, %o0 */ emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx); } static void emit_tail_call(struct jit_ctx *ctx) { const u8 bpf_array = bpf2sparc[BPF_REG_2]; const u8 bpf_index = bpf2sparc[BPF_REG_3]; const u8 tmp = bpf2sparc[TMP_REG_1]; u32 off; ctx->saw_tail_call = true; off = offsetof(struct bpf_array, map.max_entries); emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx); emit_cmp(bpf_index, tmp, ctx); #define OFFSET1 17 emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx); emit_nop(ctx); off = BPF_TAILCALL_CNT_SP_OFF; emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx); #define OFFSET2 13 emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET2, ctx); emit_nop(ctx); emit_alu_K(ADD, tmp, 1, ctx); off = BPF_TAILCALL_CNT_SP_OFF; emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); emit_alu3_K(SLL, bpf_index, 3, tmp, ctx); emit_alu(ADD, bpf_array, tmp, ctx); off = offsetof(struct bpf_array, ptrs); emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); emit_cmpi(tmp, 0, ctx); #define OFFSET3 5 emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx); emit_nop(ctx); off = offsetof(struct bpf_prog, bpf_func); emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); off = BPF_TAILCALL_PROLOGUE_SKIP; emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx); emit_nop(ctx); } static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) { const u8 code = insn->code; const u8 dst = bpf2sparc[insn->dst_reg]; const u8 src = bpf2sparc[insn->src_reg]; const int i = insn - ctx->prog->insnsi; const s16 off = insn->off; const s32 imm = insn->imm; if (insn->src_reg == BPF_REG_FP) ctx->saw_frame_pointer = true; switch (code) { /* dst = src */ case BPF_ALU | BPF_MOV | BPF_X: emit_alu3_K(SRL, src, 0, dst, ctx); if (insn_is_zext(&insn[1])) return 1; break; case BPF_ALU64 | BPF_MOV | BPF_X: emit_reg_move(src, dst, ctx); break; /* dst = dst OP src */ case BPF_ALU | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_X: emit_alu(ADD, src, dst, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_SUB | BPF_X: case BPF_ALU64 | BPF_SUB | BPF_X: emit_alu(SUB, src, dst, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_AND | BPF_X: case BPF_ALU64 | BPF_AND | BPF_X: emit_alu(AND, src, dst, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU64 | BPF_OR | BPF_X: emit_alu(OR, src, dst, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_X: emit_alu(XOR, src, dst, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_MUL | BPF_X: emit_alu(MUL, src, dst, ctx); goto do_alu32_trunc; case BPF_ALU64 | BPF_MUL | BPF_X: emit_alu(MULX, src, dst, ctx); break; case BPF_ALU | BPF_DIV | BPF_X: emit_write_y(G0, ctx); emit_alu(DIV, src, dst, ctx); if (insn_is_zext(&insn[1])) return 1; break; case BPF_ALU64 | BPF_DIV | BPF_X: emit_alu(UDIVX, src, dst, ctx); break; case BPF_ALU | BPF_MOD | BPF_X: { const u8 tmp = bpf2sparc[TMP_REG_1]; ctx->tmp_1_used = true; emit_write_y(G0, ctx); emit_alu3(DIV, dst, src, tmp, ctx); emit_alu3(MULX, tmp, src, tmp, ctx); emit_alu3(SUB, dst, tmp, dst, ctx); goto do_alu32_trunc; } case BPF_ALU64 | BPF_MOD | BPF_X: { const u8 tmp = bpf2sparc[TMP_REG_1]; ctx->tmp_1_used = true; emit_alu3(UDIVX, dst, src, tmp, ctx); emit_alu3(MULX, tmp, src, tmp, ctx); emit_alu3(SUB, dst, tmp, dst, ctx); break; } case BPF_ALU | BPF_LSH | BPF_X: emit_alu(SLL, src, dst, ctx); goto do_alu32_trunc; case BPF_ALU64 | BPF_LSH | BPF_X: emit_alu(SLLX, src, dst, ctx); break; case BPF_ALU | BPF_RSH | BPF_X: emit_alu(SRL, src, dst, ctx); if (insn_is_zext(&insn[1])) return 1; break; case BPF_ALU64 | BPF_RSH | BPF_X: emit_alu(SRLX, src, dst, ctx); break; case BPF_ALU | BPF_ARSH | BPF_X: emit_alu(SRA, src, dst, ctx); goto do_alu32_trunc; case BPF_ALU64 | BPF_ARSH | BPF_X: emit_alu(SRAX, src, dst, ctx); break; /* dst = -dst */ case BPF_ALU | BPF_NEG: case BPF_ALU64 | BPF_NEG: emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx); goto do_alu32_trunc; case BPF_ALU | BPF_END | BPF_FROM_BE: switch (imm) { case 16: emit_alu_K(SLL, dst, 16, ctx); emit_alu_K(SRL, dst, 16, ctx); if (insn_is_zext(&insn[1])) return 1; break; case 32: if (!ctx->prog->aux->verifier_zext) emit_alu_K(SRL, dst, 0, ctx); break; case 64: /* nop */ break; } break; /* dst = BSWAP##imm(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: { const u8 tmp = bpf2sparc[TMP_REG_1]; const u8 tmp2 = bpf2sparc[TMP_REG_2]; ctx->tmp_1_used = true; switch (imm) { case 16: emit_alu3_K(AND, dst, 0xff, tmp, ctx); emit_alu3_K(SRL, dst, 8, dst, ctx); emit_alu3_K(AND, dst, 0xff, dst, ctx); emit_alu3_K(SLL, tmp, 8, tmp, ctx); emit_alu(OR, tmp, dst, ctx); if (insn_is_zext(&insn[1])) return 1; break; case 32: ctx->tmp_2_used = true; emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */ emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */ emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */ emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */ emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */ emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */ emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */ emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */ if (insn_is_zext(&insn[1])) return 1; break; case 64: emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx); emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx); emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx); break; } break; } /* dst = imm */ case BPF_ALU | BPF_MOV | BPF_K: emit_loadimm32(imm, dst, ctx); if (insn_is_zext(&insn[1])) return 1; break; case BPF_ALU64 | BPF_MOV | BPF_K: emit_loadimm_sext(imm, dst, ctx); break; /* dst = dst OP imm */ case BPF_ALU | BPF_ADD | BPF_K: case BPF_ALU64 | BPF_ADD | BPF_K: emit_alu_K(ADD, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_SUB | BPF_K: case BPF_ALU64 | BPF_SUB | BPF_K: emit_alu_K(SUB, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU64 | BPF_AND | BPF_K: emit_alu_K(AND, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_OR | BPF_K: case BPF_ALU64 | BPF_OR | BPF_K: emit_alu_K(OR, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_XOR | BPF_K: case BPF_ALU64 | BPF_XOR | BPF_K: emit_alu_K(XOR, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU | BPF_MUL | BPF_K: emit_alu_K(MUL, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU64 | BPF_MUL | BPF_K: emit_alu_K(MULX, dst, imm, ctx); break; case BPF_ALU | BPF_DIV | BPF_K: if (imm == 0) return -EINVAL; emit_write_y(G0, ctx); emit_alu_K(DIV, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU64 | BPF_DIV | BPF_K: if (imm == 0) return -EINVAL; emit_alu_K(UDIVX, dst, imm, ctx); break; case BPF_ALU64 | BPF_MOD | BPF_K: case BPF_ALU | BPF_MOD | BPF_K: { const u8 tmp = bpf2sparc[TMP_REG_2]; unsigned int div; if (imm == 0) return -EINVAL; div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV; ctx->tmp_2_used = true; if (BPF_CLASS(code) != BPF_ALU64) emit_write_y(G0, ctx); if (is_simm13(imm)) { emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx); emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx); emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); } else { const u8 tmp1 = bpf2sparc[TMP_REG_1]; ctx->tmp_1_used = true; emit_set_const_sext(imm, tmp1, ctx); emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx); emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx); emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); } goto do_alu32_trunc; } case BPF_ALU | BPF_LSH | BPF_K: emit_alu_K(SLL, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU64 | BPF_LSH | BPF_K: emit_alu_K(SLLX, dst, imm, ctx); break; case BPF_ALU | BPF_RSH | BPF_K: emit_alu_K(SRL, dst, imm, ctx); if (insn_is_zext(&insn[1])) return 1; break; case BPF_ALU64 | BPF_RSH | BPF_K: emit_alu_K(SRLX, dst, imm, ctx); break; case BPF_ALU | BPF_ARSH | BPF_K: emit_alu_K(SRA, dst, imm, ctx); goto do_alu32_trunc; case BPF_ALU64 | BPF_ARSH | BPF_K: emit_alu_K(SRAX, dst, imm, ctx); break; do_alu32_trunc: if (BPF_CLASS(code) == BPF_ALU && !ctx->prog->aux->verifier_zext) emit_alu_K(SRL, dst, 0, ctx); break; /* JUMP off */ case BPF_JMP | BPF_JA: emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx); emit_nop(ctx); break; /* IF (dst COND src) JUMP off */ case BPF_JMP | BPF_JEQ | BPF_X: case BPF_JMP | BPF_JGT | BPF_X: case BPF_JMP | BPF_JLT | BPF_X: case BPF_JMP | BPF_JGE | BPF_X: case BPF_JMP | BPF_JLE | BPF_X: case BPF_JMP | BPF_JNE | BPF_X: case BPF_JMP | BPF_JSGT | BPF_X: case BPF_JMP | BPF_JSLT | BPF_X: case BPF_JMP | BPF_JSGE | BPF_X: case BPF_JMP | BPF_JSLE | BPF_X: case BPF_JMP | BPF_JSET | BPF_X: { int err; err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx); if (err) return err; break; } /* IF (dst COND imm) JUMP off */ case BPF_JMP | BPF_JEQ | BPF_K: case BPF_JMP | BPF_JGT | BPF_K: case BPF_JMP | BPF_JLT | BPF_K: case BPF_JMP | BPF_JGE | BPF_K: case BPF_JMP | BPF_JLE | BPF_K: case BPF_JMP | BPF_JNE | BPF_K: case BPF_JMP | BPF_JSGT | BPF_K: case BPF_JMP | BPF_JSLT | BPF_K: case BPF_JMP | BPF_JSGE | BPF_K: case BPF_JMP | BPF_JSLE | BPF_K: case BPF_JMP | BPF_JSET | BPF_K: { int err; err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx); if (err) return err; break; } /* function call */ case BPF_JMP | BPF_CALL: { u8 *func = ((u8 *)__bpf_call_base) + imm; ctx->saw_call = true; emit_call((u32 *)func, ctx); emit_nop(ctx); emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx); break; } /* tail call */ case BPF_JMP | BPF_TAIL_CALL: emit_tail_call(ctx); break; /* function return */ case BPF_JMP | BPF_EXIT: /* Optimization: when last instruction is EXIT, simply fallthrough to epilogue. */ if (i == ctx->prog->len - 1) break; emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx); emit_nop(ctx); break; /* dst = imm64 */ case BPF_LD | BPF_IMM | BPF_DW: { const struct bpf_insn insn1 = insn[1]; u64 imm64; imm64 = (u64)insn1.imm << 32 | (u32)imm; emit_loadimm64(imm64, dst, ctx); return 1; } /* LDX: dst = *(size *)(src + off) */ case BPF_LDX | BPF_MEM | BPF_W: case BPF_LDX | BPF_MEM | BPF_H: case BPF_LDX | BPF_MEM | BPF_B: case BPF_LDX | BPF_MEM | BPF_DW: { const u8 tmp = bpf2sparc[TMP_REG_1]; u32 opcode = 0, rs2; ctx->tmp_1_used = true; switch (BPF_SIZE(code)) { case BPF_W: opcode = LD32; break; case BPF_H: opcode = LD16; break; case BPF_B: opcode = LD8; break; case BPF_DW: opcode = LD64; break; } if (is_simm13(off)) { opcode |= IMMED; rs2 = S13(off); } else { emit_loadimm(off, tmp, ctx); rs2 = RS2(tmp); } emit(opcode | RS1(src) | rs2 | RD(dst), ctx); if (opcode != LD64 && insn_is_zext(&insn[1])) return 1; break; } /* speculation barrier */ case BPF_ST | BPF_NOSPEC: break; /* ST: *(size *)(dst + off) = imm */ case BPF_ST | BPF_MEM | BPF_W: case BPF_ST | BPF_MEM | BPF_H: case BPF_ST | BPF_MEM | BPF_B: case BPF_ST | BPF_MEM | BPF_DW: { const u8 tmp = bpf2sparc[TMP_REG_1]; const u8 tmp2 = bpf2sparc[TMP_REG_2]; u32 opcode = 0, rs2; if (insn->dst_reg == BPF_REG_FP) ctx->saw_frame_pointer = true; ctx->tmp_2_used = true; emit_loadimm(imm, tmp2, ctx); switch (BPF_SIZE(code)) { case BPF_W: opcode = ST32; break; case BPF_H: opcode = ST16; break; case BPF_B: opcode = ST8; break; case BPF_DW: opcode = ST64; break; } if (is_simm13(off)) { opcode |= IMMED; rs2 = S13(off); } else { ctx->tmp_1_used = true; emit_loadimm(off, tmp, ctx); rs2 = RS2(tmp); } emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx); break; } /* STX: *(size *)(dst + off) = src */ case BPF_STX | BPF_MEM | BPF_W: case BPF_STX | BPF_MEM | BPF_H: case BPF_STX | BPF_MEM | BPF_B: case BPF_STX | BPF_MEM | BPF_DW: { const u8 tmp = bpf2sparc[TMP_REG_1]; u32 opcode = 0, rs2; if (insn->dst_reg == BPF_REG_FP) ctx->saw_frame_pointer = true; switch (BPF_SIZE(code)) { case BPF_W: opcode = ST32; break; case BPF_H: opcode = ST16; break; case BPF_B: opcode = ST8; break; case BPF_DW: opcode = ST64; break; } if (is_simm13(off)) { opcode |= IMMED; rs2 = S13(off); } else { ctx->tmp_1_used = true; emit_loadimm(off, tmp, ctx); rs2 = RS2(tmp); } emit(opcode | RS1(dst) | rs2 | RD(src), ctx); break; } case BPF_STX | BPF_ATOMIC | BPF_W: { const u8 tmp = bpf2sparc[TMP_REG_1]; const u8 tmp2 = bpf2sparc[TMP_REG_2]; const u8 tmp3 = bpf2sparc[TMP_REG_3]; if (insn->imm != BPF_ADD) { pr_err_once("unknown atomic op %02x\n", insn->imm); return -EINVAL; } /* lock *(u32 *)(dst + off) += src */ if (insn->dst_reg == BPF_REG_FP) ctx->saw_frame_pointer = true; ctx->tmp_1_used = true; ctx->tmp_2_used = true; ctx->tmp_3_used = true; emit_loadimm(off, tmp, ctx); emit_alu3(ADD, dst, tmp, tmp, ctx); emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); emit_alu3(ADD, tmp2, src, tmp3, ctx); emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); emit_cmp(tmp2, tmp3, ctx); emit_branch(BNE, 4, 0, ctx); emit_nop(ctx); break; } /* STX XADD: lock *(u64 *)(dst + off) += src */ case BPF_STX | BPF_ATOMIC | BPF_DW: { const u8 tmp = bpf2sparc[TMP_REG_1]; const u8 tmp2 = bpf2sparc[TMP_REG_2]; const u8 tmp3 = bpf2sparc[TMP_REG_3]; if (insn->imm != BPF_ADD) { pr_err_once("unknown atomic op %02x\n", insn->imm); return -EINVAL; } if (insn->dst_reg == BPF_REG_FP) ctx->saw_frame_pointer = true; ctx->tmp_1_used = true; ctx->tmp_2_used = true; ctx->tmp_3_used = true; emit_loadimm(off, tmp, ctx); emit_alu3(ADD, dst, tmp, tmp, ctx); emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); emit_alu3(ADD, tmp2, src, tmp3, ctx); emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); emit_cmp(tmp2, tmp3, ctx); emit_branch(BNE, 4, 0, ctx); emit_nop(ctx); break; } default: pr_err_once("unknown opcode %02x\n", code); return -EINVAL; } return 0; } static int build_body(struct jit_ctx *ctx) { const struct bpf_prog *prog = ctx->prog; int i; for (i = 0; i < prog->len; i++) { const struct bpf_insn *insn = &prog->insnsi[i]; int ret; ret = build_insn(insn, ctx); if (ret > 0) { i++; ctx->offset[i] = ctx->idx; continue; } ctx->offset[i] = ctx->idx; if (ret) return ret; } return 0; } static void jit_fill_hole(void *area, unsigned int size) { u32 *ptr; /* We are guaranteed to have aligned memory. */ for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) *ptr++ = 0x91d02005; /* ta 5 */ } bool bpf_jit_needs_zext(void) { return true; } struct sparc64_jit_data { struct bpf_binary_header *header; u8 *image; struct jit_ctx ctx; }; struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { struct bpf_prog *tmp, *orig_prog = prog; struct sparc64_jit_data *jit_data; struct bpf_binary_header *header; u32 prev_image_size, image_size; bool tmp_blinded = false; bool extra_pass = false; struct jit_ctx ctx; u8 *image_ptr; int pass, i; if (!prog->jit_requested) return orig_prog; tmp = bpf_jit_blind_constants(prog); /* If blinding was requested and we failed during blinding, * we must fall back to the interpreter. */ if (IS_ERR(tmp)) return orig_prog; if (tmp != prog) { tmp_blinded = true; prog = tmp; } jit_data = prog->aux->jit_data; if (!jit_data) { jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); if (!jit_data) { prog = orig_prog; goto out; } prog->aux->jit_data = jit_data; } if (jit_data->ctx.offset) { ctx = jit_data->ctx; image_ptr = jit_data->image; header = jit_data->header; extra_pass = true; image_size = sizeof(u32) * ctx.idx; prev_image_size = image_size; pass = 1; goto skip_init_ctx; } memset(&ctx, 0, sizeof(ctx)); ctx.prog = prog; ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL); if (ctx.offset == NULL) { prog = orig_prog; goto out_off; } /* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook * the offset array so that we converge faster. */ for (i = 0; i < prog->len; i++) ctx.offset[i] = i * (12 * 4); prev_image_size = ~0U; for (pass = 1; pass < 40; pass++) { ctx.idx = 0; build_prologue(&ctx); if (build_body(&ctx)) { prog = orig_prog; goto out_off; } build_epilogue(&ctx); if (bpf_jit_enable > 1) pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass, ctx.idx * 4, ctx.tmp_1_used ? '1' : ' ', ctx.tmp_2_used ? '2' : ' ', ctx.tmp_3_used ? '3' : ' ', ctx.saw_frame_pointer ? 'F' : ' ', ctx.saw_call ? 'C' : ' ', ctx.saw_tail_call ? 'T' : ' '); if (ctx.idx * 4 == prev_image_size) break; prev_image_size = ctx.idx * 4; cond_resched(); } /* Now we know the actual image size. */ image_size = sizeof(u32) * ctx.idx; header = bpf_jit_binary_alloc(image_size, &image_ptr, sizeof(u32), jit_fill_hole); if (header == NULL) { prog = orig_prog; goto out_off; } ctx.image = (u32 *)image_ptr; skip_init_ctx: ctx.idx = 0; build_prologue(&ctx); if (build_body(&ctx)) { bpf_jit_binary_free(header); prog = orig_prog; goto out_off; } build_epilogue(&ctx); if (ctx.idx * 4 != prev_image_size) { pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n", prev_image_size, ctx.idx * 4); bpf_jit_binary_free(header); prog = orig_prog; goto out_off; } if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, image_size, pass, ctx.image); bpf_flush_icache(header, (u8 *)header + header->size); if (!prog->is_func || extra_pass) { if (bpf_jit_binary_lock_ro(header)) { bpf_jit_binary_free(header); prog = orig_prog; goto out_off; } } else { jit_data->ctx = ctx; jit_data->image = image_ptr; jit_data->header = header; } prog->bpf_func = (void *)ctx.image; prog->jited = 1; prog->jited_len = image_size; if (!prog->is_func || extra_pass) { bpf_prog_fill_jited_linfo(prog, ctx.offset); out_off: kfree(ctx.offset); kfree(jit_data); prog->aux->jit_data = NULL; } out: if (tmp_blinded) bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); return prog; }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1