| // SPDX-License-Identifier: GPL-2.0-only |
| /* |
| * Just-In-Time compiler for eBPF bytecode on MIPS. |
| * Implementation of JIT functions for 32-bit CPUs. |
| * |
| * Copyright (c) 2021 Anyfi Networks AB. |
| * Author: Johan Almbladh <johan.almbladh@gmail.com> |
| * |
| * Based on code and ideas from |
| * Copyright (c) 2017 Cavium, Inc. |
| * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com> |
| * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com> |
| */ |
| |
| #include <linux/math64.h> |
| #include <linux/errno.h> |
| #include <linux/filter.h> |
| #include <linux/bpf.h> |
| #include <asm/cpu-features.h> |
| #include <asm/isa-rev.h> |
| #include <asm/uasm.h> |
| |
| #include "bpf_jit_comp.h" |
| |
| /* MIPS a4-a7 are not available in the o32 ABI */ |
| #undef MIPS_R_A4 |
| #undef MIPS_R_A5 |
| #undef MIPS_R_A6 |
| #undef MIPS_R_A7 |
| |
| /* Stack is 8-byte aligned in o32 ABI */ |
| #define MIPS_STACK_ALIGNMENT 8 |
| |
| /* |
| * The top 16 bytes of a stack frame is reserved for the callee in O32 ABI. |
| * This corresponds to stack space for register arguments a0-a3. |
| */ |
| #define JIT_RESERVED_STACK 16 |
| |
| /* Temporary 64-bit register used by JIT */ |
| #define JIT_REG_TMP MAX_BPF_JIT_REG |
| |
| /* |
| * Number of prologue bytes to skip when doing a tail call. |
| * Tail call count (TCC) initialization (8 bytes) always, plus |
| * R0-to-v0 assignment (4 bytes) if big endian. |
| */ |
| #ifdef __BIG_ENDIAN |
| #define JIT_TCALL_SKIP 12 |
| #else |
| #define JIT_TCALL_SKIP 8 |
| #endif |
| |
| /* CPU registers holding the callee return value */ |
| #define JIT_RETURN_REGS \ |
| (BIT(MIPS_R_V0) | \ |
| BIT(MIPS_R_V1)) |
| |
| /* CPU registers arguments passed to callee directly */ |
| #define JIT_ARG_REGS \ |
| (BIT(MIPS_R_A0) | \ |
| BIT(MIPS_R_A1) | \ |
| BIT(MIPS_R_A2) | \ |
| BIT(MIPS_R_A3)) |
| |
| /* CPU register arguments passed to callee on stack */ |
| #define JIT_STACK_REGS \ |
| (BIT(MIPS_R_T0) | \ |
| BIT(MIPS_R_T1) | \ |
| BIT(MIPS_R_T2) | \ |
| BIT(MIPS_R_T3) | \ |
| BIT(MIPS_R_T4) | \ |
| BIT(MIPS_R_T5)) |
| |
| /* Caller-saved CPU registers */ |
| #define JIT_CALLER_REGS \ |
| (JIT_RETURN_REGS | \ |
| JIT_ARG_REGS | \ |
| JIT_STACK_REGS) |
| |
| /* Callee-saved CPU registers */ |
| #define JIT_CALLEE_REGS \ |
| (BIT(MIPS_R_S0) | \ |
| BIT(MIPS_R_S1) | \ |
| BIT(MIPS_R_S2) | \ |
| BIT(MIPS_R_S3) | \ |
| BIT(MIPS_R_S4) | \ |
| BIT(MIPS_R_S5) | \ |
| BIT(MIPS_R_S6) | \ |
| BIT(MIPS_R_S7) | \ |
| BIT(MIPS_R_GP) | \ |
| BIT(MIPS_R_FP) | \ |
| BIT(MIPS_R_RA)) |
| |
| /* |
| * Mapping of 64-bit eBPF registers to 32-bit native MIPS registers. |
| * |
| * 1) Native register pairs are ordered according to CPU endianness, following |
| * the MIPS convention for passing 64-bit arguments and return values. |
| * 2) The eBPF return value, arguments and callee-saved registers are mapped |
| * to their native MIPS equivalents. |
| * 3) Since the 32 highest bits in the eBPF FP register are always zero, |
| * only one general-purpose register is actually needed for the mapping. |
| * We use the fp register for this purpose, and map the highest bits to |
| * the MIPS register r0 (zero). |
| * 4) We use the MIPS gp and at registers as internal temporary registers |
| * for constant blinding. The gp register is callee-saved. |
| * 5) One 64-bit temporary register is mapped for use when sign-extending |
| * immediate operands. MIPS registers t6-t9 are available to the JIT |
| * for as temporaries when implementing complex 64-bit operations. |
| * |
| * With this scheme all eBPF registers are being mapped to native MIPS |
| * registers without having to use any stack scratch space. The direct |
| * register mapping (2) simplifies the handling of function calls. |
| */ |
| static const u8 bpf2mips32[][2] = { |
| /* Return value from in-kernel function, and exit value from eBPF */ |
| [BPF_REG_0] = {MIPS_R_V1, MIPS_R_V0}, |
| /* Arguments from eBPF program to in-kernel function */ |
| [BPF_REG_1] = {MIPS_R_A1, MIPS_R_A0}, |
| [BPF_REG_2] = {MIPS_R_A3, MIPS_R_A2}, |
| /* Remaining arguments, to be passed on the stack per O32 ABI */ |
| [BPF_REG_3] = {MIPS_R_T1, MIPS_R_T0}, |
| [BPF_REG_4] = {MIPS_R_T3, MIPS_R_T2}, |
| [BPF_REG_5] = {MIPS_R_T5, MIPS_R_T4}, |
| /* Callee-saved registers that in-kernel function will preserve */ |
| [BPF_REG_6] = {MIPS_R_S1, MIPS_R_S0}, |
| [BPF_REG_7] = {MIPS_R_S3, MIPS_R_S2}, |
| [BPF_REG_8] = {MIPS_R_S5, MIPS_R_S4}, |
| [BPF_REG_9] = {MIPS_R_S7, MIPS_R_S6}, |
| /* Read-only frame pointer to access the eBPF stack */ |
| #ifdef __BIG_ENDIAN |
| [BPF_REG_FP] = {MIPS_R_FP, MIPS_R_ZERO}, |
| #else |
| [BPF_REG_FP] = {MIPS_R_ZERO, MIPS_R_FP}, |
| #endif |
| /* Temporary register for blinding constants */ |
| [BPF_REG_AX] = {MIPS_R_GP, MIPS_R_AT}, |
| /* Temporary register for internal JIT use */ |
| [JIT_REG_TMP] = {MIPS_R_T7, MIPS_R_T6}, |
| }; |
| |
| /* Get low CPU register for a 64-bit eBPF register mapping */ |
| static inline u8 lo(const u8 reg[]) |
| { |
| #ifdef __BIG_ENDIAN |
| return reg[0]; |
| #else |
| return reg[1]; |
| #endif |
| } |
| |
| /* Get high CPU register for a 64-bit eBPF register mapping */ |
| static inline u8 hi(const u8 reg[]) |
| { |
| #ifdef __BIG_ENDIAN |
| return reg[1]; |
| #else |
| return reg[0]; |
| #endif |
| } |
| |
| /* |
| * Mark a 64-bit CPU register pair as clobbered, it needs to be |
| * saved/restored by the program if callee-saved. |
| */ |
| static void clobber_reg64(struct jit_context *ctx, const u8 reg[]) |
| { |
| clobber_reg(ctx, reg[0]); |
| clobber_reg(ctx, reg[1]); |
| } |
| |
| /* dst = imm (sign-extended) */ |
| static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm) |
| { |
| emit_mov_i(ctx, lo(dst), imm); |
| if (imm < 0) |
| emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); |
| else |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* Zero extension, if verifier does not do it for us */ |
| static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) |
| { |
| if (!ctx->program->aux->verifier_zext) { |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| clobber_reg(ctx, hi(dst)); |
| } |
| } |
| |
| /* Load delay slot, if ISA mandates it */ |
| static void emit_load_delay(struct jit_context *ctx) |
| { |
| if (!cpu_has_mips_2_3_4_5_r) |
| emit(ctx, nop); |
| } |
| |
| /* ALU immediate operation (64-bit) */ |
| static void emit_alu_i64(struct jit_context *ctx, |
| const u8 dst[], s32 imm, u8 op) |
| { |
| u8 src = MIPS_R_T6; |
| |
| /* |
| * ADD/SUB with all but the max negative imm can be handled by |
| * inverting the operation and the imm value, saving one insn. |
| */ |
| if (imm > S32_MIN && imm < 0) |
| switch (op) { |
| case BPF_ADD: |
| op = BPF_SUB; |
| imm = -imm; |
| break; |
| case BPF_SUB: |
| op = BPF_ADD; |
| imm = -imm; |
| break; |
| } |
| |
| /* Move immediate to temporary register */ |
| emit_mov_i(ctx, src, imm); |
| |
| switch (op) { |
| /* dst = dst + imm */ |
| case BPF_ADD: |
| emit(ctx, addu, lo(dst), lo(dst), src); |
| emit(ctx, sltu, MIPS_R_T9, lo(dst), src); |
| emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); |
| if (imm < 0) |
| emit(ctx, addiu, hi(dst), hi(dst), -1); |
| break; |
| /* dst = dst - imm */ |
| case BPF_SUB: |
| emit(ctx, sltu, MIPS_R_T9, lo(dst), src); |
| emit(ctx, subu, lo(dst), lo(dst), src); |
| emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); |
| if (imm < 0) |
| emit(ctx, addiu, hi(dst), hi(dst), 1); |
| break; |
| /* dst = dst | imm */ |
| case BPF_OR: |
| emit(ctx, or, lo(dst), lo(dst), src); |
| if (imm < 0) |
| emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); |
| break; |
| /* dst = dst & imm */ |
| case BPF_AND: |
| emit(ctx, and, lo(dst), lo(dst), src); |
| if (imm >= 0) |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| break; |
| /* dst = dst ^ imm */ |
| case BPF_XOR: |
| emit(ctx, xor, lo(dst), lo(dst), src); |
| if (imm < 0) { |
| emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); |
| emit(ctx, addiu, hi(dst), hi(dst), -1); |
| } |
| break; |
| } |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* ALU register operation (64-bit) */ |
| static void emit_alu_r64(struct jit_context *ctx, |
| const u8 dst[], const u8 src[], u8 op) |
| { |
| switch (BPF_OP(op)) { |
| /* dst = dst + src */ |
| case BPF_ADD: |
| if (src == dst) { |
| emit(ctx, srl, MIPS_R_T9, lo(dst), 31); |
| emit(ctx, addu, lo(dst), lo(dst), lo(dst)); |
| } else { |
| emit(ctx, addu, lo(dst), lo(dst), lo(src)); |
| emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); |
| } |
| emit(ctx, addu, hi(dst), hi(dst), hi(src)); |
| emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); |
| break; |
| /* dst = dst - src */ |
| case BPF_SUB: |
| emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); |
| emit(ctx, subu, lo(dst), lo(dst), lo(src)); |
| emit(ctx, subu, hi(dst), hi(dst), hi(src)); |
| emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); |
| break; |
| /* dst = dst | src */ |
| case BPF_OR: |
| emit(ctx, or, lo(dst), lo(dst), lo(src)); |
| emit(ctx, or, hi(dst), hi(dst), hi(src)); |
| break; |
| /* dst = dst & src */ |
| case BPF_AND: |
| emit(ctx, and, lo(dst), lo(dst), lo(src)); |
| emit(ctx, and, hi(dst), hi(dst), hi(src)); |
| break; |
| /* dst = dst ^ src */ |
| case BPF_XOR: |
| emit(ctx, xor, lo(dst), lo(dst), lo(src)); |
| emit(ctx, xor, hi(dst), hi(dst), hi(src)); |
| break; |
| } |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* ALU invert (64-bit) */ |
| static void emit_neg_i64(struct jit_context *ctx, const u8 dst[]) |
| { |
| emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst)); |
| emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst)); |
| emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); |
| emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); |
| |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* ALU shift immediate (64-bit) */ |
| static void emit_shift_i64(struct jit_context *ctx, |
| const u8 dst[], u32 imm, u8 op) |
| { |
| switch (BPF_OP(op)) { |
| /* dst = dst << imm */ |
| case BPF_LSH: |
| if (imm < 32) { |
| emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm); |
| emit(ctx, sll, lo(dst), lo(dst), imm); |
| emit(ctx, sll, hi(dst), hi(dst), imm); |
| emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9); |
| } else { |
| emit(ctx, sll, hi(dst), lo(dst), imm - 32); |
| emit(ctx, move, lo(dst), MIPS_R_ZERO); |
| } |
| break; |
| /* dst = dst >> imm */ |
| case BPF_RSH: |
| if (imm < 32) { |
| emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); |
| emit(ctx, srl, lo(dst), lo(dst), imm); |
| emit(ctx, srl, hi(dst), hi(dst), imm); |
| emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); |
| } else { |
| emit(ctx, srl, lo(dst), hi(dst), imm - 32); |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| } |
| break; |
| /* dst = dst >> imm (arithmetic) */ |
| case BPF_ARSH: |
| if (imm < 32) { |
| emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); |
| emit(ctx, srl, lo(dst), lo(dst), imm); |
| emit(ctx, sra, hi(dst), hi(dst), imm); |
| emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); |
| } else { |
| emit(ctx, sra, lo(dst), hi(dst), imm - 32); |
| emit(ctx, sra, hi(dst), hi(dst), 31); |
| } |
| break; |
| } |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* ALU shift register (64-bit) */ |
| static void emit_shift_r64(struct jit_context *ctx, |
| const u8 dst[], u8 src, u8 op) |
| { |
| u8 t1 = MIPS_R_T8; |
| u8 t2 = MIPS_R_T9; |
| |
| emit(ctx, andi, t1, src, 32); /* t1 = src & 32 */ |
| emit(ctx, beqz, t1, 16); /* PC += 16 if t1 == 0 */ |
| emit(ctx, nor, t2, src, MIPS_R_ZERO); /* t2 = ~src (delay slot) */ |
| |
| switch (BPF_OP(op)) { |
| /* dst = dst << src */ |
| case BPF_LSH: |
| /* Next: shift >= 32 */ |
| emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */ |
| emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */ |
| emit(ctx, b, 20); /* PC += 20 */ |
| /* +16: shift < 32 */ |
| emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */ |
| emit(ctx, srlv, t1, t1, t2); /* t1 = t1 >> t2 */ |
| emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */ |
| emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */ |
| emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */ |
| break; |
| /* dst = dst >> src */ |
| case BPF_RSH: |
| /* Next: shift >= 32 */ |
| emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */ |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */ |
| emit(ctx, b, 20); /* PC += 20 */ |
| /* +16: shift < 32 */ |
| emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ |
| emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ |
| emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */ |
| emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */ |
| emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ |
| break; |
| /* dst = dst >> src (arithmetic) */ |
| case BPF_ARSH: |
| /* Next: shift >= 32 */ |
| emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */ |
| emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */ |
| emit(ctx, b, 20); /* PC += 20 */ |
| /* +16: shift < 32 */ |
| emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ |
| emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ |
| emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */ |
| emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */ |
| emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ |
| break; |
| } |
| |
| /* +20: Done */ |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* ALU mul immediate (64x32-bit) */ |
| static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm) |
| { |
| u8 src = MIPS_R_T6; |
| u8 tmp = MIPS_R_T9; |
| |
| switch (imm) { |
| /* dst = dst * 1 is a no-op */ |
| case 1: |
| break; |
| /* dst = dst * -1 */ |
| case -1: |
| emit_neg_i64(ctx, dst); |
| break; |
| case 0: |
| emit_mov_r(ctx, lo(dst), MIPS_R_ZERO); |
| emit_mov_r(ctx, hi(dst), MIPS_R_ZERO); |
| break; |
| /* Full 64x32 multiply */ |
| default: |
| /* hi(dst) = hi(dst) * src(imm) */ |
| emit_mov_i(ctx, src, imm); |
| if (cpu_has_mips32r1 || cpu_has_mips32r6) { |
| emit(ctx, mul, hi(dst), hi(dst), src); |
| } else { |
| emit(ctx, multu, hi(dst), src); |
| emit(ctx, mflo, hi(dst)); |
| } |
| |
| /* hi(dst) = hi(dst) - lo(dst) */ |
| if (imm < 0) |
| emit(ctx, subu, hi(dst), hi(dst), lo(dst)); |
| |
| /* tmp = lo(dst) * src(imm) >> 32 */ |
| /* lo(dst) = lo(dst) * src(imm) */ |
| if (cpu_has_mips32r6) { |
| emit(ctx, muhu, tmp, lo(dst), src); |
| emit(ctx, mulu, lo(dst), lo(dst), src); |
| } else { |
| emit(ctx, multu, lo(dst), src); |
| emit(ctx, mflo, lo(dst)); |
| emit(ctx, mfhi, tmp); |
| } |
| |
| /* hi(dst) += tmp */ |
| emit(ctx, addu, hi(dst), hi(dst), tmp); |
| clobber_reg64(ctx, dst); |
| break; |
| } |
| } |
| |
| /* ALU mul register (64x64-bit) */ |
| static void emit_mul_r64(struct jit_context *ctx, |
| const u8 dst[], const u8 src[]) |
| { |
| u8 acc = MIPS_R_T8; |
| u8 tmp = MIPS_R_T9; |
| |
| /* acc = hi(dst) * lo(src) */ |
| if (cpu_has_mips32r1 || cpu_has_mips32r6) { |
| emit(ctx, mul, acc, hi(dst), lo(src)); |
| } else { |
| emit(ctx, multu, hi(dst), lo(src)); |
| emit(ctx, mflo, acc); |
| } |
| |
| /* tmp = lo(dst) * hi(src) */ |
| if (cpu_has_mips32r1 || cpu_has_mips32r6) { |
| emit(ctx, mul, tmp, lo(dst), hi(src)); |
| } else { |
| emit(ctx, multu, lo(dst), hi(src)); |
| emit(ctx, mflo, tmp); |
| } |
| |
| /* acc += tmp */ |
| emit(ctx, addu, acc, acc, tmp); |
| |
| /* tmp = lo(dst) * lo(src) >> 32 */ |
| /* lo(dst) = lo(dst) * lo(src) */ |
| if (cpu_has_mips32r6) { |
| emit(ctx, muhu, tmp, lo(dst), lo(src)); |
| emit(ctx, mulu, lo(dst), lo(dst), lo(src)); |
| } else { |
| emit(ctx, multu, lo(dst), lo(src)); |
| emit(ctx, mflo, lo(dst)); |
| emit(ctx, mfhi, tmp); |
| } |
| |
| /* hi(dst) = acc + tmp */ |
| emit(ctx, addu, hi(dst), acc, tmp); |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* Helper function for 64-bit modulo */ |
| static u64 jit_mod64(u64 a, u64 b) |
| { |
| u64 rem; |
| |
| div64_u64_rem(a, b, &rem); |
| return rem; |
| } |
| |
| /* ALU div/mod register (64-bit) */ |
| static void emit_divmod_r64(struct jit_context *ctx, |
| const u8 dst[], const u8 src[], u8 op) |
| { |
| const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ |
| const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ |
| const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ |
| int exclude, k; |
| u32 addr = 0; |
| |
| /* Push caller-saved registers on stack */ |
| push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| 0, JIT_RESERVED_STACK); |
| |
| /* Put 64-bit arguments 1 and 2 in registers a0-a3 */ |
| for (k = 0; k < 2; k++) { |
| emit(ctx, move, MIPS_R_T9, src[k]); |
| emit(ctx, move, r1[k], dst[k]); |
| emit(ctx, move, r2[k], MIPS_R_T9); |
| } |
| |
| /* Emit function call */ |
| switch (BPF_OP(op)) { |
| /* dst = dst / src */ |
| case BPF_DIV: |
| addr = (u32)&div64_u64; |
| break; |
| /* dst = dst % src */ |
| case BPF_MOD: |
| addr = (u32)&jit_mod64; |
| break; |
| } |
| emit_mov_i(ctx, MIPS_R_T9, addr); |
| emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); |
| emit(ctx, nop); /* Delay slot */ |
| |
| /* Store the 64-bit result in dst */ |
| emit(ctx, move, dst[0], r0[0]); |
| emit(ctx, move, dst[1], r0[1]); |
| |
| /* Restore caller-saved registers, excluding the computed result */ |
| exclude = BIT(lo(dst)) | BIT(hi(dst)); |
| pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| exclude, JIT_RESERVED_STACK); |
| emit_load_delay(ctx); |
| |
| clobber_reg64(ctx, dst); |
| clobber_reg(ctx, MIPS_R_V0); |
| clobber_reg(ctx, MIPS_R_V1); |
| clobber_reg(ctx, MIPS_R_RA); |
| } |
| |
| /* Swap bytes in a register word */ |
| static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask) |
| { |
| u8 tmp = MIPS_R_T9; |
| |
| emit(ctx, and, tmp, src, mask); /* tmp = src & 0x00ff00ff */ |
| emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ |
| emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */ |
| emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */ |
| emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ |
| } |
| |
| /* Swap half words in a register word */ |
| static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src) |
| { |
| u8 tmp = MIPS_R_T9; |
| |
| emit(ctx, sll, tmp, src, 16); /* tmp = src << 16 */ |
| emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */ |
| emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ |
| } |
| |
| /* Swap bytes and truncate a register double word, word or half word */ |
| static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width) |
| { |
| u8 tmp = MIPS_R_T8; |
| |
| switch (width) { |
| /* Swap bytes in a double word */ |
| case 64: |
| if (cpu_has_mips32r2 || cpu_has_mips32r6) { |
| emit(ctx, rotr, tmp, hi(dst), 16); |
| emit(ctx, rotr, hi(dst), lo(dst), 16); |
| emit(ctx, wsbh, lo(dst), tmp); |
| emit(ctx, wsbh, hi(dst), hi(dst)); |
| } else { |
| emit_swap16_r(ctx, tmp, lo(dst)); |
| emit_swap16_r(ctx, lo(dst), hi(dst)); |
| emit(ctx, move, hi(dst), tmp); |
| |
| emit(ctx, lui, tmp, 0xff); /* tmp = 0x00ff0000 */ |
| emit(ctx, ori, tmp, tmp, 0xff); /* tmp = 0x00ff00ff */ |
| emit_swap8_r(ctx, lo(dst), lo(dst), tmp); |
| emit_swap8_r(ctx, hi(dst), hi(dst), tmp); |
| } |
| break; |
| /* Swap bytes in a word */ |
| /* Swap bytes in a half word */ |
| case 32: |
| case 16: |
| emit_bswap_r(ctx, lo(dst), width); |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| break; |
| } |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* Truncate a register double word, word or half word */ |
| static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width) |
| { |
| switch (width) { |
| case 64: |
| break; |
| /* Zero-extend a word */ |
| case 32: |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| clobber_reg(ctx, hi(dst)); |
| break; |
| /* Zero-extend a half word */ |
| case 16: |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| emit(ctx, andi, lo(dst), lo(dst), 0xffff); |
| clobber_reg64(ctx, dst); |
| break; |
| } |
| } |
| |
| /* Load operation: dst = *(size*)(src + off) */ |
| static void emit_ldx(struct jit_context *ctx, |
| const u8 dst[], u8 src, s16 off, u8 size) |
| { |
| switch (size) { |
| /* Load a byte */ |
| case BPF_B: |
| emit(ctx, lbu, lo(dst), off, src); |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| break; |
| /* Load a half word */ |
| case BPF_H: |
| emit(ctx, lhu, lo(dst), off, src); |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| break; |
| /* Load a word */ |
| case BPF_W: |
| emit(ctx, lw, lo(dst), off, src); |
| emit(ctx, move, hi(dst), MIPS_R_ZERO); |
| break; |
| /* Load a double word */ |
| case BPF_DW: |
| if (dst[1] == src) { |
| emit(ctx, lw, dst[0], off + 4, src); |
| emit(ctx, lw, dst[1], off, src); |
| } else { |
| emit(ctx, lw, dst[1], off, src); |
| emit(ctx, lw, dst[0], off + 4, src); |
| } |
| emit_load_delay(ctx); |
| break; |
| } |
| clobber_reg64(ctx, dst); |
| } |
| |
| /* Store operation: *(size *)(dst + off) = src */ |
| static void emit_stx(struct jit_context *ctx, |
| const u8 dst, const u8 src[], s16 off, u8 size) |
| { |
| switch (size) { |
| /* Store a byte */ |
| case BPF_B: |
| emit(ctx, sb, lo(src), off, dst); |
| break; |
| /* Store a half word */ |
| case BPF_H: |
| emit(ctx, sh, lo(src), off, dst); |
| break; |
| /* Store a word */ |
| case BPF_W: |
| emit(ctx, sw, lo(src), off, dst); |
| break; |
| /* Store a double word */ |
| case BPF_DW: |
| emit(ctx, sw, src[1], off, dst); |
| emit(ctx, sw, src[0], off + 4, dst); |
| break; |
| } |
| } |
| |
| /* Atomic read-modify-write (32-bit, non-ll/sc fallback) */ |
| static void emit_atomic_r32(struct jit_context *ctx, |
| u8 dst, u8 src, s16 off, u8 code) |
| { |
| u32 exclude = 0; |
| u32 addr = 0; |
| |
| /* Push caller-saved registers on stack */ |
| push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| 0, JIT_RESERVED_STACK); |
| /* |
| * Argument 1: dst+off if xchg, otherwise src, passed in register a0 |
| * Argument 2: src if xchg, otherwise dst+off, passed in register a1 |
| */ |
| emit(ctx, move, MIPS_R_T9, dst); |
| if (code == BPF_XCHG) { |
| emit(ctx, move, MIPS_R_A1, src); |
| emit(ctx, addiu, MIPS_R_A0, MIPS_R_T9, off); |
| } else { |
| emit(ctx, move, MIPS_R_A0, src); |
| emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off); |
| } |
| |
| /* Emit function call */ |
| switch (code) { |
| case BPF_ADD: |
| addr = (u32)&atomic_add; |
| break; |
| case BPF_ADD | BPF_FETCH: |
| addr = (u32)&atomic_fetch_add; |
| break; |
| case BPF_SUB: |
| addr = (u32)&atomic_sub; |
| break; |
| case BPF_SUB | BPF_FETCH: |
| addr = (u32)&atomic_fetch_sub; |
| break; |
| case BPF_OR: |
| addr = (u32)&atomic_or; |
| break; |
| case BPF_OR | BPF_FETCH: |
| addr = (u32)&atomic_fetch_or; |
| break; |
| case BPF_AND: |
| addr = (u32)&atomic_and; |
| break; |
| case BPF_AND | BPF_FETCH: |
| addr = (u32)&atomic_fetch_and; |
| break; |
| case BPF_XOR: |
| addr = (u32)&atomic_xor; |
| break; |
| case BPF_XOR | BPF_FETCH: |
| addr = (u32)&atomic_fetch_xor; |
| break; |
| case BPF_XCHG: |
| addr = (u32)&atomic_xchg; |
| break; |
| } |
| emit_mov_i(ctx, MIPS_R_T9, addr); |
| emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); |
| emit(ctx, nop); /* Delay slot */ |
| |
| /* Update src register with old value, if specified */ |
| if (code & BPF_FETCH) { |
| emit(ctx, move, src, MIPS_R_V0); |
| exclude = BIT(src); |
| clobber_reg(ctx, src); |
| } |
| |
| /* Restore caller-saved registers, except any fetched value */ |
| pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| exclude, JIT_RESERVED_STACK); |
| emit_load_delay(ctx); |
| clobber_reg(ctx, MIPS_R_RA); |
| } |
| |
| /* Helper function for 64-bit atomic exchange */ |
| static s64 jit_xchg64(s64 a, atomic64_t *v) |
| { |
| return atomic64_xchg(v, a); |
| } |
| |
| /* Atomic read-modify-write (64-bit) */ |
| static void emit_atomic_r64(struct jit_context *ctx, |
| u8 dst, const u8 src[], s16 off, u8 code) |
| { |
| const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ |
| const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ |
| u32 exclude = 0; |
| u32 addr = 0; |
| |
| /* Push caller-saved registers on stack */ |
| push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| 0, JIT_RESERVED_STACK); |
| /* |
| * Argument 1: 64-bit src, passed in registers a0-a1 |
| * Argument 2: 32-bit dst+off, passed in register a2 |
| */ |
| emit(ctx, move, MIPS_R_T9, dst); |
| emit(ctx, move, r1[0], src[0]); |
| emit(ctx, move, r1[1], src[1]); |
| emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off); |
| |
| /* Emit function call */ |
| switch (code) { |
| case BPF_ADD: |
| addr = (u32)&atomic64_add; |
| break; |
| case BPF_ADD | BPF_FETCH: |
| addr = (u32)&atomic64_fetch_add; |
| break; |
| case BPF_SUB: |
| addr = (u32)&atomic64_sub; |
| break; |
| case BPF_SUB | BPF_FETCH: |
| addr = (u32)&atomic64_fetch_sub; |
| break; |
| case BPF_OR: |
| addr = (u32)&atomic64_or; |
| break; |
| case BPF_OR | BPF_FETCH: |
| addr = (u32)&atomic64_fetch_or; |
| break; |
| case BPF_AND: |
| addr = (u32)&atomic64_and; |
| break; |
| case BPF_AND | BPF_FETCH: |
| addr = (u32)&atomic64_fetch_and; |
| break; |
| case BPF_XOR: |
| addr = (u32)&atomic64_xor; |
| break; |
| case BPF_XOR | BPF_FETCH: |
| addr = (u32)&atomic64_fetch_xor; |
| break; |
| case BPF_XCHG: |
| addr = (u32)&jit_xchg64; |
| break; |
| } |
| emit_mov_i(ctx, MIPS_R_T9, addr); |
| emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); |
| emit(ctx, nop); /* Delay slot */ |
| |
| /* Update src register with old value, if specified */ |
| if (code & BPF_FETCH) { |
| emit(ctx, move, lo(src), lo(r0)); |
| emit(ctx, move, hi(src), hi(r0)); |
| exclude = BIT(src[0]) | BIT(src[1]); |
| clobber_reg64(ctx, src); |
| } |
| |
| /* Restore caller-saved registers, except any fetched value */ |
| pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| exclude, JIT_RESERVED_STACK); |
| emit_load_delay(ctx); |
| clobber_reg(ctx, MIPS_R_RA); |
| } |
| |
| /* Atomic compare-and-exchange (32-bit, non-ll/sc fallback) */ |
| static void emit_cmpxchg_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off) |
| { |
| const u8 *r0 = bpf2mips32[BPF_REG_0]; |
| |
| /* Push caller-saved registers on stack */ |
| push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32)); |
| /* |
| * Argument 1: 32-bit dst+off, passed in register a0 |
| * Argument 2: 32-bit r0, passed in register a1 |
| * Argument 3: 32-bit src, passed in register a2 |
| */ |
| emit(ctx, addiu, MIPS_R_T9, dst, off); |
| emit(ctx, move, MIPS_R_T8, src); |
| emit(ctx, move, MIPS_R_A1, lo(r0)); |
| emit(ctx, move, MIPS_R_A0, MIPS_R_T9); |
| emit(ctx, move, MIPS_R_A2, MIPS_R_T8); |
| |
| /* Emit function call */ |
| emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic_cmpxchg); |
| emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); |
| emit(ctx, nop); /* Delay slot */ |
| |
| #ifdef __BIG_ENDIAN |
| emit(ctx, move, lo(r0), MIPS_R_V0); |
| #endif |
| /* Restore caller-saved registers, except the return value */ |
| pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32)); |
| emit_load_delay(ctx); |
| clobber_reg(ctx, MIPS_R_V0); |
| clobber_reg(ctx, MIPS_R_V1); |
| clobber_reg(ctx, MIPS_R_RA); |
| } |
| |
| /* Atomic compare-and-exchange (64-bit) */ |
| static void emit_cmpxchg_r64(struct jit_context *ctx, |
| u8 dst, const u8 src[], s16 off) |
| { |
| const u8 *r0 = bpf2mips32[BPF_REG_0]; |
| const u8 *r2 = bpf2mips32[BPF_REG_2]; |
| |
| /* Push caller-saved registers on stack */ |
| push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32)); |
| /* |
| * Argument 1: 32-bit dst+off, passed in register a0 (a1 unused) |
| * Argument 2: 64-bit r0, passed in registers a2-a3 |
| * Argument 3: 64-bit src, passed on stack |
| */ |
| push_regs(ctx, BIT(src[0]) | BIT(src[1]), 0, JIT_RESERVED_STACK); |
| emit(ctx, addiu, MIPS_R_T9, dst, off); |
| emit(ctx, move, r2[0], r0[0]); |
| emit(ctx, move, r2[1], r0[1]); |
| emit(ctx, move, MIPS_R_A0, MIPS_R_T9); |
| |
| /* Emit function call */ |
| emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic64_cmpxchg); |
| emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); |
| emit(ctx, nop); /* Delay slot */ |
| |
| /* Restore caller-saved registers, except the return value */ |
| pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, |
| JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32)); |
| emit_load_delay(ctx); |
| clobber_reg(ctx, MIPS_R_V0); |
| clobber_reg(ctx, MIPS_R_V1); |
| clobber_reg(ctx, MIPS_R_RA); |
| } |
| |
| /* |
| * Conditional movz or an emulated equivalent. |
| * Note that the rs register may be modified. |
| */ |
| static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) |
| { |
| if (cpu_has_mips_2) { |
| emit(ctx, movz, rd, rs, rt); /* rd = rt ? rd : rs */ |
| } else if (cpu_has_mips32r6) { |
| if (rs != MIPS_R_ZERO) |
| emit(ctx, seleqz, rs, rs, rt); /* rs = 0 if rt == 0 */ |
| emit(ctx, selnez, rd, rd, rt); /* rd = 0 if rt != 0 */ |
| if (rs != MIPS_R_ZERO) |
| emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ |
| } else { |
| emit(ctx, bnez, rt, 8); /* PC += 8 if rd != 0 */ |
| emit(ctx, nop); /* +0: delay slot */ |
| emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ |
| } |
| clobber_reg(ctx, rd); |
| clobber_reg(ctx, rs); |
| } |
| |
| /* |
| * Conditional movn or an emulated equivalent. |
| * Note that the rs register may be modified. |
| */ |
| static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) |
| { |
| if (cpu_has_mips_2) { |
| emit(ctx, movn, rd, rs, rt); /* rd = rt ? rs : rd */ |
| } else if (cpu_has_mips32r6) { |
| if (rs != MIPS_R_ZERO) |
| emit(ctx, selnez, rs, rs, rt); /* rs = 0 if rt == 0 */ |
| emit(ctx, seleqz, rd, rd, rt); /* rd = 0 if rt != 0 */ |
| if (rs != MIPS_R_ZERO) |
| emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ |
| } else { |
| emit(ctx, beqz, rt, 8); /* PC += 8 if rd == 0 */ |
| emit(ctx, nop); /* +0: delay slot */ |
| emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ |
| } |
| clobber_reg(ctx, rd); |
| clobber_reg(ctx, rs); |
| } |
| |
| /* Emulation of 64-bit sltiu rd, rs, imm, where imm may be S32_MAX + 1 */ |
| static void emit_sltiu_r64(struct jit_context *ctx, u8 rd, |
| const u8 rs[], s64 imm) |
| { |
| u8 tmp = MIPS_R_T9; |
| |
| if (imm < 0) { |
| emit_mov_i(ctx, rd, imm); /* rd = imm */ |
| emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ |
| emit(ctx, sltiu, tmp, hi(rs), -1); /* tmp = rsh < ~0U */ |
| emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ |
| } else { /* imm >= 0 */ |
| if (imm > 0x7fff) { |
| emit_mov_i(ctx, rd, (s32)imm); /* rd = imm */ |
| emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ |
| } else { |
| emit(ctx, sltiu, rd, lo(rs), imm); /* rd = rsl < imm */ |
| } |
| emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs)); /* rd = 0 if rsh */ |
| } |
| } |
| |
| /* Emulation of 64-bit sltu rd, rs, rt */ |
| static void emit_sltu_r64(struct jit_context *ctx, u8 rd, |
| const u8 rs[], const u8 rt[]) |
| { |
| u8 tmp = MIPS_R_T9; |
| |
| emit(ctx, sltu, rd, lo(rs), lo(rt)); /* rd = rsl < rtl */ |
| emit(ctx, subu, tmp, hi(rs), hi(rt)); /* tmp = rsh - rth */ |
| emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp); /* rd = 0 if tmp != 0 */ |
| emit(ctx, sltu, tmp, hi(rs), hi(rt)); /* tmp = rsh < rth */ |
| emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ |
| } |
| |
| /* Emulation of 64-bit slti rd, rs, imm, where imm may be S32_MAX + 1 */ |
| static void emit_slti_r64(struct jit_context *ctx, u8 rd, |
| const u8 rs[], s64 imm) |
| { |
| u8 t1 = MIPS_R_T8; |
| u8 t2 = MIPS_R_T9; |
| u8 cmp; |
| |
| /* |
| * if ((rs < 0) ^ (imm < 0)) t1 = imm >u rsl |
| * else t1 = rsl <u imm |
| */ |
| emit_mov_i(ctx, rd, (s32)imm); |
| emit(ctx, sltu, t1, lo(rs), rd); /* t1 = rsl <u imm */ |
| emit(ctx, sltu, t2, rd, lo(rs)); /* t2 = imm <u rsl */ |
| emit(ctx, srl, rd, hi(rs), 31); /* rd = rsh >> 31 */ |
| if (imm < 0) |
| emit_movz_r(ctx, t1, t2, rd); /* t1 = rd ? t1 : t2 */ |
| else |
| emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ |
| /* |
| * if ((imm < 0 && rsh != 0xffffffff) || |
| * (imm >= 0 && rsh != 0)) |
| * t1 = 0 |
| */ |
| if (imm < 0) { |
| emit(ctx, addiu, rd, hi(rs), 1); /* rd = rsh + 1 */ |
| cmp = rd; |
| } else { /* imm >= 0 */ |
| cmp = hi(rs); |
| } |
| emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp); /* t1 = 0 if cmp != 0 */ |
| |
| /* |
| * if (imm < 0) rd = rsh < -1 |
| * else rd = rsh != 0 |
| * rd = rd | t1 |
| */ |
| emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0); /* rd = rsh < hi(imm) */ |
| emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ |
| } |
| |
| /* Emulation of 64-bit(slt rd, rs, rt) */ |
| static void emit_slt_r64(struct jit_context *ctx, u8 rd, |
| const u8 rs[], const u8 rt[]) |
| { |
| u8 t1 = MIPS_R_T7; |
| u8 t2 = MIPS_R_T8; |
| u8 t3 = MIPS_R_T9; |
| |
| /* |
| * if ((rs < 0) ^ (rt < 0)) t1 = rtl <u rsl |
| * else t1 = rsl <u rtl |
| * if (rsh == rth) t1 = 0 |
| */ |
| emit(ctx, sltu, t1, lo(rs), lo(rt)); /* t1 = rsl <u rtl */ |
| emit(ctx, sltu, t2, lo(rt), lo(rs)); /* t2 = rtl <u rsl */ |
| emit(ctx, xor, t3, hi(rs), hi(rt)); /* t3 = rlh ^ rth */ |
| emit(ctx, srl, rd, t3, 31); /* rd = t3 >> 31 */ |
| emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ |
| emit_movn_r(ctx, t1, MIPS_R_ZERO, t3); /* t1 = 0 if t3 != 0 */ |
| |
| /* rd = (rsh < rth) | t1 */ |
| emit(ctx, slt, rd, hi(rs), hi(rt)); /* rd = rsh <s rth */ |
| emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ |
| } |
| |
| /* Jump immediate (64-bit) */ |
| static void emit_jmp_i64(struct jit_context *ctx, |
| const u8 dst[], s32 imm, s32 off, u8 op) |
| { |
| u8 tmp = MIPS_R_T6; |
| |
| switch (op) { |
| /* No-op, used internally for branch optimization */ |
| case JIT_JNOP: |
| break; |
| /* PC += off if dst == imm */ |
| /* PC += off if dst != imm */ |
| case BPF_JEQ: |
| case BPF_JNE: |
| if (imm >= -0x7fff && imm <= 0x8000) { |
| emit(ctx, addiu, tmp, lo(dst), -imm); |
| } else if ((u32)imm <= 0xffff) { |
| emit(ctx, xori, tmp, lo(dst), imm); |
| } else { /* Register fallback */ |
| emit_mov_i(ctx, tmp, imm); |
| emit(ctx, xor, tmp, lo(dst), tmp); |
| } |
| if (imm < 0) { /* Compare sign extension */ |
| emit(ctx, addu, MIPS_R_T9, hi(dst), 1); |
| emit(ctx, or, tmp, tmp, MIPS_R_T9); |
| } else { /* Compare zero extension */ |
| emit(ctx, or, tmp, tmp, hi(dst)); |
| } |
| if (op == BPF_JEQ) |
| emit(ctx, beqz, tmp, off); |
| else /* BPF_JNE */ |
| emit(ctx, bnez, tmp, off); |
| break; |
| /* PC += off if dst & imm */ |
| /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ |
| case BPF_JSET: |
| case JIT_JNSET: |
| if ((u32)imm <= 0xffff) { |
| emit(ctx, andi, tmp, lo(dst), imm); |
| } else { /* Register fallback */ |
| emit_mov_i(ctx, tmp, imm); |
| emit(ctx, and, tmp, lo(dst), tmp); |
| } |
| if (imm < 0) /* Sign-extension pulls in high word */ |
| emit(ctx, or, tmp, tmp, hi(dst)); |
| if (op == BPF_JSET) |
| emit(ctx, bnez, tmp, off); |
| else /* JIT_JNSET */ |
| emit(ctx, beqz, tmp, off); |
| break; |
| /* PC += off if dst > imm */ |
| case BPF_JGT: |
| emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); |
| emit(ctx, beqz, tmp, off); |
| break; |
| /* PC += off if dst >= imm */ |
| case BPF_JGE: |
| emit_sltiu_r64(ctx, tmp, dst, imm); |
| emit(ctx, beqz, tmp, off); |
| break; |
| /* PC += off if dst < imm */ |
| case BPF_JLT: |
| emit_sltiu_r64(ctx, tmp, dst, imm); |
| emit(ctx, bnez, tmp, off); |
| break; |
| /* PC += off if dst <= imm */ |
| case BPF_JLE: |
| emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); |
| emit(ctx, bnez, tmp, off); |
| break; |
| /* PC += off if dst > imm (signed) */ |
| case BPF_JSGT: |
| emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); |
| emit(ctx, beqz, tmp, off); |
| break; |
| /* PC += off if dst >= imm (signed) */ |
| case BPF_JSGE: |
| emit_slti_r64(ctx, tmp, dst, imm); |
| emit(ctx, beqz, tmp, off); |
| break; |
| /* PC += off if dst < imm (signed) */ |
| case BPF_JSLT: |
| emit_slti_r64(ctx, tmp, dst, imm); |
| emit(ctx, bnez, tmp, off); |
| break; |
| /* PC += off if dst <= imm (signed) */ |
| case BPF_JSLE: |
| emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); |
| emit(ctx, bnez, tmp, off); |
| break; |
| } |
| } |
| |
| /* Jump register (64-bit) */ |
| static void emit_jmp_r64(struct jit_context *ctx, |
| const u8 dst[], const u8 src[], s32 off, u8 op) |
| { |
| u8 t1 = MIPS_R_T6; |
| u8 t2 = MIPS_R_T7; |
| |
| switch (op) { |
| /* No-op, used internally for branch optimization */ |
| case JIT_JNOP: |
| break; |
| /* PC += off if dst == src */ |
| /* PC += off if dst != src */ |
| case BPF_JEQ: |
| case BPF_JNE: |
| emit(ctx, subu, t1, lo(dst), lo(src)); |
| emit(ctx, subu, t2, hi(dst), hi(src)); |
| emit(ctx, or, t1, t1, t2); |
| if (op == BPF_JEQ) |
| emit(ctx, beqz, t1, off); |
| else /* BPF_JNE */ |
| emit(ctx, bnez, t1, off); |
| break; |
| /* PC += off if dst & src */ |
| /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ |
| case BPF_JSET: |
| case JIT_JNSET: |
| emit(ctx, and, t1, lo(dst), lo(src)); |
| emit(ctx, and, t2, hi(dst), hi(src)); |
| emit(ctx, or, t1, t1, t2); |
| if (op == BPF_JSET) |
| emit(ctx, bnez, t1, off); |
| else /* JIT_JNSET */ |
| emit(ctx, beqz, t1, off); |
| break; |
| /* PC += off if dst > src */ |
| case BPF_JGT: |
| emit_sltu_r64(ctx, t1, src, dst); |
| emit(ctx, bnez, t1, off); |
| break; |
| /* PC += off if dst >= src */ |
| case BPF_JGE: |
| emit_sltu_r64(ctx, t1, dst, src); |
| emit(ctx, beqz, t1, off); |
| break; |
| /* PC += off if dst < src */ |
| case BPF_JLT: |
| emit_sltu_r64(ctx, t1, dst, src); |
| emit(ctx, bnez, t1, off); |
| break; |
| /* PC += off if dst <= src */ |
| case BPF_JLE: |
| emit_sltu_r64(ctx, t1, src, dst); |
| emit(ctx, beqz, t1, off); |
| break; |
| /* PC += off if dst > src (signed) */ |
| case BPF_JSGT: |
| emit_slt_r64(ctx, t1, src, dst); |
| emit(ctx, bnez, t1, off); |
| break; |
| /* PC += off if dst >= src (signed) */ |
| case BPF_JSGE: |
| emit_slt_r64(ctx, t1, dst, src); |
| emit(ctx, beqz, t1, off); |
| break; |
| /* PC += off if dst < src (signed) */ |
| case BPF_JSLT: |
| emit_slt_r64(ctx, t1, dst, src); |
| emit(ctx, bnez, t1, off); |
| break; |
| /* PC += off if dst <= src (signed) */ |
| case BPF_JSLE: |
| emit_slt_r64(ctx, t1, src, dst); |
| emit(ctx, beqz, t1, off); |
| break; |
| } |
| } |
| |
| /* Function call */ |
| static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) |
| { |
| bool fixed; |
| u64 addr; |
| |
| /* Decode the call address */ |
| if (bpf_jit_get_func_addr(ctx->program, insn, false, |
| &addr, &fixed) < 0) |
| return -1; |
| if (!fixed) |
| return -1; |
| |
| /* Push stack arguments */ |
| push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK); |
| |
| /* Emit function call */ |
| emit_mov_i(ctx, MIPS_R_T9, addr); |
| emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); |
| emit(ctx, nop); /* Delay slot */ |
| |
| clobber_reg(ctx, MIPS_R_RA); |
| clobber_reg(ctx, MIPS_R_V0); |
| clobber_reg(ctx, MIPS_R_V1); |
| return 0; |
| } |
| |
| /* Function tail call */ |
| static int emit_tail_call(struct jit_context *ctx) |
| { |
| u8 ary = lo(bpf2mips32[BPF_REG_2]); |
| u8 ind = lo(bpf2mips32[BPF_REG_3]); |
| u8 t1 = MIPS_R_T8; |
| u8 t2 = MIPS_R_T9; |
| int off; |
| |
| /* |
| * Tail call: |
| * eBPF R1 - function argument (context ptr), passed in a0-a1 |
| * eBPF R2 - ptr to object with array of function entry points |
| * eBPF R3 - array index of function to be called |
| * stack[sz] - remaining tail call count, initialized in prologue |
| */ |
| |
| /* if (ind >= ary->map.max_entries) goto out */ |
| off = offsetof(struct bpf_array, map.max_entries); |
| if (off > 0x7fff) |
| return -1; |
| emit(ctx, lw, t1, off, ary); /* t1 = ary->map.max_entries*/ |
| emit_load_delay(ctx); /* Load delay slot */ |
| emit(ctx, sltu, t1, ind, t1); /* t1 = ind < t1 */ |
| emit(ctx, beqz, t1, get_offset(ctx, 1)); /* PC += off(1) if t1 == 0 */ |
| /* (next insn delay slot) */ |
| /* if (TCC-- <= 0) goto out */ |
| emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP); /* t2 = *(SP + size) */ |
| emit_load_delay(ctx); /* Load delay slot */ |
| emit(ctx, blez, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 <= 0 */ |
| emit(ctx, addiu, t2, t2, -1); /* t2-- (delay slot) */ |
| emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP); /* *(SP + size) = t2 */ |
| |
| /* prog = ary->ptrs[ind] */ |
| off = offsetof(struct bpf_array, ptrs); |
| if (off > 0x7fff) |
| return -1; |
| emit(ctx, sll, t1, ind, 2); /* t1 = ind << 2 */ |
| emit(ctx, addu, t1, t1, ary); /* t1 += ary */ |
| emit(ctx, lw, t2, off, t1); /* t2 = *(t1 + off) */ |
| emit_load_delay(ctx); /* Load delay slot */ |
| |
| /* if (prog == 0) goto out */ |
| emit(ctx, beqz, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 == 0 */ |
| emit(ctx, nop); /* Delay slot */ |
| |
| /* func = prog->bpf_func + 8 (prologue skip offset) */ |
| off = offsetof(struct bpf_prog, bpf_func); |
| if (off > 0x7fff) |
| return -1; |
| emit(ctx, lw, t1, off, t2); /* t1 = *(t2 + off) */ |
| emit_load_delay(ctx); /* Load delay slot */ |
| emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP); /* t1 += skip (8 or 12) */ |
| |
| /* goto func */ |
| build_epilogue(ctx, t1); |
| return 0; |
| } |
| |
| /* |
| * Stack frame layout for a JITed program (stack grows down). |
| * |
| * Higher address : Caller's stack frame : |
| * :----------------------------: |
| * : 64-bit eBPF args r3-r5 : |
| * :----------------------------: |
| * : Reserved / tail call count : |
| * +============================+ <--- MIPS sp before call |
| * | Callee-saved registers, | |
| * | including RA and FP | |
| * +----------------------------+ <--- eBPF FP (MIPS zero,fp) |
| * | Local eBPF variables | |
| * | allocated by program | |
| * +----------------------------+ |
| * | Reserved for caller-saved | |
| * | registers | |
| * +----------------------------+ |
| * | Reserved for 64-bit eBPF | |
| * | args r3-r5 & args passed | |
| * | on stack in kernel calls | |
| * Lower address +============================+ <--- MIPS sp |
| */ |
| |
| /* Build program prologue to set up the stack and registers */ |
| void build_prologue(struct jit_context *ctx) |
| { |
| const u8 *r1 = bpf2mips32[BPF_REG_1]; |
| const u8 *fp = bpf2mips32[BPF_REG_FP]; |
| int stack, saved, locals, reserved; |
| |
| /* |
| * In the unlikely event that the TCC limit is raised to more |
| * than 16 bits, it is clamped to the maximum value allowed for |
| * the generated code (0xffff). It is better fail to compile |
| * instead of degrading gracefully. |
| */ |
| BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff); |
| |
| /* |
| * The first two instructions initialize TCC in the reserved (for us) |
| * 16-byte area in the parent's stack frame. On a tail call, the |
| * calling function jumps into the prologue after these instructions. |
| */ |
| emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, MAX_TAIL_CALL_CNT); |
| emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); |
| |
| /* |
| * Register eBPF R1 contains the 32-bit context pointer argument. |
| * A 32-bit argument is always passed in MIPS register a0, regardless |
| * of CPU endianness. Initialize R1 accordingly and zero-extend. |
| */ |
| #ifdef __BIG_ENDIAN |
| emit(ctx, move, lo(r1), MIPS_R_A0); |
| #endif |
| |
| /* === Entry-point for tail calls === */ |
| |
| /* Zero-extend the 32-bit argument */ |
| emit(ctx, move, hi(r1), MIPS_R_ZERO); |
| |
| /* If the eBPF frame pointer was accessed it must be saved */ |
| if (ctx->accessed & BIT(BPF_REG_FP)) |
| clobber_reg64(ctx, fp); |
| |
| /* Compute the stack space needed for callee-saved registers */ |
| saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32); |
| saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); |
| |
| /* Stack space used by eBPF program local data */ |
| locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); |
| |
| /* |
| * If we are emitting function calls, reserve extra stack space for |
| * caller-saved registers and function arguments passed on the stack. |
| * The required space is computed automatically during resource |
| * usage discovery (pass 1). |
| */ |
| reserved = ctx->stack_used; |
| |
| /* Allocate the stack frame */ |
| stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); |
| emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack); |
| |
| /* Store callee-saved registers on stack */ |
| push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); |
| |
| /* Initialize the eBPF frame pointer if accessed */ |
| if (ctx->accessed & BIT(BPF_REG_FP)) |
| emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved); |
| |
| ctx->saved_size = saved; |
| ctx->stack_size = stack; |
| } |
| |
| /* Build the program epilogue to restore the stack and registers */ |
| void build_epilogue(struct jit_context *ctx, int dest_reg) |
| { |
| /* Restore callee-saved registers from stack */ |
| pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, |
| ctx->stack_size - ctx->saved_size); |
| /* |
| * A 32-bit return value is always passed in MIPS register v0, |
| * but on big-endian targets the low part of R0 is mapped to v1. |
| */ |
| #ifdef __BIG_ENDIAN |
| emit(ctx, move, MIPS_R_V0, MIPS_R_V1); |
| #endif |
| |
| /* Jump to the return address and adjust the stack pointer */ |
| emit(ctx, jr, dest_reg); |
| emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); |
| } |
| |
| /* Build one eBPF instruction */ |
| int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) |
| { |
| const u8 *dst = bpf2mips32[insn->dst_reg]; |
| const u8 *src = bpf2mips32[insn->src_reg]; |
| const u8 *res = bpf2mips32[BPF_REG_0]; |
| const u8 *tmp = bpf2mips32[JIT_REG_TMP]; |
| u8 code = insn->code; |
| s16 off = insn->off; |
| s32 imm = insn->imm; |
| s32 val, rel; |
| u8 alu, jmp; |
| |
| switch (code) { |
| /* ALU operations */ |
| /* dst = imm */ |
| case BPF_ALU | BPF_MOV | BPF_K: |
| emit_mov_i(ctx, lo(dst), imm); |
| emit_zext_ver(ctx, dst); |
| break; |
| /* dst = src */ |
| case BPF_ALU | BPF_MOV | BPF_X: |
| if (imm == 1) { |
| /* Special mov32 for zext */ |
| emit_mov_i(ctx, hi(dst), 0); |
| } else { |
| emit_mov_r(ctx, lo(dst), lo(src)); |
| emit_zext_ver(ctx, dst); |
| } |
| break; |
| /* dst = -dst */ |
| case BPF_ALU | BPF_NEG: |
| emit_alu_i(ctx, lo(dst), 0, BPF_NEG); |
| emit_zext_ver(ctx, dst); |
| break; |
| /* dst = dst & imm */ |
| /* dst = dst | imm */ |
| /* dst = dst ^ imm */ |
| /* dst = dst << imm */ |
| /* dst = dst >> imm */ |
| /* dst = dst >> imm (arithmetic) */ |
| /* dst = dst + imm */ |
| /* dst = dst - imm */ |
| /* dst = dst * imm */ |
| /* dst = dst / imm */ |
| /* dst = dst % imm */ |
| case BPF_ALU | BPF_OR | BPF_K: |
| case BPF_ALU | BPF_AND | BPF_K: |
| case BPF_ALU | BPF_XOR | BPF_K: |
| case BPF_ALU | BPF_LSH | BPF_K: |
| case BPF_ALU | BPF_RSH | BPF_K: |
| case BPF_ALU | BPF_ARSH | BPF_K: |
| case BPF_ALU | BPF_ADD | BPF_K: |
| case BPF_ALU | BPF_SUB | BPF_K: |
| case BPF_ALU | BPF_MUL | BPF_K: |
| case BPF_ALU | BPF_DIV | BPF_K: |
| case BPF_ALU | BPF_MOD | BPF_K: |
| if (!valid_alu_i(BPF_OP(code), imm)) { |
| emit_mov_i(ctx, MIPS_R_T6, imm); |
| emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); |
| } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { |
| emit_alu_i(ctx, lo(dst), val, alu); |
| } |
| emit_zext_ver(ctx, dst); |
| break; |
| /* dst = dst & src */ |
| /* dst = dst | src */ |
| /* dst = dst ^ src */ |
| /* dst = dst << src */ |
| /* dst = dst >> src */ |
| /* dst = dst >> src (arithmetic) */ |
| /* dst = dst + src */ |
| /* dst = dst - src */ |
| /* dst = dst * src */ |
| /* dst = dst / src */ |
| /* dst = dst % src */ |
| case BPF_ALU | BPF_AND | BPF_X: |
| case BPF_ALU | BPF_OR | BPF_X: |
| case BPF_ALU | BPF_XOR | BPF_X: |
| case BPF_ALU | BPF_LSH | BPF_X: |
| case BPF_ALU | BPF_RSH | BPF_X: |
| case BPF_ALU | BPF_ARSH | BPF_X: |
| case BPF_ALU | BPF_ADD | BPF_X: |
| case BPF_ALU | BPF_SUB | BPF_X: |
| case BPF_ALU | BPF_MUL | BPF_X: |
| case BPF_ALU | BPF_DIV | BPF_X: |
| case BPF_ALU | BPF_MOD | BPF_X: |
| emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); |
| emit_zext_ver(ctx, dst); |
| break; |
| /* dst = imm (64-bit) */ |
| case BPF_ALU64 | BPF_MOV | BPF_K: |
| emit_mov_se_i64(ctx, dst, imm); |
| break; |
| /* dst = src (64-bit) */ |
| case BPF_ALU64 | BPF_MOV | BPF_X: |
| emit_mov_r(ctx, lo(dst), lo(src)); |
| emit_mov_r(ctx, hi(dst), hi(src)); |
| break; |
| /* dst = -dst (64-bit) */ |
| case BPF_ALU64 | BPF_NEG: |
| emit_neg_i64(ctx, dst); |
| break; |
| /* dst = dst & imm (64-bit) */ |
| case BPF_ALU64 | BPF_AND | BPF_K: |
| emit_alu_i64(ctx, dst, imm, BPF_OP(code)); |
| break; |
| /* dst = dst | imm (64-bit) */ |
| /* dst = dst ^ imm (64-bit) */ |
| /* dst = dst + imm (64-bit) */ |
| /* dst = dst - imm (64-bit) */ |
| case BPF_ALU64 | BPF_OR | BPF_K: |
| case BPF_ALU64 | BPF_XOR | BPF_K: |
| case BPF_ALU64 | BPF_ADD | BPF_K: |
| case BPF_ALU64 | BPF_SUB | BPF_K: |
| if (imm) |
| emit_alu_i64(ctx, dst, imm, BPF_OP(code)); |
| break; |
| /* dst = dst << imm (64-bit) */ |
| /* dst = dst >> imm (64-bit) */ |
| /* dst = dst >> imm (64-bit, arithmetic) */ |
| case BPF_ALU64 | BPF_LSH | BPF_K: |
| case BPF_ALU64 | BPF_RSH | BPF_K: |
| case BPF_ALU64 | BPF_ARSH | BPF_K: |
| if (imm) |
| emit_shift_i64(ctx, dst, imm, BPF_OP(code)); |
| break; |
| /* dst = dst * imm (64-bit) */ |
| case BPF_ALU64 | BPF_MUL | BPF_K: |
| emit_mul_i64(ctx, dst, imm); |
| break; |
| /* dst = dst / imm (64-bit) */ |
| /* dst = dst % imm (64-bit) */ |
| case BPF_ALU64 | BPF_DIV | BPF_K: |
| case BPF_ALU64 | BPF_MOD | BPF_K: |
| /* |
| * Sign-extend the immediate value into a temporary register, |
| * and then do the operation on this register. |
| */ |
| emit_mov_se_i64(ctx, tmp, imm); |
| emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); |
| break; |
| /* dst = dst & src (64-bit) */ |
| /* dst = dst | src (64-bit) */ |
| /* dst = dst ^ src (64-bit) */ |
| /* dst = dst + src (64-bit) */ |
| /* dst = dst - src (64-bit) */ |
| case BPF_ALU64 | BPF_AND | BPF_X: |
| case BPF_ALU64 | BPF_OR | BPF_X: |
| case BPF_ALU64 | BPF_XOR | BPF_X: |
| case BPF_ALU64 | BPF_ADD | BPF_X: |
| case BPF_ALU64 | BPF_SUB | BPF_X: |
| emit_alu_r64(ctx, dst, src, BPF_OP(code)); |
| break; |
| /* dst = dst << src (64-bit) */ |
| /* dst = dst >> src (64-bit) */ |
| /* dst = dst >> src (64-bit, arithmetic) */ |
| case BPF_ALU64 | BPF_LSH | BPF_X: |
| case BPF_ALU64 | BPF_RSH | BPF_X: |
| case BPF_ALU64 | BPF_ARSH | BPF_X: |
| emit_shift_r64(ctx, dst, lo(src), BPF_OP(code)); |
| break; |
| /* dst = dst * src (64-bit) */ |
| case BPF_ALU64 | BPF_MUL | BPF_X: |
| emit_mul_r64(ctx, dst, src); |
| break; |
| /* dst = dst / src (64-bit) */ |
| /* dst = dst % src (64-bit) */ |
| case BPF_ALU64 | BPF_DIV | BPF_X: |
| case BPF_ALU64 | BPF_MOD | BPF_X: |
| emit_divmod_r64(ctx, dst, src, BPF_OP(code)); |
| break; |
| /* dst = htole(dst) */ |
| /* dst = htobe(dst) */ |
| case BPF_ALU | BPF_END | BPF_FROM_LE: |
| case BPF_ALU | BPF_END | BPF_FROM_BE: |
| if (BPF_SRC(code) == |
| #ifdef __BIG_ENDIAN |
| BPF_FROM_LE |
| #else |
| BPF_FROM_BE |
| #endif |
| ) |
| emit_bswap_r64(ctx, dst, imm); |
| else |
| emit_trunc_r64(ctx, dst, imm); |
| break; |
| /* dst = imm64 */ |
| case BPF_LD | BPF_IMM | BPF_DW: |
| emit_mov_i(ctx, lo(dst), imm); |
| emit_mov_i(ctx, hi(dst), insn[1].imm); |
| return 1; |
| /* LDX: dst = *(size *)(src + off) */ |
| case BPF_LDX | BPF_MEM | BPF_W: |
| case BPF_LDX | BPF_MEM | BPF_H: |
| case BPF_LDX | BPF_MEM | BPF_B: |
| case BPF_LDX | BPF_MEM | BPF_DW: |
| emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code)); |
| break; |
| /* ST: *(size *)(dst + off) = imm */ |
| case BPF_ST | BPF_MEM | BPF_W: |
| case BPF_ST | BPF_MEM | BPF_H: |
| case BPF_ST | BPF_MEM | BPF_B: |
| case BPF_ST | BPF_MEM | BPF_DW: |
| switch (BPF_SIZE(code)) { |
| case BPF_DW: |
| /* Sign-extend immediate value into temporary reg */ |
| emit_mov_se_i64(ctx, tmp, imm); |
| break; |
| case BPF_W: |
| case BPF_H: |
| case BPF_B: |
| emit_mov_i(ctx, lo(tmp), imm); |
| break; |
| } |
| emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code)); |
| break; |
| /* STX: *(size *)(dst + off) = src */ |
| case BPF_STX | BPF_MEM | BPF_W: |
| case BPF_STX | BPF_MEM | BPF_H: |
| case BPF_STX | BPF_MEM | BPF_B: |
| case BPF_STX | BPF_MEM | BPF_DW: |
| emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code)); |
| break; |
| /* Speculation barrier */ |
| case BPF_ST | BPF_NOSPEC: |
| break; |
| /* Atomics */ |
| case BPF_STX | BPF_ATOMIC | BPF_W: |
| switch (imm) { |
| case BPF_ADD: |
| case BPF_ADD | BPF_FETCH: |
| case BPF_AND: |
| case BPF_AND | BPF_FETCH: |
| case BPF_OR: |
| case BPF_OR | BPF_FETCH: |
| case BPF_XOR: |
| case BPF_XOR | BPF_FETCH: |
| case BPF_XCHG: |
| if (cpu_has_llsc) |
| emit_atomic_r(ctx, lo(dst), lo(src), off, imm); |
| else /* Non-ll/sc fallback */ |
| emit_atomic_r32(ctx, lo(dst), lo(src), |
| off, imm); |
| if (imm & BPF_FETCH) |
| emit_zext_ver(ctx, src); |
| break; |
| case BPF_CMPXCHG: |
| if (cpu_has_llsc) |
| emit_cmpxchg_r(ctx, lo(dst), lo(src), |
| lo(res), off); |
| else /* Non-ll/sc fallback */ |
| emit_cmpxchg_r32(ctx, lo(dst), lo(src), off); |
| /* Result zero-extension inserted by verifier */ |
| break; |
| default: |
| goto notyet; |
| } |
| break; |
| /* Atomics (64-bit) */ |
| case BPF_STX | BPF_ATOMIC | BPF_DW: |
| switch (imm) { |
| case BPF_ADD: |
| case BPF_ADD | BPF_FETCH: |
| case BPF_AND: |
| case BPF_AND | BPF_FETCH: |
| case BPF_OR: |
| case BPF_OR | BPF_FETCH: |
| case BPF_XOR: |
| case BPF_XOR | BPF_FETCH: |
| case BPF_XCHG: |
| emit_atomic_r64(ctx, lo(dst), src, off, imm); |
| break; |
| case BPF_CMPXCHG: |
| emit_cmpxchg_r64(ctx, lo(dst), src, off); |
| break; |
| default: |
| goto notyet; |
| } |
| break; |
| /* PC += off if dst == src */ |
| /* PC += off if dst != src */ |
| /* PC += off if dst & src */ |
| /* PC += off if dst > src */ |
| /* PC += off if dst >= src */ |
| /* PC += off if dst < src */ |
| /* PC += off if dst <= src */ |
| /* PC += off if dst > src (signed) */ |
| /* PC += off if dst >= src (signed) */ |
| /* PC += off if dst < src (signed) */ |
| /* PC += off if dst <= src (signed) */ |
| case BPF_JMP32 | BPF_JEQ | BPF_X: |
| case BPF_JMP32 | BPF_JNE | BPF_X: |
| case BPF_JMP32 | BPF_JSET | BPF_X: |
| case BPF_JMP32 | BPF_JGT | BPF_X: |
| case BPF_JMP32 | BPF_JGE | BPF_X: |
| case BPF_JMP32 | BPF_JLT | BPF_X: |
| case BPF_JMP32 | BPF_JLE | BPF_X: |
| case BPF_JMP32 | BPF_JSGT | BPF_X: |
| case BPF_JMP32 | BPF_JSGE | BPF_X: |
| case BPF_JMP32 | BPF_JSLT | BPF_X: |
| case BPF_JMP32 | BPF_JSLE | BPF_X: |
| if (off == 0) |
| break; |
| setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); |
| emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp); |
| if (finish_jmp(ctx, jmp, off) < 0) |
| goto toofar; |
| break; |
| /* PC += off if dst == imm */ |
| /* PC += off if dst != imm */ |
| /* PC += off if dst & imm */ |
| /* PC += off if dst > imm */ |
| /* PC += off if dst >= imm */ |
| /* PC += off if dst < imm */ |
| /* PC += off if dst <= imm */ |
| /* PC += off if dst > imm (signed) */ |
| /* PC += off if dst >= imm (signed) */ |
| /* PC += off if dst < imm (signed) */ |
| /* PC += off if dst <= imm (signed) */ |
| case BPF_JMP32 | BPF_JEQ | BPF_K: |
| case BPF_JMP32 | BPF_JNE | BPF_K: |
| case BPF_JMP32 | BPF_JSET | BPF_K: |
| case BPF_JMP32 | BPF_JGT | BPF_K: |
| case BPF_JMP32 | BPF_JGE | BPF_K: |
| case BPF_JMP32 | BPF_JLT | BPF_K: |
| case BPF_JMP32 | BPF_JLE | BPF_K: |
| case BPF_JMP32 | BPF_JSGT | BPF_K: |
| case BPF_JMP32 | BPF_JSGE | BPF_K: |
| case BPF_JMP32 | BPF_JSLT | BPF_K: |
| case BPF_JMP32 | BPF_JSLE | BPF_K: |
| if (off == 0) |
| break; |
| setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); |
| if (valid_jmp_i(jmp, imm)) { |
| emit_jmp_i(ctx, lo(dst), imm, rel, jmp); |
| } else { |
| /* Move large immediate to register */ |
| emit_mov_i(ctx, MIPS_R_T6, imm); |
| emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp); |
| } |
| if (finish_jmp(ctx, jmp, off) < 0) |
| goto toofar; |
| break; |
| /* PC += off if dst == src */ |
| /* PC += off if dst != src */ |
| /* PC += off if dst & src */ |
| /* PC += off if dst > src */ |
| /* PC += off if dst >= src */ |
| /* PC += off if dst < src */ |
| /* PC += off if dst <= src */ |
| /* PC += off if dst > src (signed) */ |
| /* PC += off if dst >= src (signed) */ |
| /* PC += off if dst < src (signed) */ |
| /* PC += off if dst <= src (signed) */ |
| case BPF_JMP | BPF_JEQ | BPF_X: |
| case BPF_JMP | BPF_JNE | BPF_X: |
| case BPF_JMP | BPF_JSET | BPF_X: |
| case BPF_JMP | BPF_JGT | BPF_X: |
| case BPF_JMP | BPF_JGE | BPF_X: |
| case BPF_JMP | BPF_JLT | BPF_X: |
| case BPF_JMP | BPF_JLE | BPF_X: |
| case BPF_JMP | BPF_JSGT | BPF_X: |
| case BPF_JMP | BPF_JSGE | BPF_X: |
| case BPF_JMP | BPF_JSLT | BPF_X: |
| case BPF_JMP | BPF_JSLE | BPF_X: |
| if (off == 0) |
| break; |
| setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); |
| emit_jmp_r64(ctx, dst, src, rel, jmp); |
| if (finish_jmp(ctx, jmp, off) < 0) |
| goto toofar; |
| break; |
| /* PC += off if dst == imm */ |
| /* PC += off if dst != imm */ |
| /* PC += off if dst & imm */ |
| /* PC += off if dst > imm */ |
| /* PC += off if dst >= imm */ |
| /* PC += off if dst < imm */ |
| /* PC += off if dst <= imm */ |
| /* PC += off if dst > imm (signed) */ |
| /* PC += off if dst >= imm (signed) */ |
| /* PC += off if dst < imm (signed) */ |
| /* PC += off if dst <= imm (signed) */ |
| case BPF_JMP | BPF_JEQ | BPF_K: |
| case BPF_JMP | BPF_JNE | BPF_K: |
| case BPF_JMP | BPF_JSET | BPF_K: |
| case BPF_JMP | BPF_JGT | BPF_K: |
| case BPF_JMP | BPF_JGE | BPF_K: |
| case BPF_JMP | BPF_JLT | BPF_K: |
| case BPF_JMP | BPF_JLE | BPF_K: |
| case BPF_JMP | BPF_JSGT | BPF_K: |
| case BPF_JMP | BPF_JSGE | BPF_K: |
| case BPF_JMP | BPF_JSLT | BPF_K: |
| case BPF_JMP | BPF_JSLE | BPF_K: |
| if (off == 0) |
| break; |
| setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); |
| emit_jmp_i64(ctx, dst, imm, rel, jmp); |
| if (finish_jmp(ctx, jmp, off) < 0) |
| goto toofar; |
| break; |
| /* PC += off */ |
| case BPF_JMP | BPF_JA: |
| if (off == 0) |
| break; |
| if (emit_ja(ctx, off) < 0) |
| goto toofar; |
| break; |
| /* Tail call */ |
| case BPF_JMP | BPF_TAIL_CALL: |
| if (emit_tail_call(ctx) < 0) |
| goto invalid; |
| break; |
| /* Function call */ |
| case BPF_JMP | BPF_CALL: |
| if (emit_call(ctx, insn) < 0) |
| goto invalid; |
| break; |
| /* Function return */ |
| case BPF_JMP | BPF_EXIT: |
| /* |
| * Optimization: when last instruction is EXIT |
| * simply continue to epilogue. |
| */ |
| if (ctx->bpf_index == ctx->program->len - 1) |
| break; |
| if (emit_exit(ctx) < 0) |
| goto toofar; |
| break; |
| |
| default: |
| invalid: |
| pr_err_once("unknown opcode %02x\n", code); |
| return -EINVAL; |
| notyet: |
| pr_info_once("*** NOT YET: opcode %02x ***\n", code); |
| return -EFAULT; |
| toofar: |
| pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", |
| ctx->bpf_index, code); |
| return -E2BIG; |
| } |
| return 0; |
| } |