Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 1 | /* |
| 2 | * bpf_jit_comp.c: BPF JIT compiler |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 3 | * |
Eric Dumazet | 3b58908 | 2013-01-30 17:51:44 -0800 | [diff] [blame] | 4 | * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 5 | * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 6 | * |
| 7 | * This program is free software; you can redistribute it and/or |
| 8 | * modify it under the terms of the GNU General Public License |
| 9 | * as published by the Free Software Foundation; version 2 |
| 10 | * of the License. |
| 11 | */ |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 12 | #include <linux/netdevice.h> |
| 13 | #include <linux/filter.h> |
Eric Dumazet | 855ddb5 | 2012-10-27 02:26:22 +0000 | [diff] [blame] | 14 | #include <linux/if_vlan.h> |
Daniel Borkmann | 71d22d5 | 2018-02-26 22:13:52 +0100 | [diff] [blame] | 15 | #include <linux/bpf.h> |
| 16 | |
Laura Abbott | d116365 | 2017-05-08 15:58:11 -0700 | [diff] [blame] | 17 | #include <asm/set_memory.h> |
Daniel Borkmann | a493a87 | 2018-02-22 15:12:53 +0100 | [diff] [blame] | 18 | #include <asm/nospec-branch.h> |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 19 | |
Joe Perches | 5cccc70 | 2014-12-04 17:01:24 -0800 | [diff] [blame] | 20 | static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 21 | { |
| 22 | if (len == 1) |
| 23 | *ptr = bytes; |
| 24 | else if (len == 2) |
| 25 | *(u16 *)ptr = bytes; |
| 26 | else { |
| 27 | *(u32 *)ptr = bytes; |
| 28 | barrier(); |
| 29 | } |
| 30 | return ptr + len; |
| 31 | } |
| 32 | |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 33 | #define EMIT(bytes, len) \ |
| 34 | do { prog = emit_code(prog, bytes, len); cnt += len; } while (0) |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 35 | |
| 36 | #define EMIT1(b1) EMIT(b1, 1) |
| 37 | #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) |
| 38 | #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) |
| 39 | #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 40 | |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 41 | #define EMIT1_off32(b1, off) \ |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 42 | do { EMIT1(b1); EMIT(off, 4); } while (0) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 43 | #define EMIT2_off32(b1, b2, off) \ |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 44 | do { EMIT2(b1, b2); EMIT(off, 4); } while (0) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 45 | #define EMIT3_off32(b1, b2, b3, off) \ |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 46 | do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 47 | #define EMIT4_off32(b1, b2, b3, b4, off) \ |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 48 | do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 49 | |
Joe Perches | 5cccc70 | 2014-12-04 17:01:24 -0800 | [diff] [blame] | 50 | static bool is_imm8(int value) |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 51 | { |
| 52 | return value <= 127 && value >= -128; |
| 53 | } |
| 54 | |
Joe Perches | 5cccc70 | 2014-12-04 17:01:24 -0800 | [diff] [blame] | 55 | static bool is_simm32(s64 value) |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 56 | { |
Daniel Borkmann | 6fe8b9c | 2018-02-24 01:07:59 +0100 | [diff] [blame] | 57 | return value == (s64)(s32)value; |
| 58 | } |
| 59 | |
| 60 | static bool is_uimm32(u64 value) |
| 61 | { |
| 62 | return value == (u64)(u32)value; |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 63 | } |
| 64 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 65 | /* mov dst, src */ |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 66 | #define EMIT_mov(DST, SRC) \ |
| 67 | do { \ |
| 68 | if (DST != SRC) \ |
| 69 | EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 70 | } while (0) |
| 71 | |
| 72 | static int bpf_size_to_x86_bytes(int bpf_size) |
| 73 | { |
| 74 | if (bpf_size == BPF_W) |
| 75 | return 4; |
| 76 | else if (bpf_size == BPF_H) |
| 77 | return 2; |
| 78 | else if (bpf_size == BPF_B) |
| 79 | return 1; |
| 80 | else if (bpf_size == BPF_DW) |
| 81 | return 4; /* imm32 */ |
| 82 | else |
| 83 | return 0; |
| 84 | } |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 85 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 86 | /* |
| 87 | * List of x86 cond jumps opcodes (. + s8) |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 88 | * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32) |
| 89 | */ |
| 90 | #define X86_JB 0x72 |
| 91 | #define X86_JAE 0x73 |
| 92 | #define X86_JE 0x74 |
| 93 | #define X86_JNE 0x75 |
| 94 | #define X86_JBE 0x76 |
| 95 | #define X86_JA 0x77 |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 96 | #define X86_JL 0x7C |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 97 | #define X86_JGE 0x7D |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 98 | #define X86_JLE 0x7E |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 99 | #define X86_JG 0x7F |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 100 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 101 | /* Pick a register outside of BPF range for JIT internal work */ |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 102 | #define AUX_REG (MAX_BPF_JIT_REG + 1) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 103 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 104 | /* |
| 105 | * The following table maps BPF registers to x86-64 registers. |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 106 | * |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 107 | * x86-64 register R12 is unused, since if used as base address |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 108 | * register in load/store instructions, it always needs an |
| 109 | * extra byte of encoding and is callee saved. |
| 110 | * |
Daniel Borkmann | e782bdc | 2018-05-04 01:08:16 +0200 | [diff] [blame] | 111 | * Also x86-64 register R9 is unused. x86-64 register R10 is |
| 112 | * used for blinding (if enabled). |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 113 | */ |
| 114 | static const int reg2hex[] = { |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 115 | [BPF_REG_0] = 0, /* RAX */ |
| 116 | [BPF_REG_1] = 7, /* RDI */ |
| 117 | [BPF_REG_2] = 6, /* RSI */ |
| 118 | [BPF_REG_3] = 2, /* RDX */ |
| 119 | [BPF_REG_4] = 1, /* RCX */ |
| 120 | [BPF_REG_5] = 0, /* R8 */ |
| 121 | [BPF_REG_6] = 3, /* RBX callee saved */ |
| 122 | [BPF_REG_7] = 5, /* R13 callee saved */ |
| 123 | [BPF_REG_8] = 6, /* R14 callee saved */ |
| 124 | [BPF_REG_9] = 7, /* R15 callee saved */ |
| 125 | [BPF_REG_FP] = 5, /* RBP readonly */ |
| 126 | [BPF_REG_AX] = 2, /* R10 temp register */ |
| 127 | [AUX_REG] = 3, /* R11 temp register */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 128 | }; |
| 129 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 130 | /* |
| 131 | * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15 |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 132 | * which need extra byte of encoding. |
| 133 | * rax,rcx,...,rbp have simpler encoding |
| 134 | */ |
Joe Perches | 5cccc70 | 2014-12-04 17:01:24 -0800 | [diff] [blame] | 135 | static bool is_ereg(u32 reg) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 136 | { |
Joe Perches | d148134 | 2014-12-04 15:00:48 -0800 | [diff] [blame] | 137 | return (1 << reg) & (BIT(BPF_REG_5) | |
| 138 | BIT(AUX_REG) | |
| 139 | BIT(BPF_REG_7) | |
| 140 | BIT(BPF_REG_8) | |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 141 | BIT(BPF_REG_9) | |
| 142 | BIT(BPF_REG_AX)); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 143 | } |
| 144 | |
Daniel Borkmann | de0a444 | 2018-01-20 01:24:35 +0100 | [diff] [blame] | 145 | static bool is_axreg(u32 reg) |
| 146 | { |
| 147 | return reg == BPF_REG_0; |
| 148 | } |
| 149 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 150 | /* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */ |
Joe Perches | 5cccc70 | 2014-12-04 17:01:24 -0800 | [diff] [blame] | 151 | static u8 add_1mod(u8 byte, u32 reg) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 152 | { |
| 153 | if (is_ereg(reg)) |
| 154 | byte |= 1; |
| 155 | return byte; |
| 156 | } |
| 157 | |
Joe Perches | 5cccc70 | 2014-12-04 17:01:24 -0800 | [diff] [blame] | 158 | static u8 add_2mod(u8 byte, u32 r1, u32 r2) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 159 | { |
| 160 | if (is_ereg(r1)) |
| 161 | byte |= 1; |
| 162 | if (is_ereg(r2)) |
| 163 | byte |= 4; |
| 164 | return byte; |
| 165 | } |
| 166 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 167 | /* Encode 'dst_reg' register into x86-64 opcode 'byte' */ |
Joe Perches | 5cccc70 | 2014-12-04 17:01:24 -0800 | [diff] [blame] | 168 | static u8 add_1reg(u8 byte, u32 dst_reg) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 169 | { |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 170 | return byte + reg2hex[dst_reg]; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 171 | } |
| 172 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 173 | /* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */ |
Joe Perches | 5cccc70 | 2014-12-04 17:01:24 -0800 | [diff] [blame] | 174 | static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 175 | { |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 176 | return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 177 | } |
| 178 | |
Daniel Borkmann | 738cbe7 | 2014-09-08 08:04:47 +0200 | [diff] [blame] | 179 | static void jit_fill_hole(void *area, unsigned int size) |
| 180 | { |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 181 | /* Fill whole space with INT3 instructions */ |
Daniel Borkmann | 738cbe7 | 2014-09-08 08:04:47 +0200 | [diff] [blame] | 182 | memset(area, 0xcc, size); |
| 183 | } |
| 184 | |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 185 | struct jit_context { |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 186 | int cleanup_addr; /* Epilogue code offset */ |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 187 | }; |
| 188 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 189 | /* Maximum number of bytes emitted while JITing one eBPF insn */ |
Alexei Starovoitov | e0ee9c1 | 2014-10-10 20:30:23 -0700 | [diff] [blame] | 190 | #define BPF_MAX_INSN_SIZE 128 |
| 191 | #define BPF_INSN_SAFETY 64 |
| 192 | |
Daniel Borkmann | e782bdc | 2018-05-04 01:08:16 +0200 | [diff] [blame] | 193 | #define AUX_STACK_SPACE 40 /* Space for RBX, R13, R14, R15, tailcnt */ |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 194 | |
Daniel Borkmann | e782bdc | 2018-05-04 01:08:16 +0200 | [diff] [blame] | 195 | #define PROLOGUE_SIZE 37 |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 196 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 197 | /* |
| 198 | * Emit x86-64 prologue code for BPF program and check its size. |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 199 | * bpf_tail_call helper will skip it while jumping into another program |
| 200 | */ |
Daniel Borkmann | 0869175 | 2018-02-24 01:08:02 +0100 | [diff] [blame] | 201 | static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 202 | { |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 203 | u8 *prog = *pprog; |
| 204 | int cnt = 0; |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 205 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 206 | /* push rbp */ |
| 207 | EMIT1(0x55); |
| 208 | |
| 209 | /* mov rbp,rsp */ |
| 210 | EMIT3(0x48, 0x89, 0xE5); |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 211 | |
Alexei Starovoitov | 2960ae4 | 2017-05-30 13:31:35 -0700 | [diff] [blame] | 212 | /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */ |
| 213 | EMIT3_off32(0x48, 0x81, 0xEC, |
| 214 | round_up(stack_depth, 8) + AUX_STACK_SPACE); |
Alexei Starovoitov | 177366b | 2017-05-30 13:31:34 -0700 | [diff] [blame] | 215 | |
| 216 | /* sub rbp, AUX_STACK_SPACE */ |
| 217 | EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE); |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 218 | |
Alexei Starovoitov | 177366b | 2017-05-30 13:31:34 -0700 | [diff] [blame] | 219 | /* mov qword ptr [rbp+0],rbx */ |
| 220 | EMIT4(0x48, 0x89, 0x5D, 0); |
Alexei Starovoitov | 177366b | 2017-05-30 13:31:34 -0700 | [diff] [blame] | 221 | /* mov qword ptr [rbp+8],r13 */ |
| 222 | EMIT4(0x4C, 0x89, 0x6D, 8); |
| 223 | /* mov qword ptr [rbp+16],r14 */ |
| 224 | EMIT4(0x4C, 0x89, 0x75, 16); |
| 225 | /* mov qword ptr [rbp+24],r15 */ |
| 226 | EMIT4(0x4C, 0x89, 0x7D, 24); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 227 | |
Daniel Borkmann | 0869175 | 2018-02-24 01:08:02 +0100 | [diff] [blame] | 228 | if (!ebpf_from_cbpf) { |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 229 | /* |
| 230 | * Clear the tail call counter (tail_call_cnt): for eBPF tail |
Daniel Borkmann | 0869175 | 2018-02-24 01:08:02 +0100 | [diff] [blame] | 231 | * calls we need to reset the counter to 0. It's done in two |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 232 | * instructions, resetting RAX register to 0, and moving it |
Daniel Borkmann | 0869175 | 2018-02-24 01:08:02 +0100 | [diff] [blame] | 233 | * to the counter location. |
| 234 | */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 235 | |
Daniel Borkmann | 0869175 | 2018-02-24 01:08:02 +0100 | [diff] [blame] | 236 | /* xor eax, eax */ |
| 237 | EMIT2(0x31, 0xc0); |
| 238 | /* mov qword ptr [rbp+32], rax */ |
| 239 | EMIT4(0x48, 0x89, 0x45, 32); |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 240 | |
Daniel Borkmann | 0869175 | 2018-02-24 01:08:02 +0100 | [diff] [blame] | 241 | BUILD_BUG_ON(cnt != PROLOGUE_SIZE); |
| 242 | } |
| 243 | |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 244 | *pprog = prog; |
| 245 | } |
| 246 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 247 | /* |
| 248 | * Generate the following code: |
| 249 | * |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 250 | * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... |
| 251 | * if (index >= array->map.max_entries) |
| 252 | * goto out; |
| 253 | * if (++tail_call_cnt > MAX_TAIL_CALL_CNT) |
| 254 | * goto out; |
Wang Nan | 2a36f0b | 2015-08-06 07:02:33 +0000 | [diff] [blame] | 255 | * prog = array->ptrs[index]; |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 256 | * if (prog == NULL) |
| 257 | * goto out; |
| 258 | * goto *(prog->bpf_func + prologue_size); |
| 259 | * out: |
| 260 | */ |
| 261 | static void emit_bpf_tail_call(u8 **pprog) |
| 262 | { |
| 263 | u8 *prog = *pprog; |
| 264 | int label1, label2, label3; |
| 265 | int cnt = 0; |
| 266 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 267 | /* |
| 268 | * rdi - pointer to ctx |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 269 | * rsi - pointer to bpf_array |
| 270 | * rdx - index in bpf_array |
| 271 | */ |
| 272 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 273 | /* |
| 274 | * if (index >= array->map.max_entries) |
| 275 | * goto out; |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 276 | */ |
Alexei Starovoitov | 90caccd | 2017-10-03 15:37:20 -0700 | [diff] [blame] | 277 | EMIT2(0x89, 0xD2); /* mov edx, edx */ |
| 278 | EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 279 | offsetof(struct bpf_array, map.max_entries)); |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 280 | #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */ |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 281 | EMIT2(X86_JBE, OFFSET1); /* jbe out */ |
| 282 | label1 = cnt; |
| 283 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 284 | /* |
| 285 | * if (tail_call_cnt > MAX_TAIL_CALL_CNT) |
| 286 | * goto out; |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 287 | */ |
Alexei Starovoitov | 177366b | 2017-05-30 13:31:34 -0700 | [diff] [blame] | 288 | EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */ |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 289 | EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ |
Daniel Borkmann | a493a87 | 2018-02-22 15:12:53 +0100 | [diff] [blame] | 290 | #define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE) |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 291 | EMIT2(X86_JA, OFFSET2); /* ja out */ |
| 292 | label2 = cnt; |
| 293 | EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ |
Alexei Starovoitov | 177366b | 2017-05-30 13:31:34 -0700 | [diff] [blame] | 294 | EMIT2_off32(0x89, 0x85, 36); /* mov dword ptr [rbp + 36], eax */ |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 295 | |
Wang Nan | 2a36f0b | 2015-08-06 07:02:33 +0000 | [diff] [blame] | 296 | /* prog = array->ptrs[index]; */ |
Eric Dumazet | 84ccac6 | 2017-08-31 04:53:42 -0700 | [diff] [blame] | 297 | EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ |
Wang Nan | 2a36f0b | 2015-08-06 07:02:33 +0000 | [diff] [blame] | 298 | offsetof(struct bpf_array, ptrs)); |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 299 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 300 | /* |
| 301 | * if (prog == NULL) |
| 302 | * goto out; |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 303 | */ |
Eric Dumazet | 84ccac6 | 2017-08-31 04:53:42 -0700 | [diff] [blame] | 304 | EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ |
Daniel Borkmann | a493a87 | 2018-02-22 15:12:53 +0100 | [diff] [blame] | 305 | #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE) |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 306 | EMIT2(X86_JE, OFFSET3); /* je out */ |
| 307 | label3 = cnt; |
| 308 | |
| 309 | /* goto *(prog->bpf_func + prologue_size); */ |
| 310 | EMIT4(0x48, 0x8B, 0x40, /* mov rax, qword ptr [rax + 32] */ |
| 311 | offsetof(struct bpf_prog, bpf_func)); |
| 312 | EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE); /* add rax, prologue_size */ |
| 313 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 314 | /* |
| 315 | * Wow we're ready to jump into next BPF program |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 316 | * rdi == ctx (1st arg) |
| 317 | * rax == prog->bpf_func + prologue_size |
| 318 | */ |
Daniel Borkmann | a493a87 | 2018-02-22 15:12:53 +0100 | [diff] [blame] | 319 | RETPOLINE_RAX_BPF_JIT(); |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 320 | |
| 321 | /* out: */ |
| 322 | BUILD_BUG_ON(cnt - label1 != OFFSET1); |
| 323 | BUILD_BUG_ON(cnt - label2 != OFFSET2); |
| 324 | BUILD_BUG_ON(cnt - label3 != OFFSET3); |
| 325 | *pprog = prog; |
| 326 | } |
| 327 | |
Daniel Borkmann | 6fe8b9c | 2018-02-24 01:07:59 +0100 | [diff] [blame] | 328 | static void emit_mov_imm32(u8 **pprog, bool sign_propagate, |
| 329 | u32 dst_reg, const u32 imm32) |
| 330 | { |
| 331 | u8 *prog = *pprog; |
| 332 | u8 b1, b2, b3; |
| 333 | int cnt = 0; |
| 334 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 335 | /* |
| 336 | * Optimization: if imm32 is positive, use 'mov %eax, imm32' |
Daniel Borkmann | 6fe8b9c | 2018-02-24 01:07:59 +0100 | [diff] [blame] | 337 | * (which zero-extends imm32) to save 2 bytes. |
| 338 | */ |
| 339 | if (sign_propagate && (s32)imm32 < 0) { |
| 340 | /* 'mov %rax, imm32' sign extends imm32 */ |
| 341 | b1 = add_1mod(0x48, dst_reg); |
| 342 | b2 = 0xC7; |
| 343 | b3 = 0xC0; |
| 344 | EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32); |
| 345 | goto done; |
| 346 | } |
| 347 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 348 | /* |
| 349 | * Optimization: if imm32 is zero, use 'xor %eax, %eax' |
Daniel Borkmann | 6fe8b9c | 2018-02-24 01:07:59 +0100 | [diff] [blame] | 350 | * to save 3 bytes. |
| 351 | */ |
| 352 | if (imm32 == 0) { |
| 353 | if (is_ereg(dst_reg)) |
| 354 | EMIT1(add_2mod(0x40, dst_reg, dst_reg)); |
| 355 | b2 = 0x31; /* xor */ |
| 356 | b3 = 0xC0; |
| 357 | EMIT2(b2, add_2reg(b3, dst_reg, dst_reg)); |
| 358 | goto done; |
| 359 | } |
| 360 | |
| 361 | /* mov %eax, imm32 */ |
| 362 | if (is_ereg(dst_reg)) |
| 363 | EMIT1(add_1mod(0x40, dst_reg)); |
| 364 | EMIT1_off32(add_1reg(0xB8, dst_reg), imm32); |
| 365 | done: |
| 366 | *pprog = prog; |
| 367 | } |
| 368 | |
| 369 | static void emit_mov_imm64(u8 **pprog, u32 dst_reg, |
| 370 | const u32 imm32_hi, const u32 imm32_lo) |
| 371 | { |
| 372 | u8 *prog = *pprog; |
| 373 | int cnt = 0; |
| 374 | |
| 375 | if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) { |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 376 | /* |
| 377 | * For emitting plain u32, where sign bit must not be |
Daniel Borkmann | 6fe8b9c | 2018-02-24 01:07:59 +0100 | [diff] [blame] | 378 | * propagated LLVM tends to load imm64 over mov32 |
| 379 | * directly, so save couple of bytes by just doing |
| 380 | * 'mov %eax, imm32' instead. |
| 381 | */ |
| 382 | emit_mov_imm32(&prog, false, dst_reg, imm32_lo); |
| 383 | } else { |
| 384 | /* movabsq %rax, imm64 */ |
| 385 | EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg)); |
| 386 | EMIT(imm32_lo, 4); |
| 387 | EMIT(imm32_hi, 4); |
| 388 | } |
| 389 | |
| 390 | *pprog = prog; |
| 391 | } |
| 392 | |
Daniel Borkmann | 4c38e2f | 2018-02-24 01:08:01 +0100 | [diff] [blame] | 393 | static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg) |
| 394 | { |
| 395 | u8 *prog = *pprog; |
| 396 | int cnt = 0; |
| 397 | |
| 398 | if (is64) { |
| 399 | /* mov dst, src */ |
| 400 | EMIT_mov(dst_reg, src_reg); |
| 401 | } else { |
| 402 | /* mov32 dst, src */ |
| 403 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
| 404 | EMIT1(add_2mod(0x40, dst_reg, src_reg)); |
| 405 | EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg)); |
| 406 | } |
| 407 | |
| 408 | *pprog = prog; |
| 409 | } |
| 410 | |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 411 | static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, |
| 412 | int oldproglen, struct jit_context *ctx) |
| 413 | { |
| 414 | struct bpf_insn *insn = bpf_prog->insnsi; |
| 415 | int insn_cnt = bpf_prog->len; |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 416 | bool seen_exit = false; |
| 417 | u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; |
| 418 | int i, cnt = 0; |
| 419 | int proglen = 0; |
| 420 | u8 *prog = temp; |
| 421 | |
Daniel Borkmann | 0869175 | 2018-02-24 01:08:02 +0100 | [diff] [blame] | 422 | emit_prologue(&prog, bpf_prog->aux->stack_depth, |
| 423 | bpf_prog_was_classic(bpf_prog)); |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 424 | |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 425 | for (i = 0; i < insn_cnt; i++, insn++) { |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 426 | const s32 imm32 = insn->imm; |
| 427 | u32 dst_reg = insn->dst_reg; |
| 428 | u32 src_reg = insn->src_reg; |
Daniel Borkmann | 6fe8b9c | 2018-02-24 01:07:59 +0100 | [diff] [blame] | 429 | u8 b2 = 0, b3 = 0; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 430 | s64 jmp_offset; |
| 431 | u8 jmp_cond; |
| 432 | int ilen; |
| 433 | u8 *func; |
| 434 | |
| 435 | switch (insn->code) { |
| 436 | /* ALU */ |
| 437 | case BPF_ALU | BPF_ADD | BPF_X: |
| 438 | case BPF_ALU | BPF_SUB | BPF_X: |
| 439 | case BPF_ALU | BPF_AND | BPF_X: |
| 440 | case BPF_ALU | BPF_OR | BPF_X: |
| 441 | case BPF_ALU | BPF_XOR | BPF_X: |
| 442 | case BPF_ALU64 | BPF_ADD | BPF_X: |
| 443 | case BPF_ALU64 | BPF_SUB | BPF_X: |
| 444 | case BPF_ALU64 | BPF_AND | BPF_X: |
| 445 | case BPF_ALU64 | BPF_OR | BPF_X: |
| 446 | case BPF_ALU64 | BPF_XOR | BPF_X: |
| 447 | switch (BPF_OP(insn->code)) { |
| 448 | case BPF_ADD: b2 = 0x01; break; |
| 449 | case BPF_SUB: b2 = 0x29; break; |
| 450 | case BPF_AND: b2 = 0x21; break; |
| 451 | case BPF_OR: b2 = 0x09; break; |
| 452 | case BPF_XOR: b2 = 0x31; break; |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 453 | } |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 454 | if (BPF_CLASS(insn->code) == BPF_ALU64) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 455 | EMIT1(add_2mod(0x48, dst_reg, src_reg)); |
| 456 | else if (is_ereg(dst_reg) || is_ereg(src_reg)) |
| 457 | EMIT1(add_2mod(0x40, dst_reg, src_reg)); |
| 458 | EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 459 | break; |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 460 | |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 461 | case BPF_ALU64 | BPF_MOV | BPF_X: |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 462 | case BPF_ALU | BPF_MOV | BPF_X: |
Daniel Borkmann | 4c38e2f | 2018-02-24 01:08:01 +0100 | [diff] [blame] | 463 | emit_mov_reg(&prog, |
| 464 | BPF_CLASS(insn->code) == BPF_ALU64, |
| 465 | dst_reg, src_reg); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 466 | break; |
Eric Dumazet | 3b58908 | 2013-01-30 17:51:44 -0800 | [diff] [blame] | 467 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 468 | /* neg dst */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 469 | case BPF_ALU | BPF_NEG: |
| 470 | case BPF_ALU64 | BPF_NEG: |
| 471 | if (BPF_CLASS(insn->code) == BPF_ALU64) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 472 | EMIT1(add_1mod(0x48, dst_reg)); |
| 473 | else if (is_ereg(dst_reg)) |
| 474 | EMIT1(add_1mod(0x40, dst_reg)); |
| 475 | EMIT2(0xF7, add_1reg(0xD8, dst_reg)); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 476 | break; |
| 477 | |
| 478 | case BPF_ALU | BPF_ADD | BPF_K: |
| 479 | case BPF_ALU | BPF_SUB | BPF_K: |
| 480 | case BPF_ALU | BPF_AND | BPF_K: |
| 481 | case BPF_ALU | BPF_OR | BPF_K: |
| 482 | case BPF_ALU | BPF_XOR | BPF_K: |
| 483 | case BPF_ALU64 | BPF_ADD | BPF_K: |
| 484 | case BPF_ALU64 | BPF_SUB | BPF_K: |
| 485 | case BPF_ALU64 | BPF_AND | BPF_K: |
| 486 | case BPF_ALU64 | BPF_OR | BPF_K: |
| 487 | case BPF_ALU64 | BPF_XOR | BPF_K: |
| 488 | if (BPF_CLASS(insn->code) == BPF_ALU64) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 489 | EMIT1(add_1mod(0x48, dst_reg)); |
| 490 | else if (is_ereg(dst_reg)) |
| 491 | EMIT1(add_1mod(0x40, dst_reg)); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 492 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 493 | /* |
| 494 | * b3 holds 'normal' opcode, b2 short form only valid |
Daniel Borkmann | de0a444 | 2018-01-20 01:24:35 +0100 | [diff] [blame] | 495 | * in case dst is eax/rax. |
| 496 | */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 497 | switch (BPF_OP(insn->code)) { |
Daniel Borkmann | de0a444 | 2018-01-20 01:24:35 +0100 | [diff] [blame] | 498 | case BPF_ADD: |
| 499 | b3 = 0xC0; |
| 500 | b2 = 0x05; |
| 501 | break; |
| 502 | case BPF_SUB: |
| 503 | b3 = 0xE8; |
| 504 | b2 = 0x2D; |
| 505 | break; |
| 506 | case BPF_AND: |
| 507 | b3 = 0xE0; |
| 508 | b2 = 0x25; |
| 509 | break; |
| 510 | case BPF_OR: |
| 511 | b3 = 0xC8; |
| 512 | b2 = 0x0D; |
| 513 | break; |
| 514 | case BPF_XOR: |
| 515 | b3 = 0xF0; |
| 516 | b2 = 0x35; |
| 517 | break; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 518 | } |
| 519 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 520 | if (is_imm8(imm32)) |
| 521 | EMIT3(0x83, add_1reg(b3, dst_reg), imm32); |
Daniel Borkmann | de0a444 | 2018-01-20 01:24:35 +0100 | [diff] [blame] | 522 | else if (is_axreg(dst_reg)) |
| 523 | EMIT1_off32(b2, imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 524 | else |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 525 | EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 526 | break; |
| 527 | |
| 528 | case BPF_ALU64 | BPF_MOV | BPF_K: |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 529 | case BPF_ALU | BPF_MOV | BPF_K: |
Daniel Borkmann | 6fe8b9c | 2018-02-24 01:07:59 +0100 | [diff] [blame] | 530 | emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64, |
| 531 | dst_reg, imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 532 | break; |
| 533 | |
Alexei Starovoitov | 02ab695 | 2014-09-04 22:17:17 -0700 | [diff] [blame] | 534 | case BPF_LD | BPF_IMM | BPF_DW: |
Daniel Borkmann | 6fe8b9c | 2018-02-24 01:07:59 +0100 | [diff] [blame] | 535 | emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); |
Alexei Starovoitov | 02ab695 | 2014-09-04 22:17:17 -0700 | [diff] [blame] | 536 | insn++; |
| 537 | i++; |
| 538 | break; |
| 539 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 540 | /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 541 | case BPF_ALU | BPF_MOD | BPF_X: |
| 542 | case BPF_ALU | BPF_DIV | BPF_X: |
| 543 | case BPF_ALU | BPF_MOD | BPF_K: |
| 544 | case BPF_ALU | BPF_DIV | BPF_K: |
| 545 | case BPF_ALU64 | BPF_MOD | BPF_X: |
| 546 | case BPF_ALU64 | BPF_DIV | BPF_X: |
| 547 | case BPF_ALU64 | BPF_MOD | BPF_K: |
| 548 | case BPF_ALU64 | BPF_DIV | BPF_K: |
| 549 | EMIT1(0x50); /* push rax */ |
| 550 | EMIT1(0x52); /* push rdx */ |
| 551 | |
| 552 | if (BPF_SRC(insn->code) == BPF_X) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 553 | /* mov r11, src_reg */ |
| 554 | EMIT_mov(AUX_REG, src_reg); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 555 | else |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 556 | /* mov r11, imm32 */ |
| 557 | EMIT3_off32(0x49, 0xC7, 0xC3, imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 558 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 559 | /* mov rax, dst_reg */ |
| 560 | EMIT_mov(BPF_REG_0, dst_reg); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 561 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 562 | /* |
| 563 | * xor edx, edx |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 564 | * equivalent to 'xor rdx, rdx', but one byte less |
| 565 | */ |
| 566 | EMIT2(0x31, 0xd2); |
| 567 | |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 568 | if (BPF_CLASS(insn->code) == BPF_ALU64) |
| 569 | /* div r11 */ |
| 570 | EMIT3(0x49, 0xF7, 0xF3); |
| 571 | else |
| 572 | /* div r11d */ |
| 573 | EMIT3(0x41, 0xF7, 0xF3); |
| 574 | |
| 575 | if (BPF_OP(insn->code) == BPF_MOD) |
| 576 | /* mov r11, rdx */ |
| 577 | EMIT3(0x49, 0x89, 0xD3); |
| 578 | else |
| 579 | /* mov r11, rax */ |
| 580 | EMIT3(0x49, 0x89, 0xC3); |
| 581 | |
| 582 | EMIT1(0x5A); /* pop rdx */ |
| 583 | EMIT1(0x58); /* pop rax */ |
| 584 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 585 | /* mov dst_reg, r11 */ |
| 586 | EMIT_mov(dst_reg, AUX_REG); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 587 | break; |
| 588 | |
| 589 | case BPF_ALU | BPF_MUL | BPF_K: |
| 590 | case BPF_ALU | BPF_MUL | BPF_X: |
| 591 | case BPF_ALU64 | BPF_MUL | BPF_K: |
| 592 | case BPF_ALU64 | BPF_MUL | BPF_X: |
Daniel Borkmann | 4c38e2f | 2018-02-24 01:08:01 +0100 | [diff] [blame] | 593 | { |
| 594 | bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; |
| 595 | |
Daniel Borkmann | d806a0c | 2018-02-24 01:08:00 +0100 | [diff] [blame] | 596 | if (dst_reg != BPF_REG_0) |
| 597 | EMIT1(0x50); /* push rax */ |
| 598 | if (dst_reg != BPF_REG_3) |
| 599 | EMIT1(0x52); /* push rdx */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 600 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 601 | /* mov r11, dst_reg */ |
| 602 | EMIT_mov(AUX_REG, dst_reg); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 603 | |
| 604 | if (BPF_SRC(insn->code) == BPF_X) |
Daniel Borkmann | 4c38e2f | 2018-02-24 01:08:01 +0100 | [diff] [blame] | 605 | emit_mov_reg(&prog, is64, BPF_REG_0, src_reg); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 606 | else |
Daniel Borkmann | 4c38e2f | 2018-02-24 01:08:01 +0100 | [diff] [blame] | 607 | emit_mov_imm32(&prog, is64, BPF_REG_0, imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 608 | |
Daniel Borkmann | 4c38e2f | 2018-02-24 01:08:01 +0100 | [diff] [blame] | 609 | if (is64) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 610 | EMIT1(add_1mod(0x48, AUX_REG)); |
| 611 | else if (is_ereg(AUX_REG)) |
| 612 | EMIT1(add_1mod(0x40, AUX_REG)); |
| 613 | /* mul(q) r11 */ |
| 614 | EMIT2(0xF7, add_1reg(0xE0, AUX_REG)); |
| 615 | |
Daniel Borkmann | d806a0c | 2018-02-24 01:08:00 +0100 | [diff] [blame] | 616 | if (dst_reg != BPF_REG_3) |
| 617 | EMIT1(0x5A); /* pop rdx */ |
| 618 | if (dst_reg != BPF_REG_0) { |
| 619 | /* mov dst_reg, rax */ |
| 620 | EMIT_mov(dst_reg, BPF_REG_0); |
| 621 | EMIT1(0x58); /* pop rax */ |
| 622 | } |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 623 | break; |
Daniel Borkmann | 4c38e2f | 2018-02-24 01:08:01 +0100 | [diff] [blame] | 624 | } |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 625 | /* Shifts */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 626 | case BPF_ALU | BPF_LSH | BPF_K: |
| 627 | case BPF_ALU | BPF_RSH | BPF_K: |
| 628 | case BPF_ALU | BPF_ARSH | BPF_K: |
| 629 | case BPF_ALU64 | BPF_LSH | BPF_K: |
| 630 | case BPF_ALU64 | BPF_RSH | BPF_K: |
| 631 | case BPF_ALU64 | BPF_ARSH | BPF_K: |
| 632 | if (BPF_CLASS(insn->code) == BPF_ALU64) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 633 | EMIT1(add_1mod(0x48, dst_reg)); |
| 634 | else if (is_ereg(dst_reg)) |
| 635 | EMIT1(add_1mod(0x40, dst_reg)); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 636 | |
| 637 | switch (BPF_OP(insn->code)) { |
| 638 | case BPF_LSH: b3 = 0xE0; break; |
| 639 | case BPF_RSH: b3 = 0xE8; break; |
| 640 | case BPF_ARSH: b3 = 0xF8; break; |
| 641 | } |
Daniel Borkmann | 88e69a1 | 2018-02-24 01:07:58 +0100 | [diff] [blame] | 642 | |
| 643 | if (imm32 == 1) |
| 644 | EMIT2(0xD1, add_1reg(b3, dst_reg)); |
| 645 | else |
| 646 | EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 647 | break; |
| 648 | |
Alexei Starovoitov | 72b603e | 2014-08-25 12:27:02 -0700 | [diff] [blame] | 649 | case BPF_ALU | BPF_LSH | BPF_X: |
| 650 | case BPF_ALU | BPF_RSH | BPF_X: |
| 651 | case BPF_ALU | BPF_ARSH | BPF_X: |
| 652 | case BPF_ALU64 | BPF_LSH | BPF_X: |
| 653 | case BPF_ALU64 | BPF_RSH | BPF_X: |
| 654 | case BPF_ALU64 | BPF_ARSH | BPF_X: |
| 655 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 656 | /* Check for bad case when dst_reg == rcx */ |
Alexei Starovoitov | 72b603e | 2014-08-25 12:27:02 -0700 | [diff] [blame] | 657 | if (dst_reg == BPF_REG_4) { |
| 658 | /* mov r11, dst_reg */ |
| 659 | EMIT_mov(AUX_REG, dst_reg); |
| 660 | dst_reg = AUX_REG; |
| 661 | } |
| 662 | |
| 663 | if (src_reg != BPF_REG_4) { /* common case */ |
| 664 | EMIT1(0x51); /* push rcx */ |
| 665 | |
| 666 | /* mov rcx, src_reg */ |
| 667 | EMIT_mov(BPF_REG_4, src_reg); |
| 668 | } |
| 669 | |
| 670 | /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */ |
| 671 | if (BPF_CLASS(insn->code) == BPF_ALU64) |
| 672 | EMIT1(add_1mod(0x48, dst_reg)); |
| 673 | else if (is_ereg(dst_reg)) |
| 674 | EMIT1(add_1mod(0x40, dst_reg)); |
| 675 | |
| 676 | switch (BPF_OP(insn->code)) { |
| 677 | case BPF_LSH: b3 = 0xE0; break; |
| 678 | case BPF_RSH: b3 = 0xE8; break; |
| 679 | case BPF_ARSH: b3 = 0xF8; break; |
| 680 | } |
| 681 | EMIT2(0xD3, add_1reg(b3, dst_reg)); |
| 682 | |
| 683 | if (src_reg != BPF_REG_4) |
| 684 | EMIT1(0x59); /* pop rcx */ |
| 685 | |
| 686 | if (insn->dst_reg == BPF_REG_4) |
| 687 | /* mov dst_reg, r11 */ |
| 688 | EMIT_mov(insn->dst_reg, AUX_REG); |
| 689 | break; |
| 690 | |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 691 | case BPF_ALU | BPF_END | BPF_FROM_BE: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 692 | switch (imm32) { |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 693 | case 16: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 694 | /* Emit 'ror %ax, 8' to swap lower 2 bytes */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 695 | EMIT1(0x66); |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 696 | if (is_ereg(dst_reg)) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 697 | EMIT1(0x41); |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 698 | EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); |
Alexei Starovoitov | 343f845 | 2015-05-11 23:25:16 -0700 | [diff] [blame] | 699 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 700 | /* Emit 'movzwl eax, ax' */ |
Alexei Starovoitov | 343f845 | 2015-05-11 23:25:16 -0700 | [diff] [blame] | 701 | if (is_ereg(dst_reg)) |
| 702 | EMIT3(0x45, 0x0F, 0xB7); |
| 703 | else |
| 704 | EMIT2(0x0F, 0xB7); |
| 705 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 706 | break; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 707 | case 32: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 708 | /* Emit 'bswap eax' to swap lower 4 bytes */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 709 | if (is_ereg(dst_reg)) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 710 | EMIT2(0x41, 0x0F); |
| 711 | else |
| 712 | EMIT1(0x0F); |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 713 | EMIT1(add_1reg(0xC8, dst_reg)); |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 714 | break; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 715 | case 64: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 716 | /* Emit 'bswap rax' to swap 8 bytes */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 717 | EMIT3(add_1mod(0x48, dst_reg), 0x0F, |
| 718 | add_1reg(0xC8, dst_reg)); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 719 | break; |
| 720 | } |
| 721 | break; |
| 722 | |
| 723 | case BPF_ALU | BPF_END | BPF_FROM_LE: |
Alexei Starovoitov | 343f845 | 2015-05-11 23:25:16 -0700 | [diff] [blame] | 724 | switch (imm32) { |
| 725 | case 16: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 726 | /* |
| 727 | * Emit 'movzwl eax, ax' to zero extend 16-bit |
Alexei Starovoitov | 343f845 | 2015-05-11 23:25:16 -0700 | [diff] [blame] | 728 | * into 64 bit |
| 729 | */ |
| 730 | if (is_ereg(dst_reg)) |
| 731 | EMIT3(0x45, 0x0F, 0xB7); |
| 732 | else |
| 733 | EMIT2(0x0F, 0xB7); |
| 734 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); |
| 735 | break; |
| 736 | case 32: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 737 | /* Emit 'mov eax, eax' to clear upper 32-bits */ |
Alexei Starovoitov | 343f845 | 2015-05-11 23:25:16 -0700 | [diff] [blame] | 738 | if (is_ereg(dst_reg)) |
| 739 | EMIT1(0x45); |
| 740 | EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); |
| 741 | break; |
| 742 | case 64: |
| 743 | /* nop */ |
| 744 | break; |
| 745 | } |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 746 | break; |
| 747 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 748 | /* ST: *(u8*)(dst_reg + off) = imm */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 749 | case BPF_ST | BPF_MEM | BPF_B: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 750 | if (is_ereg(dst_reg)) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 751 | EMIT2(0x41, 0xC6); |
| 752 | else |
| 753 | EMIT1(0xC6); |
| 754 | goto st; |
| 755 | case BPF_ST | BPF_MEM | BPF_H: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 756 | if (is_ereg(dst_reg)) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 757 | EMIT3(0x66, 0x41, 0xC7); |
| 758 | else |
| 759 | EMIT2(0x66, 0xC7); |
| 760 | goto st; |
| 761 | case BPF_ST | BPF_MEM | BPF_W: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 762 | if (is_ereg(dst_reg)) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 763 | EMIT2(0x41, 0xC7); |
| 764 | else |
| 765 | EMIT1(0xC7); |
| 766 | goto st; |
| 767 | case BPF_ST | BPF_MEM | BPF_DW: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 768 | EMIT2(add_1mod(0x48, dst_reg), 0xC7); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 769 | |
| 770 | st: if (is_imm8(insn->off)) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 771 | EMIT2(add_1reg(0x40, dst_reg), insn->off); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 772 | else |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 773 | EMIT1_off32(add_1reg(0x80, dst_reg), insn->off); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 774 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 775 | EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 776 | break; |
| 777 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 778 | /* STX: *(u8*)(dst_reg + off) = src_reg */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 779 | case BPF_STX | BPF_MEM | BPF_B: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 780 | /* Emit 'mov byte ptr [rax + off], al' */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 781 | if (is_ereg(dst_reg) || is_ereg(src_reg) || |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 782 | /* We have to add extra byte for x86 SIL, DIL regs */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 783 | src_reg == BPF_REG_1 || src_reg == BPF_REG_2) |
| 784 | EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 785 | else |
| 786 | EMIT1(0x88); |
| 787 | goto stx; |
| 788 | case BPF_STX | BPF_MEM | BPF_H: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 789 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
| 790 | EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 791 | else |
| 792 | EMIT2(0x66, 0x89); |
| 793 | goto stx; |
| 794 | case BPF_STX | BPF_MEM | BPF_W: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 795 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
| 796 | EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 797 | else |
| 798 | EMIT1(0x89); |
| 799 | goto stx; |
| 800 | case BPF_STX | BPF_MEM | BPF_DW: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 801 | EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 802 | stx: if (is_imm8(insn->off)) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 803 | EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 804 | else |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 805 | EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 806 | insn->off); |
| 807 | break; |
| 808 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 809 | /* LDX: dst_reg = *(u8*)(src_reg + off) */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 810 | case BPF_LDX | BPF_MEM | BPF_B: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 811 | /* Emit 'movzx rax, byte ptr [rax + off]' */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 812 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 813 | goto ldx; |
| 814 | case BPF_LDX | BPF_MEM | BPF_H: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 815 | /* Emit 'movzx rax, word ptr [rax + off]' */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 816 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 817 | goto ldx; |
| 818 | case BPF_LDX | BPF_MEM | BPF_W: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 819 | /* Emit 'mov eax, dword ptr [rax+0x14]' */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 820 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
| 821 | EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 822 | else |
| 823 | EMIT1(0x8B); |
| 824 | goto ldx; |
| 825 | case BPF_LDX | BPF_MEM | BPF_DW: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 826 | /* Emit 'mov rax, qword ptr [rax+0x14]' */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 827 | EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 828 | ldx: /* |
| 829 | * If insn->off == 0 we can save one extra byte, but |
| 830 | * special case of x86 R13 which always needs an offset |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 831 | * is not worth the hassle |
| 832 | */ |
| 833 | if (is_imm8(insn->off)) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 834 | EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 835 | else |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 836 | EMIT1_off32(add_2reg(0x80, src_reg, dst_reg), |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 837 | insn->off); |
| 838 | break; |
| 839 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 840 | /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 841 | case BPF_STX | BPF_XADD | BPF_W: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 842 | /* Emit 'lock add dword ptr [rax + off], eax' */ |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 843 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
| 844 | EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 845 | else |
| 846 | EMIT2(0xF0, 0x01); |
| 847 | goto xadd; |
| 848 | case BPF_STX | BPF_XADD | BPF_DW: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 849 | EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 850 | xadd: if (is_imm8(insn->off)) |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 851 | EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 852 | else |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 853 | EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 854 | insn->off); |
| 855 | break; |
| 856 | |
| 857 | /* call */ |
| 858 | case BPF_JMP | BPF_CALL: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 859 | func = (u8 *) __bpf_call_base + imm32; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 860 | jmp_offset = func - (image + addrs[i]); |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 861 | if (!imm32 || !is_simm32(jmp_offset)) { |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 862 | pr_err("unsupported BPF func %d addr %p image %p\n", |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 863 | imm32, func, image); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 864 | return -EINVAL; |
| 865 | } |
| 866 | EMIT1_off32(0xE8, jmp_offset); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 867 | break; |
| 868 | |
Alexei Starovoitov | 71189fa | 2017-05-30 13:31:27 -0700 | [diff] [blame] | 869 | case BPF_JMP | BPF_TAIL_CALL: |
Alexei Starovoitov | b52f00e | 2015-05-19 16:59:04 -0700 | [diff] [blame] | 870 | emit_bpf_tail_call(&prog); |
| 871 | break; |
| 872 | |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 873 | /* cond jump */ |
| 874 | case BPF_JMP | BPF_JEQ | BPF_X: |
| 875 | case BPF_JMP | BPF_JNE | BPF_X: |
| 876 | case BPF_JMP | BPF_JGT | BPF_X: |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 877 | case BPF_JMP | BPF_JLT | BPF_X: |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 878 | case BPF_JMP | BPF_JGE | BPF_X: |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 879 | case BPF_JMP | BPF_JLE | BPF_X: |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 880 | case BPF_JMP | BPF_JSGT | BPF_X: |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 881 | case BPF_JMP | BPF_JSLT | BPF_X: |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 882 | case BPF_JMP | BPF_JSGE | BPF_X: |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 883 | case BPF_JMP | BPF_JSLE | BPF_X: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 884 | /* cmp dst_reg, src_reg */ |
| 885 | EMIT3(add_2mod(0x48, dst_reg, src_reg), 0x39, |
| 886 | add_2reg(0xC0, dst_reg, src_reg)); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 887 | goto emit_cond_jmp; |
| 888 | |
| 889 | case BPF_JMP | BPF_JSET | BPF_X: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 890 | /* test dst_reg, src_reg */ |
| 891 | EMIT3(add_2mod(0x48, dst_reg, src_reg), 0x85, |
| 892 | add_2reg(0xC0, dst_reg, src_reg)); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 893 | goto emit_cond_jmp; |
| 894 | |
| 895 | case BPF_JMP | BPF_JSET | BPF_K: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 896 | /* test dst_reg, imm32 */ |
| 897 | EMIT1(add_1mod(0x48, dst_reg)); |
| 898 | EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 899 | goto emit_cond_jmp; |
| 900 | |
| 901 | case BPF_JMP | BPF_JEQ | BPF_K: |
| 902 | case BPF_JMP | BPF_JNE | BPF_K: |
| 903 | case BPF_JMP | BPF_JGT | BPF_K: |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 904 | case BPF_JMP | BPF_JLT | BPF_K: |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 905 | case BPF_JMP | BPF_JGE | BPF_K: |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 906 | case BPF_JMP | BPF_JLE | BPF_K: |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 907 | case BPF_JMP | BPF_JSGT | BPF_K: |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 908 | case BPF_JMP | BPF_JSLT | BPF_K: |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 909 | case BPF_JMP | BPF_JSGE | BPF_K: |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 910 | case BPF_JMP | BPF_JSLE | BPF_K: |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 911 | /* cmp dst_reg, imm8/32 */ |
| 912 | EMIT1(add_1mod(0x48, dst_reg)); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 913 | |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 914 | if (is_imm8(imm32)) |
| 915 | EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 916 | else |
Alexei Starovoitov | e430f34 | 2014-06-06 14:46:06 -0700 | [diff] [blame] | 917 | EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 918 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 919 | emit_cond_jmp: /* Convert BPF opcode to x86 */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 920 | switch (BPF_OP(insn->code)) { |
| 921 | case BPF_JEQ: |
| 922 | jmp_cond = X86_JE; |
| 923 | break; |
| 924 | case BPF_JSET: |
| 925 | case BPF_JNE: |
| 926 | jmp_cond = X86_JNE; |
| 927 | break; |
| 928 | case BPF_JGT: |
| 929 | /* GT is unsigned '>', JA in x86 */ |
| 930 | jmp_cond = X86_JA; |
| 931 | break; |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 932 | case BPF_JLT: |
| 933 | /* LT is unsigned '<', JB in x86 */ |
| 934 | jmp_cond = X86_JB; |
| 935 | break; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 936 | case BPF_JGE: |
| 937 | /* GE is unsigned '>=', JAE in x86 */ |
| 938 | jmp_cond = X86_JAE; |
| 939 | break; |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 940 | case BPF_JLE: |
| 941 | /* LE is unsigned '<=', JBE in x86 */ |
| 942 | jmp_cond = X86_JBE; |
| 943 | break; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 944 | case BPF_JSGT: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 945 | /* Signed '>', GT in x86 */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 946 | jmp_cond = X86_JG; |
| 947 | break; |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 948 | case BPF_JSLT: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 949 | /* Signed '<', LT in x86 */ |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 950 | jmp_cond = X86_JL; |
| 951 | break; |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 952 | case BPF_JSGE: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 953 | /* Signed '>=', GE in x86 */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 954 | jmp_cond = X86_JGE; |
| 955 | break; |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 956 | case BPF_JSLE: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 957 | /* Signed '<=', LE in x86 */ |
Daniel Borkmann | 52afc51 | 2017-08-10 01:39:56 +0200 | [diff] [blame] | 958 | jmp_cond = X86_JLE; |
| 959 | break; |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 960 | default: /* to silence GCC warning */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 961 | return -EFAULT; |
| 962 | } |
| 963 | jmp_offset = addrs[i + insn->off] - addrs[i]; |
| 964 | if (is_imm8(jmp_offset)) { |
| 965 | EMIT2(jmp_cond, jmp_offset); |
| 966 | } else if (is_simm32(jmp_offset)) { |
| 967 | EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); |
| 968 | } else { |
| 969 | pr_err("cond_jmp gen bug %llx\n", jmp_offset); |
| 970 | return -EFAULT; |
| 971 | } |
| 972 | |
| 973 | break; |
| 974 | |
| 975 | case BPF_JMP | BPF_JA: |
Gianluca Borello | 1612a98 | 2018-04-25 05:42:16 +0000 | [diff] [blame] | 976 | if (insn->off == -1) |
| 977 | /* -1 jmp instructions will always jump |
| 978 | * backwards two bytes. Explicitly handling |
| 979 | * this case avoids wasting too many passes |
| 980 | * when there are long sequences of replaced |
| 981 | * dead code. |
| 982 | */ |
| 983 | jmp_offset = -2; |
| 984 | else |
| 985 | jmp_offset = addrs[i + insn->off] - addrs[i]; |
| 986 | |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 987 | if (!jmp_offset) |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 988 | /* Optimize out nop jumps */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 989 | break; |
| 990 | emit_jmp: |
| 991 | if (is_imm8(jmp_offset)) { |
| 992 | EMIT2(0xEB, jmp_offset); |
| 993 | } else if (is_simm32(jmp_offset)) { |
| 994 | EMIT1_off32(0xE9, jmp_offset); |
| 995 | } else { |
| 996 | pr_err("jmp gen bug %llx\n", jmp_offset); |
| 997 | return -EFAULT; |
| 998 | } |
| 999 | break; |
| 1000 | |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1001 | case BPF_JMP | BPF_EXIT: |
Alexei Starovoitov | 769e0de | 2014-11-29 14:46:13 -0800 | [diff] [blame] | 1002 | if (seen_exit) { |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1003 | jmp_offset = ctx->cleanup_addr - addrs[i]; |
| 1004 | goto emit_jmp; |
| 1005 | } |
Alexei Starovoitov | 769e0de | 2014-11-29 14:46:13 -0800 | [diff] [blame] | 1006 | seen_exit = true; |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 1007 | /* Update cleanup_addr */ |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1008 | ctx->cleanup_addr = proglen; |
Alexei Starovoitov | 177366b | 2017-05-30 13:31:34 -0700 | [diff] [blame] | 1009 | /* mov rbx, qword ptr [rbp+0] */ |
| 1010 | EMIT4(0x48, 0x8B, 0x5D, 0); |
| 1011 | /* mov r13, qword ptr [rbp+8] */ |
| 1012 | EMIT4(0x4C, 0x8B, 0x6D, 8); |
| 1013 | /* mov r14, qword ptr [rbp+16] */ |
| 1014 | EMIT4(0x4C, 0x8B, 0x75, 16); |
| 1015 | /* mov r15, qword ptr [rbp+24] */ |
| 1016 | EMIT4(0x4C, 0x8B, 0x7D, 24); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1017 | |
Alexei Starovoitov | 177366b | 2017-05-30 13:31:34 -0700 | [diff] [blame] | 1018 | /* add rbp, AUX_STACK_SPACE */ |
| 1019 | EMIT4(0x48, 0x83, 0xC5, AUX_STACK_SPACE); |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1020 | EMIT1(0xC9); /* leave */ |
| 1021 | EMIT1(0xC3); /* ret */ |
| 1022 | break; |
| 1023 | |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1024 | default: |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 1025 | /* |
| 1026 | * By design x86-64 JIT should support all BPF instructions. |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1027 | * This error will be seen if new instruction was added |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 1028 | * to the interpreter, but not to the JIT, or if there is |
| 1029 | * junk in bpf_prog. |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1030 | */ |
| 1031 | pr_err("bpf_jit: unknown opcode %02x\n", insn->code); |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1032 | return -EINVAL; |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1033 | } |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1034 | |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1035 | ilen = prog - temp; |
Alexei Starovoitov | e0ee9c1 | 2014-10-10 20:30:23 -0700 | [diff] [blame] | 1036 | if (ilen > BPF_MAX_INSN_SIZE) { |
Daniel Borkmann | 9383191 | 2017-02-16 22:24:49 +0100 | [diff] [blame] | 1037 | pr_err("bpf_jit: fatal insn size error\n"); |
Alexei Starovoitov | e0ee9c1 | 2014-10-10 20:30:23 -0700 | [diff] [blame] | 1038 | return -EFAULT; |
| 1039 | } |
| 1040 | |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1041 | if (image) { |
| 1042 | if (unlikely(proglen + ilen > oldproglen)) { |
Daniel Borkmann | 9383191 | 2017-02-16 22:24:49 +0100 | [diff] [blame] | 1043 | pr_err("bpf_jit: fatal error\n"); |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1044 | return -EFAULT; |
| 1045 | } |
| 1046 | memcpy(image + proglen, temp, ilen); |
| 1047 | } |
| 1048 | proglen += ilen; |
| 1049 | addrs[i] = proglen; |
| 1050 | prog = temp; |
| 1051 | } |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1052 | return proglen; |
| 1053 | } |
| 1054 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 1055 | struct x64_jit_data { |
| 1056 | struct bpf_binary_header *header; |
| 1057 | int *addrs; |
| 1058 | u8 *image; |
| 1059 | int proglen; |
| 1060 | struct jit_context ctx; |
| 1061 | }; |
| 1062 | |
Daniel Borkmann | d1c55ab | 2016-05-13 19:08:31 +0200 | [diff] [blame] | 1063 | struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) |
Alexei Starovoitov | 6225827 | 2014-05-13 19:50:46 -0700 | [diff] [blame] | 1064 | { |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1065 | struct bpf_binary_header *header = NULL; |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1066 | struct bpf_prog *tmp, *orig_prog = prog; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 1067 | struct x64_jit_data *jit_data; |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1068 | int proglen, oldproglen = 0; |
| 1069 | struct jit_context ctx = {}; |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1070 | bool tmp_blinded = false; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 1071 | bool extra_pass = false; |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1072 | u8 *image = NULL; |
| 1073 | int *addrs; |
| 1074 | int pass; |
| 1075 | int i; |
| 1076 | |
Alexei Starovoitov | 60b58afc | 2017-12-14 17:55:14 -0800 | [diff] [blame] | 1077 | if (!prog->jit_requested) |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1078 | return orig_prog; |
| 1079 | |
| 1080 | tmp = bpf_jit_blind_constants(prog); |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 1081 | /* |
| 1082 | * If blinding was requested and we failed during blinding, |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1083 | * we must fall back to the interpreter. |
| 1084 | */ |
| 1085 | if (IS_ERR(tmp)) |
| 1086 | return orig_prog; |
| 1087 | if (tmp != prog) { |
| 1088 | tmp_blinded = true; |
| 1089 | prog = tmp; |
| 1090 | } |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1091 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 1092 | jit_data = prog->aux->jit_data; |
| 1093 | if (!jit_data) { |
| 1094 | jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); |
| 1095 | if (!jit_data) { |
| 1096 | prog = orig_prog; |
| 1097 | goto out; |
| 1098 | } |
| 1099 | prog->aux->jit_data = jit_data; |
| 1100 | } |
| 1101 | addrs = jit_data->addrs; |
| 1102 | if (addrs) { |
| 1103 | ctx = jit_data->ctx; |
| 1104 | oldproglen = jit_data->proglen; |
| 1105 | image = jit_data->image; |
| 1106 | header = jit_data->header; |
| 1107 | extra_pass = true; |
| 1108 | goto skip_init_addrs; |
| 1109 | } |
Kees Cook | 6da2ec5 | 2018-06-12 13:55:00 -0700 | [diff] [blame] | 1110 | addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL); |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1111 | if (!addrs) { |
| 1112 | prog = orig_prog; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 1113 | goto out_addrs; |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1114 | } |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1115 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 1116 | /* |
| 1117 | * Before first pass, make a rough estimation of addrs[] |
| 1118 | * each BPF instruction is translated to less than 64 bytes |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1119 | */ |
| 1120 | for (proglen = 0, i = 0; i < prog->len; i++) { |
| 1121 | proglen += 64; |
| 1122 | addrs[i] = proglen; |
| 1123 | } |
| 1124 | ctx.cleanup_addr = proglen; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 1125 | skip_init_addrs: |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1126 | |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 1127 | /* |
| 1128 | * JITed image shrinks with every pass and the loop iterates |
| 1129 | * until the image stops shrinking. Very large BPF programs |
Alexei Starovoitov | 3f7352b | 2015-05-22 15:42:55 -0700 | [diff] [blame] | 1130 | * may converge on the last pass. In such case do one more |
Ingo Molnar | a2c7a98 | 2018-04-27 11:54:40 +0200 | [diff] [blame] | 1131 | * pass to emit the final image. |
Alexei Starovoitov | 3f7352b | 2015-05-22 15:42:55 -0700 | [diff] [blame] | 1132 | */ |
Daniel Borkmann | 6007b08 | 2018-03-07 22:10:01 +0100 | [diff] [blame] | 1133 | for (pass = 0; pass < 20 || image; pass++) { |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1134 | proglen = do_jit(prog, addrs, image, oldproglen, &ctx); |
| 1135 | if (proglen <= 0) { |
Daniel Borkmann | 3aab888 | 2018-05-02 20:12:22 +0200 | [diff] [blame] | 1136 | out_image: |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1137 | image = NULL; |
| 1138 | if (header) |
Daniel Borkmann | 738cbe7 | 2014-09-08 08:04:47 +0200 | [diff] [blame] | 1139 | bpf_jit_binary_free(header); |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1140 | prog = orig_prog; |
| 1141 | goto out_addrs; |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1142 | } |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1143 | if (image) { |
Alexei Starovoitov | e0ee9c1 | 2014-10-10 20:30:23 -0700 | [diff] [blame] | 1144 | if (proglen != oldproglen) { |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1145 | pr_err("bpf_jit: proglen=%d != oldproglen=%d\n", |
| 1146 | proglen, oldproglen); |
Daniel Borkmann | 3aab888 | 2018-05-02 20:12:22 +0200 | [diff] [blame] | 1147 | goto out_image; |
Alexei Starovoitov | e0ee9c1 | 2014-10-10 20:30:23 -0700 | [diff] [blame] | 1148 | } |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1149 | break; |
| 1150 | } |
| 1151 | if (proglen == oldproglen) { |
Daniel Borkmann | 738cbe7 | 2014-09-08 08:04:47 +0200 | [diff] [blame] | 1152 | header = bpf_jit_binary_alloc(proglen, &image, |
| 1153 | 1, jit_fill_hole); |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1154 | if (!header) { |
| 1155 | prog = orig_prog; |
| 1156 | goto out_addrs; |
| 1157 | } |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1158 | } |
| 1159 | oldproglen = proglen; |
Daniel Borkmann | 6007b08 | 2018-03-07 22:10:01 +0100 | [diff] [blame] | 1160 | cond_resched(); |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1161 | } |
Daniel Borkmann | 7961780 | 2013-03-21 22:22:03 +0100 | [diff] [blame] | 1162 | |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1163 | if (bpf_jit_enable > 1) |
Daniel Borkmann | 485d651 | 2015-07-30 12:42:48 +0200 | [diff] [blame] | 1164 | bpf_jit_dump(prog->len, proglen, pass + 1, image); |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1165 | |
| 1166 | if (image) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 1167 | if (!prog->is_func || extra_pass) { |
| 1168 | bpf_jit_binary_lock_ro(header); |
| 1169 | } else { |
| 1170 | jit_data->addrs = addrs; |
| 1171 | jit_data->ctx = ctx; |
| 1172 | jit_data->proglen = proglen; |
| 1173 | jit_data->image = image; |
| 1174 | jit_data->header = header; |
| 1175 | } |
Alexei Starovoitov | f3c2af7 | 2014-05-13 19:50:45 -0700 | [diff] [blame] | 1176 | prog->bpf_func = (void *)image; |
Daniel Borkmann | a91263d | 2015-09-30 01:41:50 +0200 | [diff] [blame] | 1177 | prog->jited = 1; |
Martin KaFai Lau | 783d28dd1 | 2017-06-05 12:15:51 -0700 | [diff] [blame] | 1178 | prog->jited_len = proglen; |
Daniel Borkmann | 9d5ecb0 | 2017-01-07 00:26:33 +0100 | [diff] [blame] | 1179 | } else { |
| 1180 | prog = orig_prog; |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1181 | } |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1182 | |
Daniel Borkmann | 39f56ca | 2018-05-02 20:12:23 +0200 | [diff] [blame] | 1183 | if (!image || !prog->is_func || extra_pass) { |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1184 | out_addrs: |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 1185 | kfree(addrs); |
| 1186 | kfree(jit_data); |
| 1187 | prog->aux->jit_data = NULL; |
| 1188 | } |
Daniel Borkmann | 959a757 | 2016-05-13 19:08:33 +0200 | [diff] [blame] | 1189 | out: |
| 1190 | if (tmp_blinded) |
| 1191 | bpf_jit_prog_release_other(prog, prog == orig_prog ? |
| 1192 | tmp : orig_prog); |
Daniel Borkmann | d1c55ab | 2016-05-13 19:08:31 +0200 | [diff] [blame] | 1193 | return prog; |
Eric Dumazet | 0a14842 | 2011-04-20 09:27:32 +0000 | [diff] [blame] | 1194 | } |