Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
Andy Lutomirski | 478dc89 | 2015-11-12 12:59:04 -0800 | [diff] [blame] | 2 | #include <linux/jump_label.h> |
Josh Poimboeuf | 8c1f755 | 2017-07-11 10:33:44 -0500 | [diff] [blame] | 3 | #include <asm/unwind_hints.h> |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 4 | #include <asm/cpufeatures.h> |
| 5 | #include <asm/page_types.h> |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 6 | #include <asm/percpu.h> |
| 7 | #include <asm/asm-offsets.h> |
| 8 | #include <asm/processor-flags.h> |
Andy Lutomirski | 478dc89 | 2015-11-12 12:59:04 -0800 | [diff] [blame] | 9 | |
Ingo Molnar | 0c2bd5a | 2008-01-30 13:32:49 +0100 | [diff] [blame] | 10 | /* |
Ingo Molnar | 063f891 | 2009-02-03 18:02:36 +0100 | [diff] [blame] | 11 | |
| 12 | x86 function call convention, 64-bit: |
| 13 | ------------------------------------- |
| 14 | arguments | callee-saved | extra caller-saved | return |
| 15 | [callee-clobbered] | | [callee-clobbered] | |
| 16 | --------------------------------------------------------------------------- |
| 17 | rdi rsi rdx rcx r8-9 | rbx rbp [*] r12-15 | r10-11 | rax, rdx [**] |
| 18 | |
| 19 | ( rsp is obviously invariant across normal function calls. (gcc can 'merge' |
| 20 | functions when it sees tail-call optimization possibilities) rflags is |
| 21 | clobbered. Leftover arguments are passed over the stack frame.) |
| 22 | |
| 23 | [*] In the frame-pointers case rbp is fixed to the stack frame. |
| 24 | |
| 25 | [**] for struct return values wider than 64 bits the return convention is a |
| 26 | bit more complex: up to 128 bits width we return small structures |
| 27 | straight in rax, rdx. For structures larger than that (3 words or |
| 28 | larger) the caller puts a pointer to an on-stack return struct |
| 29 | [allocated in the caller's stack frame] into the first argument - i.e. |
| 30 | into rdi. All other arguments shift up by one in this case. |
| 31 | Fortunately this case is rare in the kernel. |
| 32 | |
| 33 | For 32-bit we have the following conventions - kernel is built with |
| 34 | -mregparm=3 and -freg-struct-return: |
| 35 | |
| 36 | x86 function calling convention, 32-bit: |
| 37 | ---------------------------------------- |
| 38 | arguments | callee-saved | extra caller-saved | return |
| 39 | [callee-clobbered] | | [callee-clobbered] | |
| 40 | ------------------------------------------------------------------------- |
| 41 | eax edx ecx | ebx edi esi ebp [*] | <none> | eax, edx [**] |
| 42 | |
| 43 | ( here too esp is obviously invariant across normal function calls. eflags |
| 44 | is clobbered. Leftover arguments are passed over the stack frame. ) |
| 45 | |
| 46 | [*] In the frame-pointers case ebp is fixed to the stack frame. |
| 47 | |
| 48 | [**] We build with -freg-struct-return, which on 32-bit means similar |
| 49 | semantics as on 64-bit: edx can be used for a second return value |
| 50 | (i.e. covering integer and structure sizes up to 64 bits) - after that |
| 51 | it gets more complex and more expensive: 3-word or larger struct returns |
| 52 | get done in the caller's frame and the pointer to the return struct goes |
| 53 | into regparm0, i.e. eax - the other arguments shift up and the |
| 54 | function's register parameters degenerate to regparm=2 in essence. |
| 55 | |
| 56 | */ |
| 57 | |
Peter Zijlstra | 1a338ac | 2013-08-14 14:51:00 +0200 | [diff] [blame] | 58 | #ifdef CONFIG_X86_64 |
| 59 | |
Ingo Molnar | 063f891 | 2009-02-03 18:02:36 +0100 | [diff] [blame] | 60 | /* |
Tao Guo | 1b2b23d | 2012-09-26 04:28:22 -0400 | [diff] [blame] | 61 | * 64-bit system call stack frame layout defines and helpers, |
| 62 | * for assembly code: |
Ingo Molnar | 0c2bd5a | 2008-01-30 13:32:49 +0100 | [diff] [blame] | 63 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | |
Denys Vlasenko | 76f5df4 | 2015-02-26 14:40:27 -0800 | [diff] [blame] | 65 | /* The layout forms the "struct pt_regs" on the stack: */ |
| 66 | /* |
| 67 | * C ABI says these regs are callee-preserved. They aren't saved on kernel entry |
| 68 | * unless syscall needs a complete, fully filled "struct pt_regs". |
| 69 | */ |
| 70 | #define R15 0*8 |
| 71 | #define R14 1*8 |
| 72 | #define R13 2*8 |
| 73 | #define R12 3*8 |
| 74 | #define RBP 4*8 |
| 75 | #define RBX 5*8 |
| 76 | /* These regs are callee-clobbered. Always saved on kernel entry. */ |
| 77 | #define R11 6*8 |
| 78 | #define R10 7*8 |
| 79 | #define R9 8*8 |
| 80 | #define R8 9*8 |
| 81 | #define RAX 10*8 |
| 82 | #define RCX 11*8 |
| 83 | #define RDX 12*8 |
| 84 | #define RSI 13*8 |
| 85 | #define RDI 14*8 |
| 86 | /* |
| 87 | * On syscall entry, this is syscall#. On CPU exception, this is error code. |
| 88 | * On hw interrupt, it's IRQ number: |
| 89 | */ |
| 90 | #define ORIG_RAX 15*8 |
| 91 | /* Return frame for iretq */ |
| 92 | #define RIP 16*8 |
| 93 | #define CS 17*8 |
| 94 | #define EFLAGS 18*8 |
| 95 | #define RSP 19*8 |
| 96 | #define SS 20*8 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | |
Denys Vlasenko | 911d2bb | 2015-02-26 14:40:36 -0800 | [diff] [blame] | 98 | #define SIZEOF_PTREGS 21*8 |
| 99 | |
Dominik Brodowski | 9e809d1 | 2018-02-14 18:59:23 +0100 | [diff] [blame] | 100 | .macro PUSH_AND_CLEAR_REGS rdx=%rdx rax=%rax save_ret=0 |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 101 | /* |
| 102 | * Push registers and sanitize registers of values that a |
| 103 | * speculation attack might otherwise want to exploit. The |
| 104 | * lower registers are likely clobbered well before they |
| 105 | * could be put to use in a speculative execution gadget. |
| 106 | * Interleave XOR with PUSH for better uop scheduling: |
| 107 | */ |
Dominik Brodowski | 9e809d1 | 2018-02-14 18:59:23 +0100 | [diff] [blame] | 108 | .if \save_ret |
| 109 | pushq %rsi /* pt_regs->si */ |
| 110 | movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */ |
| 111 | movq %rdi, 8(%rsp) /* pt_regs->di (overwriting original return address) */ |
| 112 | .else |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 113 | pushq %rdi /* pt_regs->di */ |
| 114 | pushq %rsi /* pt_regs->si */ |
Dominik Brodowski | 9e809d1 | 2018-02-14 18:59:23 +0100 | [diff] [blame] | 115 | .endif |
Dominik Brodowski | 30907fd | 2018-02-11 11:49:46 +0100 | [diff] [blame] | 116 | pushq \rdx /* pt_regs->dx */ |
Dominik Brodowski | 6dc936f | 2018-04-05 11:53:06 +0200 | [diff] [blame] | 117 | xorl %edx, %edx /* nospec dx */ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 118 | pushq %rcx /* pt_regs->cx */ |
Dominik Brodowski | 6dc936f | 2018-04-05 11:53:06 +0200 | [diff] [blame] | 119 | xorl %ecx, %ecx /* nospec cx */ |
Dominik Brodowski | 30907fd | 2018-02-11 11:49:46 +0100 | [diff] [blame] | 120 | pushq \rax /* pt_regs->ax */ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 121 | pushq %r8 /* pt_regs->r8 */ |
Dominik Brodowski | ced5d0b | 2018-02-14 18:59:24 +0100 | [diff] [blame] | 122 | xorl %r8d, %r8d /* nospec r8 */ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 123 | pushq %r9 /* pt_regs->r9 */ |
Dominik Brodowski | ced5d0b | 2018-02-14 18:59:24 +0100 | [diff] [blame] | 124 | xorl %r9d, %r9d /* nospec r9 */ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 125 | pushq %r10 /* pt_regs->r10 */ |
Dominik Brodowski | ced5d0b | 2018-02-14 18:59:24 +0100 | [diff] [blame] | 126 | xorl %r10d, %r10d /* nospec r10 */ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 127 | pushq %r11 /* pt_regs->r11 */ |
Dominik Brodowski | ced5d0b | 2018-02-14 18:59:24 +0100 | [diff] [blame] | 128 | xorl %r11d, %r11d /* nospec r11*/ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 129 | pushq %rbx /* pt_regs->rbx */ |
| 130 | xorl %ebx, %ebx /* nospec rbx*/ |
| 131 | pushq %rbp /* pt_regs->rbp */ |
| 132 | xorl %ebp, %ebp /* nospec rbp*/ |
| 133 | pushq %r12 /* pt_regs->r12 */ |
Dominik Brodowski | ced5d0b | 2018-02-14 18:59:24 +0100 | [diff] [blame] | 134 | xorl %r12d, %r12d /* nospec r12*/ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 135 | pushq %r13 /* pt_regs->r13 */ |
Dominik Brodowski | ced5d0b | 2018-02-14 18:59:24 +0100 | [diff] [blame] | 136 | xorl %r13d, %r13d /* nospec r13*/ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 137 | pushq %r14 /* pt_regs->r14 */ |
Dominik Brodowski | ced5d0b | 2018-02-14 18:59:24 +0100 | [diff] [blame] | 138 | xorl %r14d, %r14d /* nospec r14*/ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 139 | pushq %r15 /* pt_regs->r15 */ |
Dominik Brodowski | ced5d0b | 2018-02-14 18:59:24 +0100 | [diff] [blame] | 140 | xorl %r15d, %r15d /* nospec r15*/ |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 141 | UNWIND_HINT_REGS |
Dominik Brodowski | 9e809d1 | 2018-02-14 18:59:23 +0100 | [diff] [blame] | 142 | .if \save_ret |
| 143 | pushq %rsi /* return address on top of stack */ |
| 144 | .endif |
Dominik Brodowski | 92816f5 | 2018-02-11 11:49:48 +0100 | [diff] [blame] | 145 | .endm |
Dominik Brodowski | 3f01dae | 2018-02-11 11:49:45 +0100 | [diff] [blame] | 146 | |
Dominik Brodowski | 92816f5 | 2018-02-11 11:49:48 +0100 | [diff] [blame] | 147 | .macro POP_REGS pop_rdi=1 skip_r11rcx=0 |
Andy Lutomirski | e872045 | 2017-11-02 00:59:01 -0700 | [diff] [blame] | 148 | popq %r15 |
| 149 | popq %r14 |
| 150 | popq %r13 |
| 151 | popq %r12 |
| 152 | popq %rbp |
| 153 | popq %rbx |
Dominik Brodowski | 502af0d | 2018-02-11 11:49:43 +0100 | [diff] [blame] | 154 | .if \skip_r11rcx |
| 155 | popq %rsi |
| 156 | .else |
Andy Lutomirski | e872045 | 2017-11-02 00:59:01 -0700 | [diff] [blame] | 157 | popq %r11 |
Dominik Brodowski | 502af0d | 2018-02-11 11:49:43 +0100 | [diff] [blame] | 158 | .endif |
Andy Lutomirski | e872045 | 2017-11-02 00:59:01 -0700 | [diff] [blame] | 159 | popq %r10 |
| 160 | popq %r9 |
| 161 | popq %r8 |
| 162 | popq %rax |
Dominik Brodowski | 502af0d | 2018-02-11 11:49:43 +0100 | [diff] [blame] | 163 | .if \skip_r11rcx |
| 164 | popq %rsi |
| 165 | .else |
Andy Lutomirski | e872045 | 2017-11-02 00:59:01 -0700 | [diff] [blame] | 166 | popq %rcx |
Dominik Brodowski | 502af0d | 2018-02-11 11:49:43 +0100 | [diff] [blame] | 167 | .endif |
Andy Lutomirski | e872045 | 2017-11-02 00:59:01 -0700 | [diff] [blame] | 168 | popq %rdx |
| 169 | popq %rsi |
Dominik Brodowski | 502af0d | 2018-02-11 11:49:43 +0100 | [diff] [blame] | 170 | .if \pop_rdi |
Andy Lutomirski | e872045 | 2017-11-02 00:59:01 -0700 | [diff] [blame] | 171 | popq %rdi |
Dominik Brodowski | 502af0d | 2018-02-11 11:49:43 +0100 | [diff] [blame] | 172 | .endif |
Dominik Brodowski | 92816f5 | 2018-02-11 11:49:48 +0100 | [diff] [blame] | 173 | .endm |
Peter Zijlstra | 1a338ac | 2013-08-14 14:51:00 +0200 | [diff] [blame] | 174 | |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 175 | #ifdef CONFIG_PAGE_TABLE_ISOLATION |
| 176 | |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 177 | /* |
| 178 | * PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two |
| 179 | * halves: |
| 180 | */ |
Thomas Gleixner | f10ee3d | 2018-01-14 00:23:57 +0100 | [diff] [blame] | 181 | #define PTI_USER_PGTABLE_BIT PAGE_SHIFT |
| 182 | #define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) |
| 183 | #define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT |
| 184 | #define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) |
| 185 | #define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGTABLE_MASK) |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 186 | |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 187 | .macro SET_NOFLUSH_BIT reg:req |
| 188 | bts $X86_CR3_PCID_NOFLUSH_BIT, \reg |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 189 | .endm |
| 190 | |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 191 | .macro ADJUST_KERNEL_CR3 reg:req |
| 192 | ALTERNATIVE "", "SET_NOFLUSH_BIT \reg", X86_FEATURE_PCID |
| 193 | /* Clear PCID and "PAGE_TABLE_ISOLATION bit", point CR3 at kernel pagetables: */ |
Thomas Gleixner | f10ee3d | 2018-01-14 00:23:57 +0100 | [diff] [blame] | 194 | andq $(~PTI_USER_PGTABLE_AND_PCID_MASK), \reg |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 195 | .endm |
| 196 | |
| 197 | .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req |
Thomas Gleixner | aa8c624 | 2017-12-04 15:07:36 +0100 | [diff] [blame] | 198 | ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 199 | mov %cr3, \scratch_reg |
| 200 | ADJUST_KERNEL_CR3 \scratch_reg |
| 201 | mov \scratch_reg, %cr3 |
Thomas Gleixner | aa8c624 | 2017-12-04 15:07:36 +0100 | [diff] [blame] | 202 | .Lend_\@: |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 203 | .endm |
| 204 | |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 205 | #define THIS_CPU_user_pcid_flush_mask \ |
| 206 | PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask |
| 207 | |
| 208 | .macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req |
Thomas Gleixner | aa8c624 | 2017-12-04 15:07:36 +0100 | [diff] [blame] | 209 | ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 210 | mov %cr3, \scratch_reg |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 211 | |
| 212 | ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID |
| 213 | |
| 214 | /* |
| 215 | * Test if the ASID needs a flush. |
| 216 | */ |
| 217 | movq \scratch_reg, \scratch_reg2 |
| 218 | andq $(0x7FF), \scratch_reg /* mask ASID */ |
| 219 | bt \scratch_reg, THIS_CPU_user_pcid_flush_mask |
| 220 | jnc .Lnoflush_\@ |
| 221 | |
| 222 | /* Flush needed, clear the bit */ |
| 223 | btr \scratch_reg, THIS_CPU_user_pcid_flush_mask |
| 224 | movq \scratch_reg2, \scratch_reg |
Thomas Gleixner | f10ee3d | 2018-01-14 00:23:57 +0100 | [diff] [blame] | 225 | jmp .Lwrcr3_pcid_\@ |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 226 | |
| 227 | .Lnoflush_\@: |
| 228 | movq \scratch_reg2, \scratch_reg |
| 229 | SET_NOFLUSH_BIT \scratch_reg |
| 230 | |
Thomas Gleixner | f10ee3d | 2018-01-14 00:23:57 +0100 | [diff] [blame] | 231 | .Lwrcr3_pcid_\@: |
| 232 | /* Flip the ASID to the user version */ |
| 233 | orq $(PTI_USER_PCID_MASK), \scratch_reg |
| 234 | |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 235 | .Lwrcr3_\@: |
Thomas Gleixner | f10ee3d | 2018-01-14 00:23:57 +0100 | [diff] [blame] | 236 | /* Flip the PGD to the user version */ |
| 237 | orq $(PTI_USER_PGTABLE_MASK), \scratch_reg |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 238 | mov \scratch_reg, %cr3 |
Thomas Gleixner | aa8c624 | 2017-12-04 15:07:36 +0100 | [diff] [blame] | 239 | .Lend_\@: |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 240 | .endm |
| 241 | |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 242 | .macro SWITCH_TO_USER_CR3_STACK scratch_reg:req |
| 243 | pushq %rax |
| 244 | SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax |
| 245 | popq %rax |
| 246 | .endm |
| 247 | |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 248 | .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req |
Thomas Gleixner | aa8c624 | 2017-12-04 15:07:36 +0100 | [diff] [blame] | 249 | ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 250 | movq %cr3, \scratch_reg |
| 251 | movq \scratch_reg, \save_reg |
| 252 | /* |
Thomas Gleixner | f10ee3d | 2018-01-14 00:23:57 +0100 | [diff] [blame] | 253 | * Test the user pagetable bit. If set, then the user page tables |
| 254 | * are active. If clear CR3 already has the kernel page table |
| 255 | * active. |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 256 | */ |
Thomas Gleixner | f10ee3d | 2018-01-14 00:23:57 +0100 | [diff] [blame] | 257 | bt $PTI_USER_PGTABLE_BIT, \scratch_reg |
| 258 | jnc .Ldone_\@ |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 259 | |
| 260 | ADJUST_KERNEL_CR3 \scratch_reg |
| 261 | movq \scratch_reg, %cr3 |
| 262 | |
| 263 | .Ldone_\@: |
| 264 | .endm |
| 265 | |
Peter Zijlstra | 21e9445 | 2017-12-04 15:08:00 +0100 | [diff] [blame] | 266 | .macro RESTORE_CR3 scratch_reg:req save_reg:req |
Thomas Gleixner | aa8c624 | 2017-12-04 15:07:36 +0100 | [diff] [blame] | 267 | ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI |
Peter Zijlstra | 21e9445 | 2017-12-04 15:08:00 +0100 | [diff] [blame] | 268 | |
| 269 | ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID |
| 270 | |
| 271 | /* |
| 272 | * KERNEL pages can always resume with NOFLUSH as we do |
| 273 | * explicit flushes. |
| 274 | */ |
Thomas Gleixner | f10ee3d | 2018-01-14 00:23:57 +0100 | [diff] [blame] | 275 | bt $PTI_USER_PGTABLE_BIT, \save_reg |
Peter Zijlstra | 21e9445 | 2017-12-04 15:08:00 +0100 | [diff] [blame] | 276 | jnc .Lnoflush_\@ |
| 277 | |
| 278 | /* |
| 279 | * Check if there's a pending flush for the user ASID we're |
| 280 | * about to set. |
| 281 | */ |
| 282 | movq \save_reg, \scratch_reg |
| 283 | andq $(0x7FF), \scratch_reg |
| 284 | bt \scratch_reg, THIS_CPU_user_pcid_flush_mask |
| 285 | jnc .Lnoflush_\@ |
| 286 | |
| 287 | btr \scratch_reg, THIS_CPU_user_pcid_flush_mask |
| 288 | jmp .Lwrcr3_\@ |
| 289 | |
| 290 | .Lnoflush_\@: |
| 291 | SET_NOFLUSH_BIT \save_reg |
| 292 | |
| 293 | .Lwrcr3_\@: |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 294 | /* |
| 295 | * The CR3 write could be avoided when not changing its value, |
| 296 | * but would require a CR3 read *and* a scratch register. |
| 297 | */ |
| 298 | movq \save_reg, %cr3 |
Thomas Gleixner | aa8c624 | 2017-12-04 15:07:36 +0100 | [diff] [blame] | 299 | .Lend_\@: |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 300 | .endm |
| 301 | |
| 302 | #else /* CONFIG_PAGE_TABLE_ISOLATION=n: */ |
| 303 | |
| 304 | .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req |
| 305 | .endm |
Peter Zijlstra | 6fd166a | 2017-12-04 15:07:59 +0100 | [diff] [blame] | 306 | .macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req |
| 307 | .endm |
| 308 | .macro SWITCH_TO_USER_CR3_STACK scratch_reg:req |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 309 | .endm |
| 310 | .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req |
| 311 | .endm |
Peter Zijlstra | 21e9445 | 2017-12-04 15:08:00 +0100 | [diff] [blame] | 312 | .macro RESTORE_CR3 scratch_reg:req save_reg:req |
Dave Hansen | 8a09317 | 2017-12-04 15:07:35 +0100 | [diff] [blame] | 313 | .endm |
| 314 | |
| 315 | #endif |
| 316 | |
Alexander Popov | afaef01 | 2018-08-17 01:16:58 +0300 | [diff] [blame] | 317 | .macro STACKLEAK_ERASE_NOCLOBBER |
| 318 | #ifdef CONFIG_GCC_PLUGIN_STACKLEAK |
| 319 | PUSH_AND_CLEAR_REGS |
| 320 | call stackleak_erase |
| 321 | POP_REGS |
| 322 | #endif |
| 323 | .endm |
| 324 | |
Peter Zijlstra | 1a338ac | 2013-08-14 14:51:00 +0200 | [diff] [blame] | 325 | #endif /* CONFIG_X86_64 */ |
| 326 | |
Alexander Popov | afaef01 | 2018-08-17 01:16:58 +0300 | [diff] [blame] | 327 | .macro STACKLEAK_ERASE |
| 328 | #ifdef CONFIG_GCC_PLUGIN_STACKLEAK |
| 329 | call stackleak_erase |
| 330 | #endif |
| 331 | .endm |
| 332 | |
Andy Lutomirski | 478dc89 | 2015-11-12 12:59:04 -0800 | [diff] [blame] | 333 | /* |
| 334 | * This does 'call enter_from_user_mode' unless we can avoid it based on |
| 335 | * kernel config or using the static jump infrastructure. |
| 336 | */ |
| 337 | .macro CALL_enter_from_user_mode |
| 338 | #ifdef CONFIG_CONTEXT_TRACKING |
Masahiro Yamada | e9666d1 | 2018-12-31 00:14:15 +0900 | [diff] [blame] | 339 | #ifdef CONFIG_JUMP_LABEL |
Ingo Molnar | e769742 | 2018-12-19 11:20:23 +0100 | [diff] [blame] | 340 | STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0 |
Andy Lutomirski | 478dc89 | 2015-11-12 12:59:04 -0800 | [diff] [blame] | 341 | #endif |
| 342 | call enter_from_user_mode |
| 343 | .Lafter_call_\@: |
| 344 | #endif |
| 345 | .endm |