Thomas Gleixner | caab277 | 2019-06-03 07:44:50 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 2 | /* |
| 3 | * Based on arch/arm/kernel/process.c |
| 4 | * |
| 5 | * Original Copyright (C) 1995 Linus Torvalds |
| 6 | * Copyright (C) 1996-2000 Russell King - Converted to ARM. |
| 7 | * Copyright (C) 2012 ARM Ltd. |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 8 | */ |
AKASHI Takahiro | fd92d4a | 2014-04-30 10:51:32 +0100 | [diff] [blame] | 9 | #include <linux/compat.h> |
Ard Biesheuvel | 60c0d45 | 2015-03-06 15:49:24 +0100 | [diff] [blame] | 10 | #include <linux/efi.h> |
Dave Martin | ab7876a | 2020-03-16 16:50:47 +0000 | [diff] [blame] | 11 | #include <linux/elf.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 12 | #include <linux/export.h> |
| 13 | #include <linux/sched.h> |
Ingo Molnar | b17b015 | 2017-02-08 18:51:35 +0100 | [diff] [blame] | 14 | #include <linux/sched/debug.h> |
Ingo Molnar | 2993002 | 2017-02-08 18:51:36 +0100 | [diff] [blame] | 15 | #include <linux/sched/task.h> |
Ingo Molnar | 68db0cf | 2017-02-08 18:51:37 +0100 | [diff] [blame] | 16 | #include <linux/sched/task_stack.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 17 | #include <linux/kernel.h> |
Dave Martin | ab7876a | 2020-03-16 16:50:47 +0000 | [diff] [blame] | 18 | #include <linux/mman.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 19 | #include <linux/mm.h> |
Will Deacon | 780c083 | 2020-09-28 14:03:00 +0100 | [diff] [blame] | 20 | #include <linux/nospec.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 21 | #include <linux/stddef.h> |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 22 | #include <linux/sysctl.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 23 | #include <linux/unistd.h> |
| 24 | #include <linux/user.h> |
| 25 | #include <linux/delay.h> |
| 26 | #include <linux/reboot.h> |
| 27 | #include <linux/interrupt.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 28 | #include <linux/init.h> |
| 29 | #include <linux/cpu.h> |
| 30 | #include <linux/elfcore.h> |
| 31 | #include <linux/pm.h> |
| 32 | #include <linux/tick.h> |
| 33 | #include <linux/utsname.h> |
| 34 | #include <linux/uaccess.h> |
| 35 | #include <linux/random.h> |
| 36 | #include <linux/hw_breakpoint.h> |
| 37 | #include <linux/personality.h> |
| 38 | #include <linux/notifier.h> |
Jisheng Zhang | 096b322 | 2015-09-16 22:23:21 +0800 | [diff] [blame] | 39 | #include <trace/events/power.h> |
Mark Rutland | c02433d | 2016-11-03 20:23:13 +0000 | [diff] [blame] | 40 | #include <linux/percpu.h> |
Dave Martin | bc0ee47 | 2017-10-31 15:51:05 +0000 | [diff] [blame] | 41 | #include <linux/thread_info.h> |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 42 | #include <linux/prctl.h> |
Madhavan T. Venkataraman | 4f62bb7 | 2021-11-29 14:28:45 +0000 | [diff] [blame] | 43 | #include <linux/stacktrace.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 44 | |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 45 | #include <asm/alternative.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 46 | #include <asm/compat.h> |
Julien Thierry | 19c95f2 | 2019-10-15 18:25:44 +0100 | [diff] [blame] | 47 | #include <asm/cpufeature.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 48 | #include <asm/cacheflush.h> |
James Morse | d085441 | 2016-10-18 11:27:48 +0100 | [diff] [blame] | 49 | #include <asm/exec.h> |
Will Deacon | ec45d1c | 2013-01-17 12:31:45 +0000 | [diff] [blame] | 50 | #include <asm/fpsimd.h> |
| 51 | #include <asm/mmu_context.h> |
Vincenzo Frascino | 637ec83 | 2019-09-16 11:51:17 +0100 | [diff] [blame] | 52 | #include <asm/mte.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 53 | #include <asm/processor.h> |
Mark Rutland | 7503197 | 2018-12-07 18:39:25 +0000 | [diff] [blame] | 54 | #include <asm/pointer_auth.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 55 | #include <asm/stacktrace.h> |
Maninder Singh | baa9637 | 2021-03-24 12:24:58 +0530 | [diff] [blame] | 56 | #include <asm/switch_to.h> |
| 57 | #include <asm/system_misc.h> |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 58 | |
Ard Biesheuvel | 0a1213f | 2018-12-12 13:08:44 +0100 | [diff] [blame] | 59 | #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) |
Laura Abbott | c0c264a | 2014-06-25 23:55:03 +0100 | [diff] [blame] | 60 | #include <linux/stackprotector.h> |
Dan Li | 9fcb2e9 | 2021-09-14 17:44:02 +0800 | [diff] [blame] | 61 | unsigned long __stack_chk_guard __ro_after_init; |
Laura Abbott | c0c264a | 2014-06-25 23:55:03 +0100 | [diff] [blame] | 62 | EXPORT_SYMBOL(__stack_chk_guard); |
| 63 | #endif |
| 64 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 65 | /* |
| 66 | * Function pointers to optional machine specific functions |
| 67 | */ |
| 68 | void (*pm_power_off)(void); |
| 69 | EXPORT_SYMBOL_GPL(pm_power_off); |
| 70 | |
Mark Rutland | 9327e2c | 2013-10-24 20:30:18 +0100 | [diff] [blame] | 71 | #ifdef CONFIG_HOTPLUG_CPU |
Josh Poimboeuf | 071c44e | 2023-02-13 23:05:58 -0800 | [diff] [blame] | 72 | void __noreturn arch_cpu_idle_dead(void) |
Mark Rutland | 9327e2c | 2013-10-24 20:30:18 +0100 | [diff] [blame] | 73 | { |
| 74 | cpu_die(); |
| 75 | } |
| 76 | #endif |
| 77 | |
Arun KS | 90f51a0 | 2014-05-07 02:41:22 +0100 | [diff] [blame] | 78 | /* |
| 79 | * Called by kexec, immediately prior to machine_kexec(). |
| 80 | * |
| 81 | * This must completely disable all secondary CPUs; simply causing those CPUs |
| 82 | * to execute e.g. a RAM-based pin loop is not sufficient. This allows the |
| 83 | * kexec'd kernel to use any and all RAM as it sees fit, without having to |
| 84 | * avoid any code or data used by any SW CPU pin loop. The CPU hotplug |
Qais Yousef | d66b16f | 2020-03-23 13:50:59 +0000 | [diff] [blame] | 85 | * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this. |
Arun KS | 90f51a0 | 2014-05-07 02:41:22 +0100 | [diff] [blame] | 86 | */ |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 87 | void machine_shutdown(void) |
| 88 | { |
Qais Yousef | 5efbe6a6 | 2020-03-23 13:51:00 +0000 | [diff] [blame] | 89 | smp_shutdown_nonboot_cpus(reboot_cpu); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 90 | } |
| 91 | |
Arun KS | 90f51a0 | 2014-05-07 02:41:22 +0100 | [diff] [blame] | 92 | /* |
| 93 | * Halting simply requires that the secondary CPUs stop performing any |
| 94 | * activity (executing tasks, handling interrupts). smp_send_stop() |
| 95 | * achieves this. |
| 96 | */ |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 97 | void machine_halt(void) |
| 98 | { |
Arun KS | b9acc49 | 2014-05-07 02:41:23 +0100 | [diff] [blame] | 99 | local_irq_disable(); |
Arun KS | 90f51a0 | 2014-05-07 02:41:22 +0100 | [diff] [blame] | 100 | smp_send_stop(); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 101 | while (1); |
| 102 | } |
| 103 | |
Arun KS | 90f51a0 | 2014-05-07 02:41:22 +0100 | [diff] [blame] | 104 | /* |
| 105 | * Power-off simply requires that the secondary CPUs stop performing any |
| 106 | * activity (executing tasks, handling interrupts). smp_send_stop() |
| 107 | * achieves this. When the system power is turned off, it will take all CPUs |
| 108 | * with it. |
| 109 | */ |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 110 | void machine_power_off(void) |
| 111 | { |
Arun KS | b9acc49 | 2014-05-07 02:41:23 +0100 | [diff] [blame] | 112 | local_irq_disable(); |
Arun KS | 90f51a0 | 2014-05-07 02:41:22 +0100 | [diff] [blame] | 113 | smp_send_stop(); |
Dmitry Osipenko | 0c64991 | 2022-05-10 02:32:20 +0300 | [diff] [blame] | 114 | do_kernel_power_off(); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 115 | } |
| 116 | |
Arun KS | 90f51a0 | 2014-05-07 02:41:22 +0100 | [diff] [blame] | 117 | /* |
| 118 | * Restart requires that the secondary CPUs stop performing any activity |
Mark Rutland | 68234df | 2015-04-20 10:24:35 +0100 | [diff] [blame] | 119 | * while the primary CPU resets the system. Systems with multiple CPUs must |
Arun KS | 90f51a0 | 2014-05-07 02:41:22 +0100 | [diff] [blame] | 120 | * provide a HW restart implementation, to ensure that all CPUs reset at once. |
| 121 | * This is required so that any code running after reset on the primary CPU |
| 122 | * doesn't have to co-ordinate with other CPUs to ensure they aren't still |
| 123 | * executing pre-reset code, and using RAM that the primary CPU's code wishes |
| 124 | * to use. Implementing such co-ordination would be essentially impossible. |
| 125 | */ |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 126 | void machine_restart(char *cmd) |
| 127 | { |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 128 | /* Disable interrupts first */ |
| 129 | local_irq_disable(); |
Arun KS | b9acc49 | 2014-05-07 02:41:23 +0100 | [diff] [blame] | 130 | smp_send_stop(); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 131 | |
Ard Biesheuvel | 60c0d45 | 2015-03-06 15:49:24 +0100 | [diff] [blame] | 132 | /* |
| 133 | * UpdateCapsule() depends on the system being reset via |
| 134 | * ResetSystem(). |
| 135 | */ |
| 136 | if (efi_enabled(EFI_RUNTIME_SERVICES)) |
| 137 | efi_reboot(reboot_mode, NULL); |
| 138 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 139 | /* Now call the architecture specific reboot code. */ |
Guenter Roeck | ab6cef1 | 2021-06-04 15:07:36 +0100 | [diff] [blame] | 140 | do_kernel_restart(cmd); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 141 | |
| 142 | /* |
| 143 | * Whoops - the architecture was unable to reboot. |
| 144 | */ |
| 145 | printk("Reboot failed -- System halted\n"); |
| 146 | while (1); |
| 147 | } |
| 148 | |
Dave Martin | ec94a46 | 2020-03-16 16:50:48 +0000 | [diff] [blame] | 149 | #define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = str |
| 150 | static const char *const btypes[] = { |
| 151 | bstr(NONE, "--"), |
| 152 | bstr( JC, "jc"), |
| 153 | bstr( C, "-c"), |
| 154 | bstr( J , "j-") |
| 155 | }; |
| 156 | #undef bstr |
| 157 | |
Will Deacon | b7300d4 | 2017-10-19 13:26:26 +0100 | [diff] [blame] | 158 | static void print_pstate(struct pt_regs *regs) |
| 159 | { |
| 160 | u64 pstate = regs->pstate; |
| 161 | |
| 162 | if (compat_user_mode(regs)) { |
Lingyan Huang | ec63e30 | 2021-07-22 10:20:36 +0800 | [diff] [blame] | 163 | printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c %cDIT %cSSBS)\n", |
Will Deacon | b7300d4 | 2017-10-19 13:26:26 +0100 | [diff] [blame] | 164 | pstate, |
Mark Rutland | d64567f | 2018-07-05 15:16:52 +0100 | [diff] [blame] | 165 | pstate & PSR_AA32_N_BIT ? 'N' : 'n', |
| 166 | pstate & PSR_AA32_Z_BIT ? 'Z' : 'z', |
| 167 | pstate & PSR_AA32_C_BIT ? 'C' : 'c', |
| 168 | pstate & PSR_AA32_V_BIT ? 'V' : 'v', |
| 169 | pstate & PSR_AA32_Q_BIT ? 'Q' : 'q', |
| 170 | pstate & PSR_AA32_T_BIT ? "T32" : "A32", |
| 171 | pstate & PSR_AA32_E_BIT ? "BE" : "LE", |
| 172 | pstate & PSR_AA32_A_BIT ? 'A' : 'a', |
| 173 | pstate & PSR_AA32_I_BIT ? 'I' : 'i', |
Lingyan Huang | ec63e30 | 2021-07-22 10:20:36 +0800 | [diff] [blame] | 174 | pstate & PSR_AA32_F_BIT ? 'F' : 'f', |
| 175 | pstate & PSR_AA32_DIT_BIT ? '+' : '-', |
| 176 | pstate & PSR_AA32_SSBS_BIT ? '+' : '-'); |
Will Deacon | b7300d4 | 2017-10-19 13:26:26 +0100 | [diff] [blame] | 177 | } else { |
Dave Martin | ec94a46 | 2020-03-16 16:50:48 +0000 | [diff] [blame] | 178 | const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >> |
| 179 | PSR_BTYPE_SHIFT]; |
| 180 | |
Lingyan Huang | ec63e30 | 2021-07-22 10:20:36 +0800 | [diff] [blame] | 181 | printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n", |
Will Deacon | b7300d4 | 2017-10-19 13:26:26 +0100 | [diff] [blame] | 182 | pstate, |
| 183 | pstate & PSR_N_BIT ? 'N' : 'n', |
| 184 | pstate & PSR_Z_BIT ? 'Z' : 'z', |
| 185 | pstate & PSR_C_BIT ? 'C' : 'c', |
| 186 | pstate & PSR_V_BIT ? 'V' : 'v', |
| 187 | pstate & PSR_D_BIT ? 'D' : 'd', |
| 188 | pstate & PSR_A_BIT ? 'A' : 'a', |
| 189 | pstate & PSR_I_BIT ? 'I' : 'i', |
| 190 | pstate & PSR_F_BIT ? 'F' : 'f', |
| 191 | pstate & PSR_PAN_BIT ? '+' : '-', |
Dave Martin | ec94a46 | 2020-03-16 16:50:48 +0000 | [diff] [blame] | 192 | pstate & PSR_UAO_BIT ? '+' : '-', |
Vincenzo Frascino | 637ec83 | 2019-09-16 11:51:17 +0100 | [diff] [blame] | 193 | pstate & PSR_TCO_BIT ? '+' : '-', |
Lingyan Huang | ec63e30 | 2021-07-22 10:20:36 +0800 | [diff] [blame] | 194 | pstate & PSR_DIT_BIT ? '+' : '-', |
| 195 | pstate & PSR_SSBS_BIT ? '+' : '-', |
Dave Martin | ec94a46 | 2020-03-16 16:50:48 +0000 | [diff] [blame] | 196 | btype_str); |
Will Deacon | b7300d4 | 2017-10-19 13:26:26 +0100 | [diff] [blame] | 197 | } |
| 198 | } |
| 199 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 200 | void __show_regs(struct pt_regs *regs) |
| 201 | { |
Catalin Marinas | 6ca68e8 | 2013-09-17 18:49:46 +0100 | [diff] [blame] | 202 | int i, top_reg; |
| 203 | u64 lr, sp; |
| 204 | |
| 205 | if (compat_user_mode(regs)) { |
| 206 | lr = regs->compat_lr; |
| 207 | sp = regs->compat_sp; |
| 208 | top_reg = 12; |
| 209 | } else { |
| 210 | lr = regs->regs[30]; |
| 211 | sp = regs->sp; |
| 212 | top_reg = 29; |
| 213 | } |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 214 | |
Tejun Heo | a43cb95 | 2013-04-30 15:27:17 -0700 | [diff] [blame] | 215 | show_regs_print_info(KERN_DEFAULT); |
Will Deacon | b7300d4 | 2017-10-19 13:26:26 +0100 | [diff] [blame] | 216 | print_pstate(regs); |
Will Deacon | a06f818 | 2018-02-19 16:46:57 +0000 | [diff] [blame] | 217 | |
| 218 | if (!user_mode(regs)) { |
| 219 | printk("pc : %pS\n", (void *)regs->pc); |
Mark Rutland | ca70859 | 2023-04-12 17:01:33 +0100 | [diff] [blame] | 220 | printk("lr : %pS\n", (void *)ptrauth_strip_kernel_insn_pac(lr)); |
Will Deacon | a06f818 | 2018-02-19 16:46:57 +0000 | [diff] [blame] | 221 | } else { |
| 222 | printk("pc : %016llx\n", regs->pc); |
| 223 | printk("lr : %016llx\n", lr); |
| 224 | } |
| 225 | |
Will Deacon | b7300d4 | 2017-10-19 13:26:26 +0100 | [diff] [blame] | 226 | printk("sp : %016llx\n", sp); |
Mark Rutland | db4b071 | 2016-10-20 12:23:16 +0100 | [diff] [blame] | 227 | |
Julien Thierry | 133d051 | 2019-01-31 14:58:46 +0000 | [diff] [blame] | 228 | if (system_uses_irq_prio_masking()) |
| 229 | printk("pmr_save: %08llx\n", regs->pmr_save); |
| 230 | |
Mark Rutland | db4b071 | 2016-10-20 12:23:16 +0100 | [diff] [blame] | 231 | i = top_reg; |
| 232 | |
| 233 | while (i >= 0) { |
Matthew Wilcox (Oracle) | 0bca3ec | 2021-04-20 18:22:45 +0100 | [diff] [blame] | 234 | printk("x%-2d: %016llx", i, regs->regs[i]); |
Mark Rutland | db4b071 | 2016-10-20 12:23:16 +0100 | [diff] [blame] | 235 | |
Matthew Wilcox (Oracle) | 0bca3ec | 2021-04-20 18:22:45 +0100 | [diff] [blame] | 236 | while (i-- % 3) |
| 237 | pr_cont(" x%-2d: %016llx", i, regs->regs[i]); |
Mark Rutland | db4b071 | 2016-10-20 12:23:16 +0100 | [diff] [blame] | 238 | |
| 239 | pr_cont("\n"); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 240 | } |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 241 | } |
| 242 | |
Zhiyuan Dai | d9f1b52 | 2021-02-04 09:43:49 +0800 | [diff] [blame] | 243 | void show_regs(struct pt_regs *regs) |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 244 | { |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 245 | __show_regs(regs); |
Dmitry Safonov | c768983 | 2020-06-08 21:30:23 -0700 | [diff] [blame] | 246 | dump_backtrace(regs, NULL, KERN_DEFAULT); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 247 | } |
| 248 | |
Will Deacon | eb35bdd7 | 2014-09-11 14:38:16 +0100 | [diff] [blame] | 249 | static void tls_thread_flush(void) |
| 250 | { |
Mark Rutland | adf7589 | 2016-09-08 13:55:38 +0100 | [diff] [blame] | 251 | write_sysreg(0, tpidr_el0); |
Mark Brown | a9d6915 | 2022-04-19 12:22:20 +0100 | [diff] [blame] | 252 | if (system_supports_tpidr2()) |
| 253 | write_sysreg_s(0, SYS_TPIDR2_EL0); |
Will Deacon | eb35bdd7 | 2014-09-11 14:38:16 +0100 | [diff] [blame] | 254 | |
| 255 | if (is_compat_task()) { |
Dave Martin | 6589654 | 2018-03-28 10:50:49 +0100 | [diff] [blame] | 256 | current->thread.uw.tp_value = 0; |
Will Deacon | eb35bdd7 | 2014-09-11 14:38:16 +0100 | [diff] [blame] | 257 | |
| 258 | /* |
| 259 | * We need to ensure ordering between the shadow state and the |
| 260 | * hardware state, so that we don't corrupt the hardware state |
| 261 | * with a stale shadow state during context switch. |
| 262 | */ |
| 263 | barrier(); |
Mark Rutland | adf7589 | 2016-09-08 13:55:38 +0100 | [diff] [blame] | 264 | write_sysreg(0, tpidrro_el0); |
Will Deacon | eb35bdd7 | 2014-09-11 14:38:16 +0100 | [diff] [blame] | 265 | } |
| 266 | } |
| 267 | |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 268 | static void flush_tagged_addr_state(void) |
| 269 | { |
| 270 | if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI)) |
| 271 | clear_thread_flag(TIF_TAGGED_ADDR); |
| 272 | } |
| 273 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 274 | void flush_thread(void) |
| 275 | { |
| 276 | fpsimd_flush_thread(); |
Will Deacon | eb35bdd7 | 2014-09-11 14:38:16 +0100 | [diff] [blame] | 277 | tls_thread_flush(); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 278 | flush_ptrace_hw_breakpoint(current); |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 279 | flush_tagged_addr_state(); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 280 | } |
| 281 | |
Dave Martin | bc0ee47 | 2017-10-31 15:51:05 +0000 | [diff] [blame] | 282 | void arch_release_task_struct(struct task_struct *tsk) |
| 283 | { |
| 284 | fpsimd_release_task(tsk); |
| 285 | } |
| 286 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 287 | int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) |
| 288 | { |
Janet Liu | 6eb6c80 | 2015-06-11 12:04:32 +0800 | [diff] [blame] | 289 | if (current->mm) |
| 290 | fpsimd_preserve_current_state(); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 291 | *dst = *src; |
Dave Martin | bc0ee47 | 2017-10-31 15:51:05 +0000 | [diff] [blame] | 292 | |
Masayoshi Mizuma | 4585fc5 | 2019-09-30 16:56:00 -0400 | [diff] [blame] | 293 | /* We rely on the above assignment to initialize dst's thread_flags: */ |
| 294 | BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK)); |
| 295 | |
| 296 | /* |
| 297 | * Detach src's sve_state (if any) from dst so that it does not |
Mark Brown | 8bd7f91 | 2022-04-19 12:22:24 +0100 | [diff] [blame] | 298 | * get erroneously used or freed prematurely. dst's copies |
Masayoshi Mizuma | 4585fc5 | 2019-09-30 16:56:00 -0400 | [diff] [blame] | 299 | * will be allocated on demand later on if dst uses SVE. |
| 300 | * For consistency, also clear TIF_SVE here: this could be done |
| 301 | * later in copy_process(), but to avoid tripping up future |
Mark Brown | 8bd7f91 | 2022-04-19 12:22:24 +0100 | [diff] [blame] | 302 | * maintainers it is best not to leave TIF flags and buffers in |
Masayoshi Mizuma | 4585fc5 | 2019-09-30 16:56:00 -0400 | [diff] [blame] | 303 | * an inconsistent state, even temporarily. |
| 304 | */ |
| 305 | dst->thread.sve_state = NULL; |
| 306 | clear_tsk_thread_flag(dst, TIF_SVE); |
| 307 | |
Mark Brown | 8bd7f91 | 2022-04-19 12:22:24 +0100 | [diff] [blame] | 308 | /* |
| 309 | * In the unlikely event that we create a new thread with ZA |
Mark Brown | d6138b4 | 2023-01-16 16:04:44 +0000 | [diff] [blame] | 310 | * enabled we should retain the ZA and ZT state so duplicate |
| 311 | * it here. This may be shortly freed if we exec() or if |
| 312 | * CLONE_SETTLS but it's simpler to do it here. To avoid |
| 313 | * confusing the rest of the code ensure that we have a |
| 314 | * sve_state allocated whenever sme_state is allocated. |
Mark Brown | 8bd7f91 | 2022-04-19 12:22:24 +0100 | [diff] [blame] | 315 | */ |
| 316 | if (thread_za_enabled(&src->thread)) { |
| 317 | dst->thread.sve_state = kzalloc(sve_state_size(src), |
| 318 | GFP_KERNEL); |
Wan Jiabing | 2e29b99 | 2022-04-26 19:30:53 +0800 | [diff] [blame] | 319 | if (!dst->thread.sve_state) |
Mark Brown | 8bd7f91 | 2022-04-19 12:22:24 +0100 | [diff] [blame] | 320 | return -ENOMEM; |
Mark Brown | ce51400 | 2023-01-16 16:04:36 +0000 | [diff] [blame] | 321 | |
| 322 | dst->thread.sme_state = kmemdup(src->thread.sme_state, |
| 323 | sme_state_size(src), |
| 324 | GFP_KERNEL); |
| 325 | if (!dst->thread.sme_state) { |
Mark Brown | 8bd7f91 | 2022-04-19 12:22:24 +0100 | [diff] [blame] | 326 | kfree(dst->thread.sve_state); |
| 327 | dst->thread.sve_state = NULL; |
| 328 | return -ENOMEM; |
| 329 | } |
| 330 | } else { |
Mark Brown | ce51400 | 2023-01-16 16:04:36 +0000 | [diff] [blame] | 331 | dst->thread.sme_state = NULL; |
Mark Brown | 8bd7f91 | 2022-04-19 12:22:24 +0100 | [diff] [blame] | 332 | clear_tsk_thread_flag(dst, TIF_SME); |
| 333 | } |
Mark Brown | b40c559 | 2022-04-19 12:22:21 +0100 | [diff] [blame] | 334 | |
Mark Brown | baa8515 | 2022-11-15 09:46:34 +0000 | [diff] [blame] | 335 | dst->thread.fp_type = FP_STATE_FPSIMD; |
| 336 | |
Vincenzo Frascino | 637ec83 | 2019-09-16 11:51:17 +0100 | [diff] [blame] | 337 | /* clear any pending asynchronous tag fault raised by the parent */ |
| 338 | clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT); |
| 339 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 340 | return 0; |
| 341 | } |
| 342 | |
| 343 | asmlinkage void ret_from_fork(void) asm("ret_from_fork"); |
| 344 | |
Eric W. Biederman | c5febea | 2022-04-08 18:07:50 -0500 | [diff] [blame] | 345 | int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 346 | { |
Eric W. Biederman | c5febea | 2022-04-08 18:07:50 -0500 | [diff] [blame] | 347 | unsigned long clone_flags = args->flags; |
| 348 | unsigned long stack_start = args->stack; |
Eric W. Biederman | c5febea | 2022-04-08 18:07:50 -0500 | [diff] [blame] | 349 | unsigned long tls = args->tls; |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 350 | struct pt_regs *childregs = task_pt_regs(p); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 351 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 352 | memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 353 | |
Dave Martin | bc0ee47 | 2017-10-31 15:51:05 +0000 | [diff] [blame] | 354 | /* |
Dave Martin | 071b6d4 | 2017-12-05 14:56:42 +0000 | [diff] [blame] | 355 | * In case p was allocated the same task_struct pointer as some |
| 356 | * other recently-exited task, make sure p is disassociated from |
| 357 | * any cpu that may have run that now-exited task recently. |
| 358 | * Otherwise we could erroneously skip reloading the FPSIMD |
| 359 | * registers for p. |
| 360 | */ |
| 361 | fpsimd_flush_task_state(p); |
| 362 | |
Kristina Martsenko | 33e4523 | 2020-03-13 14:34:56 +0530 | [diff] [blame] | 363 | ptrauth_thread_init_kernel(p); |
| 364 | |
Eric W. Biederman | 5bd2e97 | 2022-04-12 10:18:48 -0500 | [diff] [blame] | 365 | if (likely(!args->fn)) { |
Al Viro | 9ac0800 | 2012-10-21 15:56:52 -0400 | [diff] [blame] | 366 | *childregs = *current_pt_regs(); |
Catalin Marinas | c34501d | 2012-10-05 12:31:20 +0100 | [diff] [blame] | 367 | childregs->regs[0] = 0; |
Will Deacon | d00a381 | 2015-05-27 15:39:40 +0100 | [diff] [blame] | 368 | |
| 369 | /* |
| 370 | * Read the current TLS pointer from tpidr_el0 as it may be |
| 371 | * out-of-sync with the saved value. |
| 372 | */ |
Mark Rutland | adf7589 | 2016-09-08 13:55:38 +0100 | [diff] [blame] | 373 | *task_user_tls(p) = read_sysreg(tpidr_el0); |
Mark Brown | a9d6915 | 2022-04-19 12:22:20 +0100 | [diff] [blame] | 374 | if (system_supports_tpidr2()) |
| 375 | p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); |
Will Deacon | d00a381 | 2015-05-27 15:39:40 +0100 | [diff] [blame] | 376 | |
| 377 | if (stack_start) { |
| 378 | if (is_compat_thread(task_thread_info(p))) |
Al Viro | e0fd18c | 2012-10-18 00:55:54 -0400 | [diff] [blame] | 379 | childregs->compat_sp = stack_start; |
Will Deacon | d00a381 | 2015-05-27 15:39:40 +0100 | [diff] [blame] | 380 | else |
Al Viro | e0fd18c | 2012-10-18 00:55:54 -0400 | [diff] [blame] | 381 | childregs->sp = stack_start; |
Catalin Marinas | c34501d | 2012-10-05 12:31:20 +0100 | [diff] [blame] | 382 | } |
Will Deacon | d00a381 | 2015-05-27 15:39:40 +0100 | [diff] [blame] | 383 | |
Catalin Marinas | c34501d | 2012-10-05 12:31:20 +0100 | [diff] [blame] | 384 | /* |
Amanieu d'Antras | a4376f2 | 2020-01-02 18:24:08 +0100 | [diff] [blame] | 385 | * If a TLS pointer was passed to clone, use it for the new |
Mark Brown | a9d6915 | 2022-04-19 12:22:20 +0100 | [diff] [blame] | 386 | * thread. We also reset TPIDR2 if it's in use. |
Catalin Marinas | c34501d | 2012-10-05 12:31:20 +0100 | [diff] [blame] | 387 | */ |
Mark Brown | a9d6915 | 2022-04-19 12:22:20 +0100 | [diff] [blame] | 388 | if (clone_flags & CLONE_SETTLS) { |
Amanieu d'Antras | a4376f2 | 2020-01-02 18:24:08 +0100 | [diff] [blame] | 389 | p->thread.uw.tp_value = tls; |
Mark Brown | a9d6915 | 2022-04-19 12:22:20 +0100 | [diff] [blame] | 390 | p->thread.tpidr2_el0 = 0; |
| 391 | } |
Catalin Marinas | c34501d | 2012-10-05 12:31:20 +0100 | [diff] [blame] | 392 | } else { |
Mark Rutland | f80d034 | 2020-11-13 12:49:21 +0000 | [diff] [blame] | 393 | /* |
| 394 | * A kthread has no context to ERET to, so ensure any buggy |
| 395 | * ERET is treated as an illegal exception return. |
| 396 | * |
| 397 | * When a user task is created from a kthread, childregs will |
| 398 | * be initialized by start_thread() or start_compat_thread(). |
| 399 | */ |
Catalin Marinas | c34501d | 2012-10-05 12:31:20 +0100 | [diff] [blame] | 400 | memset(childregs, 0, sizeof(struct pt_regs)); |
Mark Rutland | f80d034 | 2020-11-13 12:49:21 +0000 | [diff] [blame] | 401 | childregs->pstate = PSR_MODE_EL1h | PSR_IL_BIT; |
Julien Thierry | 133d051 | 2019-01-31 14:58:46 +0000 | [diff] [blame] | 402 | |
Eric W. Biederman | 5bd2e97 | 2022-04-12 10:18:48 -0500 | [diff] [blame] | 403 | p->thread.cpu_context.x19 = (unsigned long)args->fn; |
| 404 | p->thread.cpu_context.x20 = (unsigned long)args->fn_arg; |
Catalin Marinas | c34501d | 2012-10-05 12:31:20 +0100 | [diff] [blame] | 405 | } |
| 406 | p->thread.cpu_context.pc = (unsigned long)ret_from_fork; |
| 407 | p->thread.cpu_context.sp = (unsigned long)childregs; |
Madhavan T. Venkataraman | 7d7b720 | 2021-05-10 12:00:26 +0100 | [diff] [blame] | 408 | /* |
| 409 | * For the benefit of the unwinder, set up childregs->stackframe |
| 410 | * as the final frame for the new task. |
| 411 | */ |
| 412 | p->thread.cpu_context.fp = (unsigned long)childregs->stackframe; |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 413 | |
| 414 | ptrace_hw_copy_thread(p); |
| 415 | |
| 416 | return 0; |
| 417 | } |
| 418 | |
Dave Martin | 936eb65 | 2017-06-21 16:00:44 +0100 | [diff] [blame] | 419 | void tls_preserve_current_state(void) |
| 420 | { |
| 421 | *task_user_tls(current) = read_sysreg(tpidr_el0); |
Mark Brown | a9d6915 | 2022-04-19 12:22:20 +0100 | [diff] [blame] | 422 | if (system_supports_tpidr2() && !is_compat_task()) |
| 423 | current->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); |
Dave Martin | 936eb65 | 2017-06-21 16:00:44 +0100 | [diff] [blame] | 424 | } |
| 425 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 426 | static void tls_thread_switch(struct task_struct *next) |
| 427 | { |
Dave Martin | 936eb65 | 2017-06-21 16:00:44 +0100 | [diff] [blame] | 428 | tls_preserve_current_state(); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 429 | |
Will Deacon | 18011ea | 2017-11-14 14:33:28 +0000 | [diff] [blame] | 430 | if (is_compat_thread(task_thread_info(next))) |
Dave Martin | 6589654 | 2018-03-28 10:50:49 +0100 | [diff] [blame] | 431 | write_sysreg(next->thread.uw.tp_value, tpidrro_el0); |
Will Deacon | 18011ea | 2017-11-14 14:33:28 +0000 | [diff] [blame] | 432 | else if (!arm64_kernel_unmapped_at_el0()) |
| 433 | write_sysreg(0, tpidrro_el0); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 434 | |
Will Deacon | 18011ea | 2017-11-14 14:33:28 +0000 | [diff] [blame] | 435 | write_sysreg(*task_user_tls(next), tpidr_el0); |
Mark Brown | a9d6915 | 2022-04-19 12:22:20 +0100 | [diff] [blame] | 436 | if (system_supports_tpidr2()) |
| 437 | write_sysreg_s(next->thread.tpidr2_el0, SYS_TPIDR2_EL0); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 438 | } |
| 439 | |
| 440 | /* |
Marc Zyngier | cbdf8a1 | 2019-07-22 14:53:09 +0100 | [diff] [blame] | 441 | * Force SSBS state on context-switch, since it may be lost after migrating |
| 442 | * from a CPU which treats the bit as RES0 in a heterogeneous system. |
| 443 | */ |
| 444 | static void ssbs_thread_switch(struct task_struct *next) |
| 445 | { |
Marc Zyngier | cbdf8a1 | 2019-07-22 14:53:09 +0100 | [diff] [blame] | 446 | /* |
| 447 | * Nothing to do for kernel threads, but 'regs' may be junk |
| 448 | * (e.g. idle task) so check the flags and bail early. |
| 449 | */ |
| 450 | if (unlikely(next->flags & PF_KTHREAD)) |
| 451 | return; |
| 452 | |
Will Deacon | fca3d33 | 2020-02-06 10:42:58 +0000 | [diff] [blame] | 453 | /* |
| 454 | * If all CPUs implement the SSBS extension, then we just need to |
| 455 | * context-switch the PSTATE field. |
| 456 | */ |
Mark Rutland | bc75d0c | 2023-10-16 11:24:50 +0100 | [diff] [blame] | 457 | if (alternative_has_cap_unlikely(ARM64_SSBS)) |
Will Deacon | fca3d33 | 2020-02-06 10:42:58 +0000 | [diff] [blame] | 458 | return; |
| 459 | |
Will Deacon | c287620 | 2020-09-18 11:54:33 +0100 | [diff] [blame] | 460 | spectre_v4_enable_task_mitigation(next); |
Marc Zyngier | cbdf8a1 | 2019-07-22 14:53:09 +0100 | [diff] [blame] | 461 | } |
| 462 | |
| 463 | /* |
Mark Rutland | c02433d | 2016-11-03 20:23:13 +0000 | [diff] [blame] | 464 | * We store our current task in sp_el0, which is clobbered by userspace. Keep a |
| 465 | * shadow copy so that we can restore this upon entry from userspace. |
| 466 | * |
| 467 | * This is *only* for exception entry from EL0, and is not valid until we |
| 468 | * __switch_to() a user task. |
| 469 | */ |
| 470 | DEFINE_PER_CPU(struct task_struct *, __entry_task); |
| 471 | |
| 472 | static void entry_task_switch(struct task_struct *next) |
| 473 | { |
| 474 | __this_cpu_write(__entry_task, next); |
| 475 | } |
| 476 | |
| 477 | /* |
Marc Zyngier | d49f7d7 | 2020-07-31 18:38:23 +0100 | [diff] [blame] | 478 | * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT. |
D Scott Phillips | 38e0257 | 2021-12-20 15:41:14 -0800 | [diff] [blame] | 479 | * Ensure access is disabled when switching to a 32bit task, ensure |
| 480 | * access is enabled when switching to a 64bit task. |
Marc Zyngier | d49f7d7 | 2020-07-31 18:38:23 +0100 | [diff] [blame] | 481 | */ |
D Scott Phillips | 38e0257 | 2021-12-20 15:41:14 -0800 | [diff] [blame] | 482 | static void erratum_1418040_thread_switch(struct task_struct *next) |
Marc Zyngier | d49f7d7 | 2020-07-31 18:38:23 +0100 | [diff] [blame] | 483 | { |
D Scott Phillips | 38e0257 | 2021-12-20 15:41:14 -0800 | [diff] [blame] | 484 | if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) || |
| 485 | !this_cpu_has_cap(ARM64_WORKAROUND_1418040)) |
Marc Zyngier | d49f7d7 | 2020-07-31 18:38:23 +0100 | [diff] [blame] | 486 | return; |
| 487 | |
D Scott Phillips | 38e0257 | 2021-12-20 15:41:14 -0800 | [diff] [blame] | 488 | if (is_compat_thread(task_thread_info(next))) |
| 489 | sysreg_clear_set(cntkctl_el1, ARCH_TIMER_USR_VCT_ACCESS_EN, 0); |
Marc Zyngier | d49f7d7 | 2020-07-31 18:38:23 +0100 | [diff] [blame] | 490 | else |
D Scott Phillips | 38e0257 | 2021-12-20 15:41:14 -0800 | [diff] [blame] | 491 | sysreg_clear_set(cntkctl_el1, 0, ARCH_TIMER_USR_VCT_ACCESS_EN); |
| 492 | } |
Marc Zyngier | d49f7d7 | 2020-07-31 18:38:23 +0100 | [diff] [blame] | 493 | |
D Scott Phillips | 38e0257 | 2021-12-20 15:41:14 -0800 | [diff] [blame] | 494 | static void erratum_1418040_new_exec(void) |
| 495 | { |
| 496 | preempt_disable(); |
| 497 | erratum_1418040_thread_switch(current); |
| 498 | preempt_enable(); |
Marc Zyngier | d49f7d7 | 2020-07-31 18:38:23 +0100 | [diff] [blame] | 499 | } |
| 500 | |
Peter Collingbourne | d2e0d8f | 2021-07-27 13:52:57 -0700 | [diff] [blame] | 501 | /* |
| 502 | * __switch_to() checks current->thread.sctlr_user as an optimisation. Therefore |
| 503 | * this function must be called with preemption disabled and the update to |
| 504 | * sctlr_user must be made in the same preemption disabled block so that |
| 505 | * __switch_to() does not see the variable update before the SCTLR_EL1 one. |
| 506 | */ |
| 507 | void update_sctlr_el1(u64 sctlr) |
Peter Collingbourne | 2f79d2f | 2021-03-18 20:10:52 -0700 | [diff] [blame] | 508 | { |
Peter Collingbourne | 2016986 | 2021-03-18 20:10:53 -0700 | [diff] [blame] | 509 | /* |
| 510 | * EnIA must not be cleared while in the kernel as this is necessary for |
| 511 | * in-kernel PAC. It will be cleared on kernel exit if needed. |
| 512 | */ |
| 513 | sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr); |
Peter Collingbourne | 2f79d2f | 2021-03-18 20:10:52 -0700 | [diff] [blame] | 514 | |
| 515 | /* ISB required for the kernel uaccess routines when setting TCF0. */ |
| 516 | isb(); |
| 517 | } |
| 518 | |
Marc Zyngier | d49f7d7 | 2020-07-31 18:38:23 +0100 | [diff] [blame] | 519 | /* |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 520 | * Thread switching. |
| 521 | */ |
Mark Rutland | 86bcbaf | 2021-11-29 14:28:43 +0000 | [diff] [blame] | 522 | __notrace_funcgraph __sched |
| 523 | struct task_struct *__switch_to(struct task_struct *prev, |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 524 | struct task_struct *next) |
| 525 | { |
| 526 | struct task_struct *last; |
| 527 | |
| 528 | fpsimd_thread_switch(next); |
| 529 | tls_thread_switch(next); |
| 530 | hw_breakpoint_thread_switch(next); |
Christopher Covington | 3325732 | 2013-04-03 19:01:01 +0100 | [diff] [blame] | 531 | contextidr_thread_switch(next); |
Mark Rutland | c02433d | 2016-11-03 20:23:13 +0000 | [diff] [blame] | 532 | entry_task_switch(next); |
Marc Zyngier | cbdf8a1 | 2019-07-22 14:53:09 +0100 | [diff] [blame] | 533 | ssbs_thread_switch(next); |
D Scott Phillips | 38e0257 | 2021-12-20 15:41:14 -0800 | [diff] [blame] | 534 | erratum_1418040_thread_switch(next); |
Peter Collingbourne | b90e483 | 2021-03-18 20:10:54 -0700 | [diff] [blame] | 535 | ptrauth_thread_switch_user(next); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 536 | |
Catalin Marinas | 5108c67 | 2013-04-24 14:47:02 +0100 | [diff] [blame] | 537 | /* |
| 538 | * Complete any pending TLB or cache maintenance on this CPU in case |
| 539 | * the thread migrates to a different CPU. |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 540 | * This full barrier is also required by the membarrier system |
| 541 | * call. |
Catalin Marinas | 5108c67 | 2013-04-24 14:47:02 +0100 | [diff] [blame] | 542 | */ |
Will Deacon | 98f7685 | 2014-05-02 16:24:10 +0100 | [diff] [blame] | 543 | dsb(ish); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 544 | |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 545 | /* |
| 546 | * MTE thread switching must happen after the DSB above to ensure that |
| 547 | * any asynchronous tag check faults have been logged in the TFSR*_EL1 |
| 548 | * registers. |
| 549 | */ |
| 550 | mte_thread_switch(next); |
Peter Collingbourne | 2f79d2f | 2021-03-18 20:10:52 -0700 | [diff] [blame] | 551 | /* avoid expensive SCTLR_EL1 accesses if no change */ |
| 552 | if (prev->thread.sctlr_user != next->thread.sctlr_user) |
| 553 | update_sctlr_el1(next->thread.sctlr_user); |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 554 | |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 555 | /* the actual thread switch */ |
| 556 | last = cpu_switch_to(prev, next); |
| 557 | |
| 558 | return last; |
| 559 | } |
| 560 | |
Madhavan T. Venkataraman | 4f62bb7 | 2021-11-29 14:28:45 +0000 | [diff] [blame] | 561 | struct wchan_info { |
| 562 | unsigned long pc; |
| 563 | int count; |
| 564 | }; |
| 565 | |
| 566 | static bool get_wchan_cb(void *arg, unsigned long pc) |
| 567 | { |
| 568 | struct wchan_info *wchan_info = arg; |
| 569 | |
| 570 | if (!in_sched_functions(pc)) { |
| 571 | wchan_info->pc = pc; |
| 572 | return false; |
| 573 | } |
| 574 | return wchan_info->count++ < 16; |
| 575 | } |
| 576 | |
Kees Cook | 42a20f8 | 2021-09-29 15:02:14 -0700 | [diff] [blame] | 577 | unsigned long __get_wchan(struct task_struct *p) |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 578 | { |
Madhavan T. Venkataraman | 4f62bb7 | 2021-11-29 14:28:45 +0000 | [diff] [blame] | 579 | struct wchan_info wchan_info = { |
| 580 | .pc = 0, |
| 581 | .count = 0, |
| 582 | }; |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 583 | |
Madhavan T. Venkataraman | 4f62bb7 | 2021-11-29 14:28:45 +0000 | [diff] [blame] | 584 | if (!try_get_task_stack(p)) |
Mark Rutland | 9bbd4c5 | 2016-11-03 20:23:08 +0000 | [diff] [blame] | 585 | return 0; |
| 586 | |
Madhavan T. Venkataraman | 4f62bb7 | 2021-11-29 14:28:45 +0000 | [diff] [blame] | 587 | arch_stack_walk(get_wchan_cb, &wchan_info, p, NULL); |
Dave Martin | f3dcbe6 | 2019-07-02 14:07:28 +0100 | [diff] [blame] | 588 | |
Mark Rutland | 9bbd4c5 | 2016-11-03 20:23:08 +0000 | [diff] [blame] | 589 | put_task_stack(p); |
Madhavan T. Venkataraman | 4f62bb7 | 2021-11-29 14:28:45 +0000 | [diff] [blame] | 590 | |
| 591 | return wchan_info.pc; |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 592 | } |
| 593 | |
| 594 | unsigned long arch_align_stack(unsigned long sp) |
| 595 | { |
| 596 | if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) |
Jason A. Donenfeld | 8032bf1 | 2022-10-09 20:44:02 -0600 | [diff] [blame] | 597 | sp -= get_random_u32_below(PAGE_SIZE); |
Catalin Marinas | b3901d5 | 2012-03-05 11:49:28 +0000 | [diff] [blame] | 598 | return sp & ~0xf; |
| 599 | } |
| 600 | |
Will Deacon | 08cd8f4 | 2021-07-30 12:24:38 +0100 | [diff] [blame] | 601 | #ifdef CONFIG_COMPAT |
| 602 | int compat_elf_check_arch(const struct elf32_hdr *hdr) |
| 603 | { |
| 604 | if (!system_supports_32bit_el0()) |
| 605 | return false; |
| 606 | |
| 607 | if ((hdr)->e_machine != EM_ARM) |
| 608 | return false; |
| 609 | |
| 610 | if (!((hdr)->e_flags & EF_ARM_EABI_MASK)) |
| 611 | return false; |
| 612 | |
| 613 | /* |
| 614 | * Prevent execve() of a 32-bit program from a deadline task |
| 615 | * if the restricted affinity mask would be inadmissible on an |
| 616 | * asymmetric system. |
| 617 | */ |
| 618 | return !static_branch_unlikely(&arm64_mismatched_32bit_el0) || |
| 619 | !dl_task_check_affinity(current, system_32bit_el0_cpumask()); |
| 620 | } |
| 621 | #endif |
| 622 | |
Yury Norov | d1be5c9 | 2017-08-20 13:20:48 +0300 | [diff] [blame] | 623 | /* |
| 624 | * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY. |
| 625 | */ |
| 626 | void arch_setup_new_exec(void) |
| 627 | { |
Will Deacon | 873c3e8 | 2021-06-08 19:02:57 +0100 | [diff] [blame] | 628 | unsigned long mmflags = 0; |
Mark Rutland | 7503197 | 2018-12-07 18:39:25 +0000 | [diff] [blame] | 629 | |
Will Deacon | 873c3e8 | 2021-06-08 19:02:57 +0100 | [diff] [blame] | 630 | if (is_compat_task()) { |
| 631 | mmflags = MMCF_AARCH32; |
Will Deacon | 08cd8f4 | 2021-07-30 12:24:38 +0100 | [diff] [blame] | 632 | |
| 633 | /* |
| 634 | * Restrict the CPU affinity mask for a 32-bit task so that |
| 635 | * it contains only 32-bit-capable CPUs. |
| 636 | * |
| 637 | * From the perspective of the task, this looks similar to |
| 638 | * what would happen if the 64-bit-only CPUs were hot-unplugged |
| 639 | * at the point of execve(), although we try a bit harder to |
| 640 | * honour the cpuset hierarchy. |
| 641 | */ |
Will Deacon | 873c3e8 | 2021-06-08 19:02:57 +0100 | [diff] [blame] | 642 | if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) |
Will Deacon | 08cd8f4 | 2021-07-30 12:24:38 +0100 | [diff] [blame] | 643 | force_compatible_cpus_allowed_ptr(current); |
Will Deacon | 08cd8f4 | 2021-07-30 12:24:38 +0100 | [diff] [blame] | 644 | } else if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) { |
| 645 | relax_compatible_cpus_allowed_ptr(current); |
Will Deacon | 873c3e8 | 2021-06-08 19:02:57 +0100 | [diff] [blame] | 646 | } |
| 647 | |
| 648 | current->mm->context.flags = mmflags; |
Peter Collingbourne | 2016986 | 2021-03-18 20:10:53 -0700 | [diff] [blame] | 649 | ptrauth_thread_init_user(); |
| 650 | mte_thread_init_user(); |
D Scott Phillips | 38e0257 | 2021-12-20 15:41:14 -0800 | [diff] [blame] | 651 | erratum_1418040_new_exec(); |
Will Deacon | 780c083 | 2020-09-28 14:03:00 +0100 | [diff] [blame] | 652 | |
| 653 | if (task_spec_ssb_noexec(current)) { |
| 654 | arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS, |
| 655 | PR_SPEC_ENABLE); |
| 656 | } |
Yury Norov | d1be5c9 | 2017-08-20 13:20:48 +0300 | [diff] [blame] | 657 | } |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 658 | |
| 659 | #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI |
| 660 | /* |
| 661 | * Control the relaxed ABI allowing tagged user addresses into the kernel. |
| 662 | */ |
Catalin Marinas | 413235f | 2019-08-15 16:44:01 +0100 | [diff] [blame] | 663 | static unsigned int tagged_addr_disabled; |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 664 | |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 665 | long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 666 | { |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 667 | unsigned long valid_mask = PR_TAGGED_ADDR_ENABLE; |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 668 | struct thread_info *ti = task_thread_info(task); |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 669 | |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 670 | if (is_compat_thread(ti)) |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 671 | return -EINVAL; |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 672 | |
| 673 | if (system_supports_mte()) |
Mark Brown | 766121b | 2022-02-16 17:32:24 +0000 | [diff] [blame] | 674 | valid_mask |= PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC \ |
| 675 | | PR_MTE_TAG_MASK; |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 676 | |
| 677 | if (arg & ~valid_mask) |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 678 | return -EINVAL; |
| 679 | |
Catalin Marinas | 413235f | 2019-08-15 16:44:01 +0100 | [diff] [blame] | 680 | /* |
| 681 | * Do not allow the enabling of the tagged address ABI if globally |
| 682 | * disabled via sysctl abi.tagged_addr_disabled. |
| 683 | */ |
| 684 | if (arg & PR_TAGGED_ADDR_ENABLE && tagged_addr_disabled) |
| 685 | return -EINVAL; |
| 686 | |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 687 | if (set_mte_ctrl(task, arg) != 0) |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 688 | return -EINVAL; |
| 689 | |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 690 | update_ti_thread_flag(ti, TIF_TAGGED_ADDR, arg & PR_TAGGED_ADDR_ENABLE); |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 691 | |
| 692 | return 0; |
| 693 | } |
| 694 | |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 695 | long get_tagged_addr_ctrl(struct task_struct *task) |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 696 | { |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 697 | long ret = 0; |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 698 | struct thread_info *ti = task_thread_info(task); |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 699 | |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 700 | if (is_compat_thread(ti)) |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 701 | return -EINVAL; |
| 702 | |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 703 | if (test_ti_thread_flag(ti, TIF_TAGGED_ADDR)) |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 704 | ret = PR_TAGGED_ADDR_ENABLE; |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 705 | |
Catalin Marinas | 93f067f | 2020-07-03 14:25:50 +0100 | [diff] [blame] | 706 | ret |= get_mte_ctrl(task); |
Catalin Marinas | 1c101da | 2019-11-27 10:30:15 +0000 | [diff] [blame] | 707 | |
| 708 | return ret; |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 709 | } |
| 710 | |
| 711 | /* |
| 712 | * Global sysctl to disable the tagged user addresses support. This control |
| 713 | * only prevents the tagged address ABI enabling via prctl() and does not |
| 714 | * disable it for tasks that already opted in to the relaxed ABI. |
| 715 | */ |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 716 | |
| 717 | static struct ctl_table tagged_addr_sysctl_table[] = { |
| 718 | { |
Catalin Marinas | 413235f | 2019-08-15 16:44:01 +0100 | [diff] [blame] | 719 | .procname = "tagged_addr_disabled", |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 720 | .mode = 0644, |
Catalin Marinas | 413235f | 2019-08-15 16:44:01 +0100 | [diff] [blame] | 721 | .data = &tagged_addr_disabled, |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 722 | .maxlen = sizeof(int), |
| 723 | .proc_handler = proc_dointvec_minmax, |
Matteo Croce | 2c614c1 | 2020-01-24 16:51:27 +0100 | [diff] [blame] | 724 | .extra1 = SYSCTL_ZERO, |
| 725 | .extra2 = SYSCTL_ONE, |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 726 | }, |
Catalin Marinas | 63f0c60 | 2019-07-23 19:58:39 +0200 | [diff] [blame] | 727 | }; |
| 728 | |
| 729 | static int __init tagged_addr_init(void) |
| 730 | { |
| 731 | if (!register_sysctl("abi", tagged_addr_sysctl_table)) |
| 732 | return -EINVAL; |
| 733 | return 0; |
| 734 | } |
| 735 | |
| 736 | core_initcall(tagged_addr_init); |
| 737 | #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */ |
Julien Thierry | 19c95f2 | 2019-10-15 18:25:44 +0100 | [diff] [blame] | 738 | |
Dave Martin | ab7876a | 2020-03-16 16:50:47 +0000 | [diff] [blame] | 739 | #ifdef CONFIG_BINFMT_ELF |
| 740 | int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state, |
| 741 | bool has_interp, bool is_interp) |
| 742 | { |
Mark Brown | 5d1b631 | 2020-03-23 17:01:19 +0000 | [diff] [blame] | 743 | /* |
| 744 | * For dynamically linked executables the interpreter is |
| 745 | * responsible for setting PROT_BTI on everything except |
| 746 | * itself. |
| 747 | */ |
Dave Martin | ab7876a | 2020-03-16 16:50:47 +0000 | [diff] [blame] | 748 | if (is_interp != has_interp) |
| 749 | return prot; |
| 750 | |
| 751 | if (!(state->flags & ARM64_ELF_BTI)) |
| 752 | return prot; |
| 753 | |
| 754 | if (prot & PROT_EXEC) |
| 755 | prot |= PROT_BTI; |
| 756 | |
| 757 | return prot; |
| 758 | } |
| 759 | #endif |