Thomas Gleixner | 2874c5f | 2019-05-27 08:55:01 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 2 | /* |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 3 | * PowerPC version |
| 4 | * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) |
| 5 | * |
| 6 | * Derived from "arch/i386/mm/fault.c" |
| 7 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds |
| 8 | * |
| 9 | * Modified by Cort Dougan and Paul Mackerras. |
| 10 | * |
| 11 | * Modified for PPC64 by Dave Engebretsen (engebret@ibm.com) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 12 | */ |
| 13 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 14 | #include <linux/signal.h> |
| 15 | #include <linux/sched.h> |
Ingo Molnar | 68db0cf | 2017-02-08 18:51:37 +0100 | [diff] [blame] | 16 | #include <linux/sched/task_stack.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 17 | #include <linux/kernel.h> |
| 18 | #include <linux/errno.h> |
| 19 | #include <linux/string.h> |
| 20 | #include <linux/types.h> |
Christophe Leroy | 0e36b0d | 2018-05-23 10:53:22 +0200 | [diff] [blame] | 21 | #include <linux/pagemap.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 22 | #include <linux/ptrace.h> |
| 23 | #include <linux/mman.h> |
| 24 | #include <linux/mm.h> |
| 25 | #include <linux/interrupt.h> |
| 26 | #include <linux/highmem.h> |
Paul Gortmaker | 8a39b05 | 2016-08-16 10:57:34 -0400 | [diff] [blame] | 27 | #include <linux/extable.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 28 | #include <linux/kprobes.h> |
Christoph Hellwig | 1eeb66a | 2007-05-08 00:27:03 -0700 | [diff] [blame] | 29 | #include <linux/kdebug.h> |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 30 | #include <linux/perf_event.h> |
Christian Dietrich | 7646223 | 2011-06-04 05:36:54 +0000 | [diff] [blame] | 31 | #include <linux/ratelimit.h> |
Li Zhong | ba12eed | 2013-05-13 16:16:41 +0000 | [diff] [blame] | 32 | #include <linux/context_tracking.h> |
Anton Blanchard | 9d57472 | 2014-09-24 16:59:58 +1000 | [diff] [blame] | 33 | #include <linux/hugetlb.h> |
David Hildenbrand | 70ffdb9 | 2015-05-11 17:52:11 +0200 | [diff] [blame] | 34 | #include <linux/uaccess.h> |
Christophe Leroy | 90cbac0 | 2021-03-04 14:35:09 +0000 | [diff] [blame] | 35 | #include <linux/kfence.h> |
Christophe Leroy | 98c26a7 | 2021-03-15 14:52:51 +0000 | [diff] [blame] | 36 | #include <linux/pkeys.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 37 | |
Brian King | 4090019 | 2008-10-22 05:53:45 +0000 | [diff] [blame] | 38 | #include <asm/firmware.h> |
Nicholas Piggin | 3a96570 | 2021-01-30 23:08:38 +1000 | [diff] [blame] | 39 | #include <asm/interrupt.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 40 | #include <asm/page.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 41 | #include <asm/mmu.h> |
| 42 | #include <asm/mmu_context.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 43 | #include <asm/siginfo.h> |
David Howells | ae3a197e | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 44 | #include <asm/debug.h> |
Michael Ellerman | 5e5be3a | 2019-04-18 16:51:25 +1000 | [diff] [blame] | 45 | #include <asm/kup.h> |
Jordan Niethe | 8094892 | 2020-05-06 13:40:28 +1000 | [diff] [blame] | 46 | #include <asm/inst.h> |
Christoph Hellwig | 9f90b99 | 2007-04-30 11:56:46 +0100 | [diff] [blame] | 47 | |
Michael Ellerman | 773b3e5 | 2020-07-24 19:25:27 +1000 | [diff] [blame] | 48 | |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 49 | /* |
| 50 | * do_page_fault error handling helpers |
| 51 | */ |
| 52 | |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 53 | static int |
Eric W. Biederman | cd60ab7a | 2018-09-18 10:42:33 +0200 | [diff] [blame] | 54 | __bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code) |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 55 | { |
| 56 | /* |
| 57 | * If we are in kernel mode, bail out with a SEGV, this will |
| 58 | * be caught by the assembly which will restore the non-volatile |
| 59 | * registers before calling bad_page_fault() |
| 60 | */ |
| 61 | if (!user_mode(regs)) |
| 62 | return SIGSEGV; |
| 63 | |
Eric W. Biederman | cd60ab7a | 2018-09-18 10:42:33 +0200 | [diff] [blame] | 64 | _exception(SIGSEGV, regs, si_code, address); |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 65 | |
| 66 | return 0; |
| 67 | } |
| 68 | |
| 69 | static noinline int bad_area_nosemaphore(struct pt_regs *regs, unsigned long address) |
| 70 | { |
Eric W. Biederman | cd60ab7a | 2018-09-18 10:42:33 +0200 | [diff] [blame] | 71 | return __bad_area_nosemaphore(regs, address, SEGV_MAPERR); |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 72 | } |
| 73 | |
Eric W. Biederman | 9f2ee69 | 2018-09-18 09:19:24 +0200 | [diff] [blame] | 74 | static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code) |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 75 | { |
| 76 | struct mm_struct *mm = current->mm; |
| 77 | |
| 78 | /* |
| 79 | * Something tried to access memory that isn't in our memory map.. |
| 80 | * Fix it, but check if it's kernel or user first.. |
| 81 | */ |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 82 | mmap_read_unlock(mm); |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 83 | |
Eric W. Biederman | cd60ab7a | 2018-09-18 10:42:33 +0200 | [diff] [blame] | 84 | return __bad_area_nosemaphore(regs, address, si_code); |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 85 | } |
| 86 | |
| 87 | static noinline int bad_area(struct pt_regs *regs, unsigned long address) |
| 88 | { |
Eric W. Biederman | 9f2ee69 | 2018-09-18 09:19:24 +0200 | [diff] [blame] | 89 | return __bad_area(regs, address, SEGV_MAPERR); |
Ram Pai | 99cd130 | 2018-01-18 17:50:42 -0800 | [diff] [blame] | 90 | } |
| 91 | |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 92 | static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, |
| 93 | struct vm_area_struct *vma) |
Ram Pai | 99cd130 | 2018-01-18 17:50:42 -0800 | [diff] [blame] | 94 | { |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 95 | struct mm_struct *mm = current->mm; |
| 96 | int pkey; |
| 97 | |
| 98 | /* |
| 99 | * We don't try to fetch the pkey from page table because reading |
| 100 | * page table without locking doesn't guarantee stable pte value. |
| 101 | * Hence the pkey value that we return to userspace can be different |
| 102 | * from the pkey that actually caused access error. |
| 103 | * |
| 104 | * It does *not* guarantee that the VMA we find here |
| 105 | * was the one that we faulted on. |
| 106 | * |
| 107 | * 1. T1 : mprotect_key(foo, PAGE_SIZE, pkey=4); |
| 108 | * 2. T1 : set AMR to deny access to pkey=4, touches, page |
| 109 | * 3. T1 : faults... |
| 110 | * 4. T2: mprotect_key(foo, PAGE_SIZE, pkey=5); |
Michel Lespinasse | c1e8d7c | 2020-06-08 21:33:54 -0700 | [diff] [blame] | 111 | * 5. T1 : enters fault handler, takes mmap_lock, etc... |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 112 | * 6. T1 : reaches here, sees vma_pkey(vma)=5, when we really |
| 113 | * faulted on a pte with its pkey=4. |
| 114 | */ |
| 115 | pkey = vma_pkey(vma); |
| 116 | |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 117 | mmap_read_unlock(mm); |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 118 | |
Eric W. Biederman | 8eb2ba2 | 2018-09-18 09:14:49 +0200 | [diff] [blame] | 119 | /* |
| 120 | * If we are in kernel mode, bail out with a SEGV, this will |
| 121 | * be caught by the assembly which will restore the non-volatile |
| 122 | * registers before calling bad_page_fault() |
| 123 | */ |
| 124 | if (!user_mode(regs)) |
| 125 | return SIGSEGV; |
| 126 | |
Eric W. Biederman | 5d8fb8a | 2018-09-18 10:56:25 +0200 | [diff] [blame] | 127 | _exception_pkey(regs, address, pkey); |
Eric W. Biederman | 8eb2ba2 | 2018-09-18 09:14:49 +0200 | [diff] [blame] | 128 | |
| 129 | return 0; |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 130 | } |
| 131 | |
John Sperbeck | ecb101a | 2017-12-31 21:24:58 -0800 | [diff] [blame] | 132 | static noinline int bad_access(struct pt_regs *regs, unsigned long address) |
| 133 | { |
Eric W. Biederman | 9f2ee69 | 2018-09-18 09:19:24 +0200 | [diff] [blame] | 134 | return __bad_area(regs, address, SEGV_ACCERR); |
John Sperbeck | ecb101a | 2017-12-31 21:24:58 -0800 | [diff] [blame] | 135 | } |
| 136 | |
Anton Blanchard | 3913fdd | 2014-09-24 16:59:57 +1000 | [diff] [blame] | 137 | static int do_sigbus(struct pt_regs *regs, unsigned long address, |
Souptick Joarder | 50a7ca3 | 2018-08-17 15:44:47 -0700 | [diff] [blame] | 138 | vm_fault_t fault) |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 139 | { |
Anton Blanchard | 63af526 | 2014-09-24 16:59:56 +1000 | [diff] [blame] | 140 | if (!user_mode(regs)) |
Benjamin Herrenschmidt | b5c8f0f | 2017-07-19 14:49:36 +1000 | [diff] [blame] | 141 | return SIGBUS; |
Anton Blanchard | 63af526 | 2014-09-24 16:59:56 +1000 | [diff] [blame] | 142 | |
| 143 | current->thread.trap_nr = BUS_ADRERR; |
Anton Blanchard | 3913fdd | 2014-09-24 16:59:57 +1000 | [diff] [blame] | 144 | #ifdef CONFIG_MEMORY_FAILURE |
| 145 | if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { |
Eric W. Biederman | f654fc0 | 2018-04-19 17:36:43 -0500 | [diff] [blame] | 146 | unsigned int lsb = 0; /* shutup gcc */ |
| 147 | |
Anton Blanchard | 3913fdd | 2014-09-24 16:59:57 +1000 | [diff] [blame] | 148 | pr_err("MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n", |
| 149 | current->comm, current->pid, address); |
Eric W. Biederman | f654fc0 | 2018-04-19 17:36:43 -0500 | [diff] [blame] | 150 | |
| 151 | if (fault & VM_FAULT_HWPOISON_LARGE) |
| 152 | lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault)); |
| 153 | if (fault & VM_FAULT_HWPOISON) |
| 154 | lsb = PAGE_SHIFT; |
| 155 | |
Eric W. Biederman | f8eac90 | 2019-02-05 18:14:19 -0600 | [diff] [blame] | 156 | force_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb); |
Eric W. Biederman | f654fc0 | 2018-04-19 17:36:43 -0500 | [diff] [blame] | 157 | return 0; |
Anton Blanchard | 3913fdd | 2014-09-24 16:59:57 +1000 | [diff] [blame] | 158 | } |
Anton Blanchard | 9d57472 | 2014-09-24 16:59:58 +1000 | [diff] [blame] | 159 | |
Anton Blanchard | 3913fdd | 2014-09-24 16:59:57 +1000 | [diff] [blame] | 160 | #endif |
Eric W. Biederman | 2e1661d2 | 2019-05-23 11:04:24 -0500 | [diff] [blame] | 161 | force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); |
Benjamin Herrenschmidt | b5c8f0f | 2017-07-19 14:49:36 +1000 | [diff] [blame] | 162 | return 0; |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 163 | } |
| 164 | |
Souptick Joarder | 50a7ca3 | 2018-08-17 15:44:47 -0700 | [diff] [blame] | 165 | static int mm_fault_error(struct pt_regs *regs, unsigned long addr, |
| 166 | vm_fault_t fault) |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 167 | { |
| 168 | /* |
Benjamin Herrenschmidt | b5c8f0f | 2017-07-19 14:49:36 +1000 | [diff] [blame] | 169 | * Kernel page fault interrupted by SIGKILL. We have no reason to |
| 170 | * continue processing. |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 171 | */ |
Benjamin Herrenschmidt | b5c8f0f | 2017-07-19 14:49:36 +1000 | [diff] [blame] | 172 | if (fatal_signal_pending(current) && !user_mode(regs)) |
| 173 | return SIGKILL; |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 174 | |
| 175 | /* Out of memory */ |
David Rientjes | c2d23f9 | 2012-12-12 13:52:10 -0800 | [diff] [blame] | 176 | if (fault & VM_FAULT_OOM) { |
David Rientjes | c2d23f9 | 2012-12-12 13:52:10 -0800 | [diff] [blame] | 177 | /* |
| 178 | * We ran out of memory, or some other thing happened to us that |
| 179 | * made us unable to handle the page fault gracefully. |
| 180 | */ |
| 181 | if (!user_mode(regs)) |
Benjamin Herrenschmidt | b5c8f0f | 2017-07-19 14:49:36 +1000 | [diff] [blame] | 182 | return SIGSEGV; |
David Rientjes | c2d23f9 | 2012-12-12 13:52:10 -0800 | [diff] [blame] | 183 | pagefault_out_of_memory(); |
Benjamin Herrenschmidt | b5c8f0f | 2017-07-19 14:49:36 +1000 | [diff] [blame] | 184 | } else { |
| 185 | if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON| |
| 186 | VM_FAULT_HWPOISON_LARGE)) |
| 187 | return do_sigbus(regs, addr, fault); |
| 188 | else if (fault & VM_FAULT_SIGSEGV) |
| 189 | return bad_area_nosemaphore(regs, addr); |
| 190 | else |
| 191 | BUG(); |
David Rientjes | c2d23f9 | 2012-12-12 13:52:10 -0800 | [diff] [blame] | 192 | } |
Benjamin Herrenschmidt | b5c8f0f | 2017-07-19 14:49:36 +1000 | [diff] [blame] | 193 | return 0; |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 194 | } |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 195 | |
Benjamin Herrenschmidt | d3ca587 | 2017-07-19 14:49:34 +1000 | [diff] [blame] | 196 | /* Is this a bad kernel fault ? */ |
Christophe Leroy | de78a9c | 2019-04-18 16:51:20 +1000 | [diff] [blame] | 197 | static bool bad_kernel_fault(struct pt_regs *regs, unsigned long error_code, |
Michael Ellerman | 5e5be3a | 2019-04-18 16:51:25 +1000 | [diff] [blame] | 198 | unsigned long address, bool is_write) |
Benjamin Herrenschmidt | d3ca587 | 2017-07-19 14:49:34 +1000 | [diff] [blame] | 199 | { |
Xiongwei Song | 7153d4b | 2021-04-14 19:00:33 +0800 | [diff] [blame] | 200 | int is_exec = TRAP(regs) == INTERRUPT_INST_STORAGE; |
Christophe Leroy | de78a9c | 2019-04-18 16:51:20 +1000 | [diff] [blame] | 201 | |
Christophe Leroy | cd5d5e6 | 2021-07-01 11:17:08 +0000 | [diff] [blame] | 202 | if (is_exec) { |
Christophe Leroy | 0fb1c25 | 2019-04-18 16:51:19 +1000 | [diff] [blame] | 203 | pr_crit_ratelimited("kernel tried to execute %s page (%lx) - exploit attempt? (uid: %d)\n", |
| 204 | address >= TASK_SIZE ? "exec-protected" : "user", |
| 205 | address, |
| 206 | from_kuid(&init_user_ns, current_uid())); |
Michael Ellerman | 5e5be3a | 2019-04-18 16:51:25 +1000 | [diff] [blame] | 207 | |
| 208 | // Kernel exec fault is always bad |
| 209 | return true; |
Benjamin Herrenschmidt | d3ca587 | 2017-07-19 14:49:34 +1000 | [diff] [blame] | 210 | } |
Christophe Leroy | de78a9c | 2019-04-18 16:51:20 +1000 | [diff] [blame] | 211 | |
Michael Ellerman | 5e5be3a | 2019-04-18 16:51:25 +1000 | [diff] [blame] | 212 | // Kernel fault on kernel address is bad |
| 213 | if (address >= TASK_SIZE) |
| 214 | return true; |
| 215 | |
Christophe Leroy | cbd7e6c | 2020-12-09 05:29:24 +0000 | [diff] [blame] | 216 | // Read/write fault blocked by KUAP is bad, it can never succeed. |
| 217 | if (bad_kuap_fault(regs, address, is_write)) { |
| 218 | pr_crit_ratelimited("Kernel attempted to %s user page (%lx) - exploit attempt? (uid: %d)\n", |
| 219 | is_write ? "write" : "read", address, |
| 220 | from_kuid(&init_user_ns, current_uid())); |
Michael Ellerman | 5e5be3a | 2019-04-18 16:51:25 +1000 | [diff] [blame] | 221 | |
Christophe Leroy | cbd7e6c | 2020-12-09 05:29:24 +0000 | [diff] [blame] | 222 | // Fault on user outside of certain regions (eg. copy_tofrom_user()) is bad |
| 223 | if (!search_exception_tables(regs->nip)) |
| 224 | return true; |
| 225 | |
| 226 | // Read/write fault in a valid region (the exception table search passed |
| 227 | // above), but blocked by KUAP is bad, it can never succeed. |
Christophe Leroy | 3dc12df | 2020-12-09 05:29:23 +0000 | [diff] [blame] | 228 | return WARN(true, "Bug: %s fault blocked by KUAP!", is_write ? "Write" : "Read"); |
Christophe Leroy | cbd7e6c | 2020-12-09 05:29:24 +0000 | [diff] [blame] | 229 | } |
Michael Ellerman | 5e5be3a | 2019-04-18 16:51:25 +1000 | [diff] [blame] | 230 | |
Christophe Leroy | cbd7e6c | 2020-12-09 05:29:24 +0000 | [diff] [blame] | 231 | // What's left? Kernel fault on user and allowed by KUAP in the faulting context. |
Michael Ellerman | 5e5be3a | 2019-04-18 16:51:25 +1000 | [diff] [blame] | 232 | return false; |
Benjamin Herrenschmidt | d3ca587 | 2017-07-19 14:49:34 +1000 | [diff] [blame] | 233 | } |
| 234 | |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 235 | static bool access_pkey_error(bool is_write, bool is_exec, bool is_pkey, |
| 236 | struct vm_area_struct *vma) |
| 237 | { |
| 238 | /* |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 239 | * Make sure to check the VMA so that we do not perform |
| 240 | * faults just to hit a pkey fault as soon as we fill in a |
| 241 | * page. Only called for current mm, hence foreign == 0 |
| 242 | */ |
| 243 | if (!arch_vma_access_permitted(vma, is_write, is_exec, 0)) |
| 244 | return true; |
| 245 | |
| 246 | return false; |
| 247 | } |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 248 | |
| 249 | static bool access_error(bool is_write, bool is_exec, struct vm_area_struct *vma) |
Benjamin Herrenschmidt | bd0d63f | 2017-07-19 14:49:43 +1000 | [diff] [blame] | 250 | { |
| 251 | /* |
| 252 | * Allow execution from readable areas if the MMU does not |
| 253 | * provide separate controls over reading and executing. |
| 254 | * |
| 255 | * Note: That code used to not be enabled for 4xx/BookE. |
| 256 | * It is now as I/D cache coherency for these is done at |
| 257 | * set_pte_at() time and I see no reason why the test |
| 258 | * below wouldn't be valid on those processors. This -may- |
| 259 | * break programs compiled with a really old ABI though. |
| 260 | */ |
| 261 | if (is_exec) { |
| 262 | return !(vma->vm_flags & VM_EXEC) && |
| 263 | (cpu_has_feature(CPU_FTR_NOEXECUTE) || |
| 264 | !(vma->vm_flags & (VM_READ | VM_WRITE))); |
| 265 | } |
| 266 | |
| 267 | if (is_write) { |
| 268 | if (unlikely(!(vma->vm_flags & VM_WRITE))) |
| 269 | return true; |
| 270 | return false; |
| 271 | } |
| 272 | |
Russell Currey | 395cac7 | 2022-08-17 15:06:39 +1000 | [diff] [blame] | 273 | /* |
Russell Currey | f2c7e35 | 2023-03-10 16:08:34 +1100 | [diff] [blame] | 274 | * VM_READ, VM_WRITE and VM_EXEC all imply read permissions, as |
| 275 | * defined in protection_map[]. Read faults can only be caused by |
| 276 | * a PROT_NONE mapping, or with a PROT_EXEC-only mapping on Radix. |
Russell Currey | 395cac7 | 2022-08-17 15:06:39 +1000 | [diff] [blame] | 277 | */ |
Russell Currey | f2c7e35 | 2023-03-10 16:08:34 +1100 | [diff] [blame] | 278 | if (unlikely(!vma_is_accessible(vma))) |
Benjamin Herrenschmidt | bd0d63f | 2017-07-19 14:49:43 +1000 | [diff] [blame] | 279 | return true; |
Russell Currey | f2c7e35 | 2023-03-10 16:08:34 +1100 | [diff] [blame] | 280 | |
| 281 | if (unlikely(radix_enabled() && ((vma->vm_flags & VM_ACCESS_FLAGS) == VM_EXEC))) |
| 282 | return true; |
| 283 | |
Aneesh Kumar K.V | f2ed480 | 2018-03-07 19:06:45 +0530 | [diff] [blame] | 284 | /* |
| 285 | * We should ideally do the vma pkey access check here. But in the |
| 286 | * fault path, handle_mm_fault() also does the same check. To avoid |
| 287 | * these multiple checks, we skip it here and handle access error due |
| 288 | * to pkeys later. |
| 289 | */ |
Benjamin Herrenschmidt | bd0d63f | 2017-07-19 14:49:43 +1000 | [diff] [blame] | 290 | return false; |
| 291 | } |
| 292 | |
Benjamin Herrenschmidt | 3da0264 | 2017-07-19 14:49:37 +1000 | [diff] [blame] | 293 | #ifdef CONFIG_PPC_SMLPAR |
| 294 | static inline void cmo_account_page_fault(void) |
| 295 | { |
| 296 | if (firmware_has_feature(FW_FEATURE_CMO)) { |
| 297 | u32 page_ins; |
| 298 | |
| 299 | preempt_disable(); |
| 300 | page_ins = be32_to_cpu(get_lppaca()->page_ins); |
| 301 | page_ins += 1 << PAGE_FACTOR; |
| 302 | get_lppaca()->page_ins = cpu_to_be32(page_ins); |
| 303 | preempt_enable(); |
| 304 | } |
| 305 | } |
| 306 | #else |
| 307 | static inline void cmo_account_page_fault(void) { } |
| 308 | #endif /* CONFIG_PPC_SMLPAR */ |
| 309 | |
Aneesh Kumar K.V | 374f3f5 | 2018-11-26 20:05:04 +0530 | [diff] [blame] | 310 | static void sanity_check_fault(bool is_write, bool is_user, |
| 311 | unsigned long error_code, unsigned long address) |
Benjamin Herrenschmidt | 2865d08 | 2017-07-19 14:49:39 +1000 | [diff] [blame] | 312 | { |
| 313 | /* |
Aneesh Kumar K.V | 374f3f5 | 2018-11-26 20:05:04 +0530 | [diff] [blame] | 314 | * Userspace trying to access kernel address, we get PROTFAULT for that. |
| 315 | */ |
| 316 | if (is_user && address >= TASK_SIZE) { |
Christophe Leroy | 0f9aee0 | 2019-12-23 07:54:22 +0000 | [diff] [blame] | 317 | if ((long)address == -1) |
| 318 | return; |
| 319 | |
Aneesh Kumar K.V | 374f3f5 | 2018-11-26 20:05:04 +0530 | [diff] [blame] | 320 | pr_crit_ratelimited("%s[%d]: User access of kernel address (%lx) - exploit attempt? (uid: %d)\n", |
| 321 | current->comm, current->pid, address, |
| 322 | from_kuid(&init_user_ns, current_uid())); |
| 323 | return; |
| 324 | } |
| 325 | |
Christophe Leroy | 7ceb400 | 2020-12-09 05:29:21 +0000 | [diff] [blame] | 326 | if (!IS_ENABLED(CONFIG_PPC_BOOK3S)) |
| 327 | return; |
| 328 | |
Aneesh Kumar K.V | 374f3f5 | 2018-11-26 20:05:04 +0530 | [diff] [blame] | 329 | /* |
Benjamin Herrenschmidt | 2865d08 | 2017-07-19 14:49:39 +1000 | [diff] [blame] | 330 | * For hash translation mode, we should never get a |
| 331 | * PROTFAULT. Any update to pte to reduce access will result in us |
| 332 | * removing the hash page table entry, thus resulting in a DSISR_NOHPTE |
| 333 | * fault instead of DSISR_PROTFAULT. |
| 334 | * |
| 335 | * A pte update to relax the access will not result in a hash page table |
| 336 | * entry invalidate and hence can result in DSISR_PROTFAULT. |
| 337 | * ptep_set_access_flags() doesn't do a hpte flush. This is why we have |
| 338 | * the special !is_write in the below conditional. |
| 339 | * |
| 340 | * For platforms that doesn't supports coherent icache and do support |
| 341 | * per page noexec bit, we do setup things such that we do the |
| 342 | * sync between D/I cache via fault. But that is handled via low level |
| 343 | * hash fault code (hash_page_do_lazy_icache()) and we should not reach |
| 344 | * here in such case. |
| 345 | * |
| 346 | * For wrong access that can result in PROTFAULT, the above vma->vm_flags |
| 347 | * check should handle those and hence we should fall to the bad_area |
| 348 | * handling correctly. |
| 349 | * |
| 350 | * For embedded with per page exec support that doesn't support coherent |
| 351 | * icache we do get PROTFAULT and we handle that D/I cache sync in |
| 352 | * set_pte_at while taking the noexec/prot fault. Hence this is WARN_ON |
| 353 | * is conditional for server MMU. |
| 354 | * |
| 355 | * For radix, we can get prot fault for autonuma case, because radix |
| 356 | * page table will have them marked noaccess for user. |
| 357 | */ |
Aneesh Kumar K.V | 374f3f5 | 2018-11-26 20:05:04 +0530 | [diff] [blame] | 358 | if (radix_enabled() || is_write) |
| 359 | return; |
| 360 | |
| 361 | WARN_ON_ONCE(error_code & DSISR_PROTFAULT); |
Benjamin Herrenschmidt | 2865d08 | 2017-07-19 14:49:39 +1000 | [diff] [blame] | 362 | } |
Benjamin Herrenschmidt | 2865d08 | 2017-07-19 14:49:39 +1000 | [diff] [blame] | 363 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 364 | /* |
Benjamin Herrenschmidt | 41b464e5 | 2017-07-19 14:49:29 +1000 | [diff] [blame] | 365 | * Define the correct "is_write" bit in error_code based |
| 366 | * on the processor family |
| 367 | */ |
| 368 | #if (defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) |
| 369 | #define page_fault_is_write(__err) ((__err) & ESR_DST) |
| 370 | #else |
| 371 | #define page_fault_is_write(__err) ((__err) & DSISR_ISSTORE) |
Christophe Leroy | 5250d02 | 2020-12-09 05:29:22 +0000 | [diff] [blame] | 372 | #endif |
| 373 | |
| 374 | #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) |
| 375 | #define page_fault_is_bad(__err) (0) |
| 376 | #elif defined(CONFIG_PPC_8xx) |
Christophe Leroy | 4915349 | 2017-08-08 13:59:00 +0200 | [diff] [blame] | 377 | #define page_fault_is_bad(__err) ((__err) & DSISR_NOEXEC_OR_G) |
Benjamin Herrenschmidt | f3d96e6 | 2017-07-19 14:49:31 +1000 | [diff] [blame] | 378 | #elif defined(CONFIG_PPC64) |
Haren Myneni | 335e1a9 | 2022-09-27 18:29:27 -0700 | [diff] [blame] | 379 | static int page_fault_is_bad(unsigned long err) |
| 380 | { |
| 381 | unsigned long flag = DSISR_BAD_FAULT_64S; |
| 382 | |
| 383 | /* |
| 384 | * PAPR+ v2.11 § 14.15.3.4.1 (unreleased) |
| 385 | * If byte 0, bit 3 of pi-attribute-specifier-type in |
| 386 | * ibm,pi-features property is defined, ignore the DSI error |
| 387 | * which is caused by the paste instruction on the |
| 388 | * suspended NX window. |
| 389 | */ |
| 390 | if (mmu_has_feature(MMU_FTR_NX_DSI)) |
| 391 | flag &= ~DSISR_BAD_COPYPASTE; |
| 392 | |
| 393 | return err & flag; |
| 394 | } |
Benjamin Herrenschmidt | f3d96e6 | 2017-07-19 14:49:31 +1000 | [diff] [blame] | 395 | #else |
| 396 | #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S) |
| 397 | #endif |
Benjamin Herrenschmidt | 41b464e5 | 2017-07-19 14:49:29 +1000 | [diff] [blame] | 398 | |
| 399 | /* |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 400 | * For 600- and 800-family processors, the error_code parameter is DSISR |
Nicholas Piggin | 31d6490 | 2021-01-30 23:08:26 +1000 | [diff] [blame] | 401 | * for a data fault, SRR1 for an instruction fault. |
| 402 | * For 400-family processors the error_code parameter is ESR for a data fault, |
| 403 | * 0 for an instruction fault. |
| 404 | * For 64-bit processors, the error_code parameter is DSISR for a data access |
| 405 | * fault, SRR1 & 0x08000000 for an instruction access fault. |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 406 | * |
| 407 | * The return value is 0 if the fault was handled, or the signal |
| 408 | * number if this is a kernel fault that can't be handled here. |
| 409 | */ |
Nicholas Piggin | a008f8f | 2021-01-30 23:08:41 +1000 | [diff] [blame] | 410 | static int ___do_page_fault(struct pt_regs *regs, unsigned long address, |
Benjamin Herrenschmidt | 7afad42 | 2017-07-19 14:49:23 +1000 | [diff] [blame] | 411 | unsigned long error_code) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 412 | { |
| 413 | struct vm_area_struct * vma; |
| 414 | struct mm_struct *mm = current->mm; |
Peter Xu | dde1607 | 2020-04-01 21:08:37 -0700 | [diff] [blame] | 415 | unsigned int flags = FAULT_FLAG_DEFAULT; |
Xiongwei Song | 7153d4b | 2021-04-14 19:00:33 +0800 | [diff] [blame] | 416 | int is_exec = TRAP(regs) == INTERRUPT_INST_STORAGE; |
Christophe Leroy | da929f6 | 2017-04-19 14:56:30 +0200 | [diff] [blame] | 417 | int is_user = user_mode(regs); |
Benjamin Herrenschmidt | 41b464e5 | 2017-07-19 14:49:29 +1000 | [diff] [blame] | 418 | int is_write = page_fault_is_write(error_code); |
Souptick Joarder | 50a7ca3 | 2018-08-17 15:44:47 -0700 | [diff] [blame] | 419 | vm_fault_t fault, major = 0; |
Anshuman Khandual | b98cca4 | 2019-07-16 16:28:00 -0700 | [diff] [blame] | 420 | bool kprobe_fault = kprobe_page_fault(regs, 11); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 421 | |
Anshuman Khandual | b98cca4 | 2019-07-16 16:28:00 -0700 | [diff] [blame] | 422 | if (unlikely(debugger_fault_handler(regs) || kprobe_fault)) |
Benjamin Herrenschmidt | 65d47fd | 2017-07-19 14:49:33 +1000 | [diff] [blame] | 423 | return 0; |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 424 | |
Benjamin Herrenschmidt | f3d96e6 | 2017-07-19 14:49:31 +1000 | [diff] [blame] | 425 | if (unlikely(page_fault_is_bad(error_code))) { |
Benjamin Herrenschmidt | 65d47fd | 2017-07-19 14:49:33 +1000 | [diff] [blame] | 426 | if (is_user) { |
Benjamin Herrenschmidt | f3d96e6 | 2017-07-19 14:49:31 +1000 | [diff] [blame] | 427 | _exception(SIGBUS, regs, BUS_OBJERR, address); |
Benjamin Herrenschmidt | 65d47fd | 2017-07-19 14:49:33 +1000 | [diff] [blame] | 428 | return 0; |
| 429 | } |
| 430 | return SIGBUS; |
Benjamin Herrenschmidt | e6c8290 | 2017-07-19 14:49:30 +1000 | [diff] [blame] | 431 | } |
Benjamin Herrenschmidt | e6c8290 | 2017-07-19 14:49:30 +1000 | [diff] [blame] | 432 | |
Benjamin Herrenschmidt | 2865d08 | 2017-07-19 14:49:39 +1000 | [diff] [blame] | 433 | /* Additional sanity check(s) */ |
Aneesh Kumar K.V | 374f3f5 | 2018-11-26 20:05:04 +0530 | [diff] [blame] | 434 | sanity_check_fault(is_write, is_user, error_code, address); |
Benjamin Herrenschmidt | 2865d08 | 2017-07-19 14:49:39 +1000 | [diff] [blame] | 435 | |
Benjamin Herrenschmidt | d7df244 | 2017-02-03 17:10:28 +1100 | [diff] [blame] | 436 | /* |
| 437 | * The kernel should never take an execute fault nor should it |
Christophe Leroy | de78a9c | 2019-04-18 16:51:20 +1000 | [diff] [blame] | 438 | * take a page fault to a kernel address or a page fault to a user |
| 439 | * address outside of dedicated places |
Benjamin Herrenschmidt | d7df244 | 2017-02-03 17:10:28 +1100 | [diff] [blame] | 440 | */ |
Christophe Leroy | 90cbac0 | 2021-03-04 14:35:09 +0000 | [diff] [blame] | 441 | if (unlikely(!is_user && bad_kernel_fault(regs, error_code, address, is_write))) { |
| 442 | if (kfence_handle_page_fault(address, is_write, regs)) |
| 443 | return 0; |
| 444 | |
Benjamin Herrenschmidt | 65d47fd | 2017-07-19 14:49:33 +1000 | [diff] [blame] | 445 | return SIGSEGV; |
Christophe Leroy | 90cbac0 | 2021-03-04 14:35:09 +0000 | [diff] [blame] | 446 | } |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 447 | |
Benjamin Herrenschmidt | 11ccdd3 | 2017-07-19 14:49:40 +1000 | [diff] [blame] | 448 | /* |
| 449 | * If we're in an interrupt, have no user context or are running |
| 450 | * in a region with pagefaults disabled then we must not take the fault |
| 451 | */ |
| 452 | if (unlikely(faulthandler_disabled() || !mm)) { |
| 453 | if (is_user) |
| 454 | printk_ratelimited(KERN_ERR "Page fault in user mode" |
| 455 | " with faulthandler_disabled()=%d" |
| 456 | " mm=%p\n", |
| 457 | faulthandler_disabled(), mm); |
| 458 | return bad_area_nosemaphore(regs, address); |
| 459 | } |
| 460 | |
Nicholas Piggin | e6f8a6c | 2021-01-30 23:08:39 +1000 | [diff] [blame] | 461 | interrupt_cond_local_irq_enable(regs); |
Benjamin Herrenschmidt | a546498 | 2012-03-07 16:48:45 +1100 | [diff] [blame] | 462 | |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 463 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
Peter Zijlstra | 7dd1fcc | 2009-03-13 12:21:33 +0100 | [diff] [blame] | 464 | |
Aneesh Kumar K.V | 69e044dd | 2013-09-10 18:44:42 +0530 | [diff] [blame] | 465 | /* |
Michel Lespinasse | c1e8d7c | 2020-06-08 21:33:54 -0700 | [diff] [blame] | 466 | * We want to do this outside mmap_lock, because reading code around nip |
Aneesh Kumar K.V | 69e044dd | 2013-09-10 18:44:42 +0530 | [diff] [blame] | 467 | * can result in fault, which will cause a deadlock when called with |
Michel Lespinasse | c1e8d7c | 2020-06-08 21:33:54 -0700 | [diff] [blame] | 468 | * mmap_lock held |
Aneesh Kumar K.V | 69e044dd | 2013-09-10 18:44:42 +0530 | [diff] [blame] | 469 | */ |
Christophe Leroy | da929f6 | 2017-04-19 14:56:30 +0200 | [diff] [blame] | 470 | if (is_user) |
Johannes Weiner | 759496b | 2013-09-12 15:13:39 -0700 | [diff] [blame] | 471 | flags |= FAULT_FLAG_USER; |
Benjamin Herrenschmidt | d2e0d2c | 2017-07-19 14:49:42 +1000 | [diff] [blame] | 472 | if (is_write) |
| 473 | flags |= FAULT_FLAG_WRITE; |
| 474 | if (is_exec) |
| 475 | flags |= FAULT_FLAG_INSTRUCTION; |
Johannes Weiner | 759496b | 2013-09-12 15:13:39 -0700 | [diff] [blame] | 476 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 477 | /* When running in the kernel we expect faults to occur only to |
| 478 | * addresses in user space. All other faults represent errors in the |
Anton Blanchard | fc5266e | 2006-04-01 11:33:12 +1100 | [diff] [blame] | 479 | * kernel and should generate an OOPS. Unfortunately, in the case of an |
Michel Lespinasse | c1e8d7c | 2020-06-08 21:33:54 -0700 | [diff] [blame] | 480 | * erroneous fault occurring in a code path which already holds mmap_lock |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 481 | * we will deadlock attempting to validate the fault against the |
| 482 | * address space. Luckily the kernel only validly references user |
| 483 | * space from well defined areas of code, which are listed in the |
| 484 | * exceptions table. |
| 485 | * |
| 486 | * As the vast majority of faults will be valid we will only perform |
Anton Blanchard | fc5266e | 2006-04-01 11:33:12 +1100 | [diff] [blame] | 487 | * the source reference check when there is a possibility of a deadlock. |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 488 | * Attempt to lock the address space, if we cannot we then validate the |
| 489 | * source. If this is invalid we can skip the address space check, |
| 490 | * thus avoiding the deadlock. |
| 491 | */ |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 492 | if (unlikely(!mmap_read_trylock(mm))) { |
Christophe Leroy | da929f6 | 2017-04-19 14:56:30 +0200 | [diff] [blame] | 493 | if (!is_user && !search_exception_tables(regs->nip)) |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 494 | return bad_area_nosemaphore(regs, address); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 495 | |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 496 | retry: |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 497 | mmap_read_lock(mm); |
Benjamin Herrenschmidt | a546498 | 2012-03-07 16:48:45 +1100 | [diff] [blame] | 498 | } else { |
| 499 | /* |
| 500 | * The above down_read_trylock() might have succeeded in |
| 501 | * which case we'll have missed the might_sleep() from |
| 502 | * down_read(): |
| 503 | */ |
| 504 | might_sleep(); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 505 | } |
| 506 | |
| 507 | vma = find_vma(mm, address); |
Benjamin Herrenschmidt | b15021d | 2017-07-19 14:49:41 +1000 | [diff] [blame] | 508 | if (unlikely(!vma)) |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 509 | return bad_area(regs, address); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 510 | |
Michael Ellerman | 773b3e5 | 2020-07-24 19:25:27 +1000 | [diff] [blame] | 511 | if (unlikely(vma->vm_start > address)) { |
| 512 | if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) |
Christophe Leroy | 0e36b0d | 2018-05-23 10:53:22 +0200 | [diff] [blame] | 513 | return bad_area(regs, address); |
| 514 | |
Michael Ellerman | 773b3e5 | 2020-07-24 19:25:27 +1000 | [diff] [blame] | 515 | if (unlikely(expand_stack(vma, address))) |
| 516 | return bad_area(regs, address); |
Christophe Leroy | 0e36b0d | 2018-05-23 10:53:22 +0200 | [diff] [blame] | 517 | } |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 518 | |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 519 | if (unlikely(access_pkey_error(is_write, is_exec, |
| 520 | (error_code & DSISR_KEYFAULT), vma))) |
| 521 | return bad_access_pkey(regs, address, vma); |
Aneesh Kumar K.V | fe4a685 | 2020-05-05 12:47:08 +0530 | [diff] [blame] | 522 | |
Benjamin Herrenschmidt | bd0d63f | 2017-07-19 14:49:43 +1000 | [diff] [blame] | 523 | if (unlikely(access_error(is_write, is_exec, vma))) |
John Sperbeck | ecb101a | 2017-12-31 21:24:58 -0800 | [diff] [blame] | 524 | return bad_access(regs, address); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 525 | |
| 526 | /* |
| 527 | * If for any reason at all we couldn't handle the fault, |
| 528 | * make sure we exit gracefully rather than endlessly redo |
| 529 | * the fault. |
| 530 | */ |
Peter Xu | 428fdc0 | 2020-08-11 18:38:31 -0700 | [diff] [blame] | 531 | fault = handle_mm_fault(vma, address, flags, regs); |
Ram Pai | e6c2a47 | 2018-01-18 17:50:40 -0800 | [diff] [blame] | 532 | |
Benjamin Herrenschmidt | f43bb27 | 2017-07-19 14:49:44 +1000 | [diff] [blame] | 533 | major |= fault & VM_FAULT_MAJOR; |
Laurent Dufour | 14c02e41 | 2017-02-14 17:45:11 +0100 | [diff] [blame] | 534 | |
Peter Xu | c9a0dad | 2020-04-01 21:08:22 -0700 | [diff] [blame] | 535 | if (fault_signal_pending(fault, regs)) |
| 536 | return user_mode(regs) ? 0 : SIGBUS; |
| 537 | |
Peter Xu | d927252 | 2022-05-30 14:34:50 -0400 | [diff] [blame] | 538 | /* The fault is fully completed (including releasing mmap lock) */ |
| 539 | if (fault & VM_FAULT_COMPLETED) |
| 540 | goto out; |
| 541 | |
Laurent Dufour | 14c02e41 | 2017-02-14 17:45:11 +0100 | [diff] [blame] | 542 | /* |
Michel Lespinasse | c1e8d7c | 2020-06-08 21:33:54 -0700 | [diff] [blame] | 543 | * Handle the retry right now, the mmap_lock has been released in that |
Laurent Dufour | 14c02e41 | 2017-02-14 17:45:11 +0100 | [diff] [blame] | 544 | * case. |
| 545 | */ |
| 546 | if (unlikely(fault & VM_FAULT_RETRY)) { |
Qi Zheng | 36ef159 | 2022-01-14 14:05:51 -0800 | [diff] [blame] | 547 | flags |= FAULT_FLAG_TRIED; |
| 548 | goto retry; |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 549 | } |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 550 | |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 551 | mmap_read_unlock(current->mm); |
Benjamin Herrenschmidt | b5c8f0f | 2017-07-19 14:49:36 +1000 | [diff] [blame] | 552 | |
| 553 | if (unlikely(fault & VM_FAULT_ERROR)) |
| 554 | return mm_fault_error(regs, address, fault); |
| 555 | |
Peter Xu | d927252 | 2022-05-30 14:34:50 -0400 | [diff] [blame] | 556 | out: |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 557 | /* |
Laurent Dufour | 14c02e41 | 2017-02-14 17:45:11 +0100 | [diff] [blame] | 558 | * Major/minor page fault accounting. |
Benjamin Herrenschmidt | 9be7257 | 2012-03-01 18:14:45 +1100 | [diff] [blame] | 559 | */ |
Peter Xu | 428fdc0 | 2020-08-11 18:38:31 -0700 | [diff] [blame] | 560 | if (major) |
Benjamin Herrenschmidt | 3da0264 | 2017-07-19 14:49:37 +1000 | [diff] [blame] | 561 | cmo_account_page_fault(); |
Peter Xu | 428fdc0 | 2020-08-11 18:38:31 -0700 | [diff] [blame] | 562 | |
Benjamin Herrenschmidt | c335060 | 2017-07-19 14:49:35 +1000 | [diff] [blame] | 563 | return 0; |
Benjamin Herrenschmidt | 7afad42 | 2017-07-19 14:49:23 +1000 | [diff] [blame] | 564 | } |
Nicholas Piggin | a008f8f | 2021-01-30 23:08:41 +1000 | [diff] [blame] | 565 | NOKPROBE_SYMBOL(___do_page_fault); |
Benjamin Herrenschmidt | 7afad42 | 2017-07-19 14:49:23 +1000 | [diff] [blame] | 566 | |
Nicholas Piggin | c45ba4f4 | 2021-03-16 20:42:03 +1000 | [diff] [blame] | 567 | static __always_inline void __do_page_fault(struct pt_regs *regs) |
Benjamin Herrenschmidt | 7afad42 | 2017-07-19 14:49:23 +1000 | [diff] [blame] | 568 | { |
Nicholas Piggin | 4cb8428 | 2021-01-30 23:08:23 +1000 | [diff] [blame] | 569 | long err; |
| 570 | |
Nicholas Piggin | a008f8f | 2021-01-30 23:08:41 +1000 | [diff] [blame] | 571 | err = ___do_page_fault(regs, regs->dar, regs->dsisr); |
Nicholas Piggin | c45ba4f4 | 2021-03-16 20:42:03 +1000 | [diff] [blame] | 572 | if (unlikely(err)) |
| 573 | bad_page_fault(regs, err); |
Nicholas Piggin | a008f8f | 2021-01-30 23:08:41 +1000 | [diff] [blame] | 574 | } |
Christophe Leroy | 5f1888a | 2020-12-09 05:29:25 +0000 | [diff] [blame] | 575 | |
Nicholas Piggin | c45ba4f4 | 2021-03-16 20:42:03 +1000 | [diff] [blame] | 576 | DEFINE_INTERRUPT_HANDLER(do_page_fault) |
Nicholas Piggin | a008f8f | 2021-01-30 23:08:41 +1000 | [diff] [blame] | 577 | { |
Nicholas Piggin | c45ba4f4 | 2021-03-16 20:42:03 +1000 | [diff] [blame] | 578 | __do_page_fault(regs); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 579 | } |
| 580 | |
Nicholas Piggin | a008f8f | 2021-01-30 23:08:41 +1000 | [diff] [blame] | 581 | #ifdef CONFIG_PPC_BOOK3S_64 |
| 582 | /* Same as do_page_fault but interrupt entry has already run in do_hash_fault */ |
Nicholas Piggin | c45ba4f4 | 2021-03-16 20:42:03 +1000 | [diff] [blame] | 583 | void hash__do_page_fault(struct pt_regs *regs) |
Nicholas Piggin | a008f8f | 2021-01-30 23:08:41 +1000 | [diff] [blame] | 584 | { |
Nicholas Piggin | c45ba4f4 | 2021-03-16 20:42:03 +1000 | [diff] [blame] | 585 | __do_page_fault(regs); |
Nicholas Piggin | a008f8f | 2021-01-30 23:08:41 +1000 | [diff] [blame] | 586 | } |
| 587 | NOKPROBE_SYMBOL(hash__do_page_fault); |
| 588 | #endif |
| 589 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 590 | /* |
| 591 | * bad_page_fault is called when we have a bad access from the kernel. |
| 592 | * It is called from the DSI and ISI handlers in head.S and from some |
| 593 | * of the procedures in traps.c. |
| 594 | */ |
Nicholas Piggin | c45ba4f4 | 2021-03-16 20:42:03 +1000 | [diff] [blame] | 595 | static void __bad_page_fault(struct pt_regs *regs, int sig) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 596 | { |
Christophe Leroy | 46ddcb3 | 2019-08-21 15:21:55 +0000 | [diff] [blame] | 597 | int is_write = page_fault_is_write(regs->dsisr); |
Michael Ellerman | d4679ac | 2022-02-22 22:34:49 +1100 | [diff] [blame] | 598 | const char *msg; |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 599 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 600 | /* kernel has accessed a bad area */ |
Olof Johansson | 723925b | 2005-11-06 14:54:36 -0800 | [diff] [blame] | 601 | |
Michael Ellerman | d4679ac | 2022-02-22 22:34:49 +1100 | [diff] [blame] | 602 | if (regs->dar < PAGE_SIZE) |
| 603 | msg = "Kernel NULL pointer dereference"; |
| 604 | else |
| 605 | msg = "Unable to handle kernel data access"; |
| 606 | |
Benjamin Herrenschmidt | 2271db2 | 2018-01-12 13:28:49 +1100 | [diff] [blame] | 607 | switch (TRAP(regs)) { |
Xiongwei Song | 7153d4b | 2021-04-14 19:00:33 +0800 | [diff] [blame] | 608 | case INTERRUPT_DATA_STORAGE: |
Xiongwei Song | 7153d4b | 2021-04-14 19:00:33 +0800 | [diff] [blame] | 609 | case INTERRUPT_H_DATA_STORAGE: |
Michael Ellerman | d4679ac | 2022-02-22 22:34:49 +1100 | [diff] [blame] | 610 | pr_alert("BUG: %s on %s at 0x%08lx\n", msg, |
Christophe Leroy | 46ddcb3 | 2019-08-21 15:21:55 +0000 | [diff] [blame] | 611 | is_write ? "write" : "read", regs->dar); |
Michael Ellerman | a416dd8 | 2006-11-08 10:22:59 +1100 | [diff] [blame] | 612 | break; |
Michael Ellerman | d4679ac | 2022-02-22 22:34:49 +1100 | [diff] [blame] | 613 | case INTERRUPT_DATA_SEGMENT: |
| 614 | pr_alert("BUG: %s at 0x%08lx\n", msg, regs->dar); |
| 615 | break; |
Xiongwei Song | 7153d4b | 2021-04-14 19:00:33 +0800 | [diff] [blame] | 616 | case INTERRUPT_INST_STORAGE: |
| 617 | case INTERRUPT_INST_SEGMENT: |
Christophe Leroy | 49a502e | 2018-12-14 15:23:33 +0000 | [diff] [blame] | 618 | pr_alert("BUG: Unable to handle kernel instruction fetch%s", |
| 619 | regs->nip < PAGE_SIZE ? " (NULL pointer?)\n" : "\n"); |
Michael Ellerman | a416dd8 | 2006-11-08 10:22:59 +1100 | [diff] [blame] | 620 | break; |
Xiongwei Song | 7153d4b | 2021-04-14 19:00:33 +0800 | [diff] [blame] | 621 | case INTERRUPT_ALIGNMENT: |
Christophe Leroy | 49a502e | 2018-12-14 15:23:33 +0000 | [diff] [blame] | 622 | pr_alert("BUG: Unable to handle kernel unaligned access at 0x%08lx\n", |
| 623 | regs->dar); |
Anton Blanchard | eab861a | 2015-07-02 14:56:20 +1000 | [diff] [blame] | 624 | break; |
Michael Ellerman | a416dd8 | 2006-11-08 10:22:59 +1100 | [diff] [blame] | 625 | default: |
Christophe Leroy | 49a502e | 2018-12-14 15:23:33 +0000 | [diff] [blame] | 626 | pr_alert("BUG: Unable to handle unknown paging fault at 0x%08lx\n", |
| 627 | regs->dar); |
Michael Ellerman | a416dd8 | 2006-11-08 10:22:59 +1100 | [diff] [blame] | 628 | break; |
Olof Johansson | 723925b | 2005-11-06 14:54:36 -0800 | [diff] [blame] | 629 | } |
| 630 | printk(KERN_ALERT "Faulting instruction address: 0x%08lx\n", |
| 631 | regs->nip); |
| 632 | |
Aaron Tomlin | a70857e | 2014-09-12 14:16:18 +0100 | [diff] [blame] | 633 | if (task_stack_end_corrupted(current)) |
Anton Blanchard | 28b5499 | 2010-08-24 13:15:28 +0000 | [diff] [blame] | 634 | printk(KERN_ALERT "Thread overran stack, or stack corrupted\n"); |
| 635 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 636 | die("Kernel access of bad area", regs, sig); |
| 637 | } |
Christophe Leroy | 5f1888a | 2020-12-09 05:29:25 +0000 | [diff] [blame] | 638 | |
Nicholas Piggin | 8458c62 | 2021-01-30 23:08:21 +1000 | [diff] [blame] | 639 | void bad_page_fault(struct pt_regs *regs, int sig) |
Christophe Leroy | 5f1888a | 2020-12-09 05:29:25 +0000 | [diff] [blame] | 640 | { |
| 641 | const struct exception_table_entry *entry; |
| 642 | |
| 643 | /* Are we prepared to handle this fault? */ |
| 644 | entry = search_exception_tables(instruction_pointer(regs)); |
| 645 | if (entry) |
| 646 | instruction_pointer_set(regs, extable_fixup(entry)); |
| 647 | else |
Nicholas Piggin | 8458c62 | 2021-01-30 23:08:21 +1000 | [diff] [blame] | 648 | __bad_page_fault(regs, sig); |
Christophe Leroy | 5f1888a | 2020-12-09 05:29:25 +0000 | [diff] [blame] | 649 | } |
Nicholas Piggin | 71f4797 | 2021-01-30 23:08:22 +1000 | [diff] [blame] | 650 | |
| 651 | #ifdef CONFIG_PPC_BOOK3S_64 |
Nicholas Piggin | 3a96570 | 2021-01-30 23:08:38 +1000 | [diff] [blame] | 652 | DEFINE_INTERRUPT_HANDLER(do_bad_page_fault_segv) |
Nicholas Piggin | 71f4797 | 2021-01-30 23:08:22 +1000 | [diff] [blame] | 653 | { |
| 654 | bad_page_fault(regs, SIGSEGV); |
| 655 | } |
Nicholas Piggin | 935b534 | 2021-12-02 00:41:39 +1000 | [diff] [blame] | 656 | |
| 657 | /* |
| 658 | * In radix, segment interrupts indicate the EA is not addressable by the |
| 659 | * page table geometry, so they are always sent here. |
| 660 | * |
| 661 | * In hash, this is called if do_slb_fault returns error. Typically it is |
| 662 | * because the EA was outside the region allowed by software. |
| 663 | */ |
| 664 | DEFINE_INTERRUPT_HANDLER(do_bad_segment_interrupt) |
| 665 | { |
| 666 | int err = regs->result; |
| 667 | |
| 668 | if (err == -EFAULT) { |
| 669 | if (user_mode(regs)) |
| 670 | _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar); |
| 671 | else |
| 672 | bad_page_fault(regs, SIGSEGV); |
| 673 | } else if (err == -EINVAL) { |
| 674 | unrecoverable_exception(regs); |
| 675 | } else { |
| 676 | BUG(); |
| 677 | } |
| 678 | } |
Nicholas Piggin | 71f4797 | 2021-01-30 23:08:22 +1000 | [diff] [blame] | 679 | #endif |