blob: 8135da06e0a44e4e320e7157e3b9fec364e67793 [file] [log] [blame]
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001/*
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Derived from "arch/i386/mm/fault.c"
6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 *
8 * Modified by Cort Dougan and Paul Mackerras.
9 *
10 * Modified for PPC64 by Dave Engebretsen (engebret@ibm.com)
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version
15 * 2 of the License, or (at your option) any later version.
16 */
17
Paul Mackerras14cf11a2005-09-26 16:04:21 +100018#include <linux/signal.h>
19#include <linux/sched.h>
20#include <linux/kernel.h>
21#include <linux/errno.h>
22#include <linux/string.h>
23#include <linux/types.h>
24#include <linux/ptrace.h>
25#include <linux/mman.h>
26#include <linux/mm.h>
27#include <linux/interrupt.h>
28#include <linux/highmem.h>
29#include <linux/module.h>
30#include <linux/kprobes.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070031#include <linux/kdebug.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100032
33#include <asm/page.h>
34#include <asm/pgtable.h>
35#include <asm/mmu.h>
36#include <asm/mmu_context.h>
37#include <asm/system.h>
38#include <asm/uaccess.h>
39#include <asm/tlbflush.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100040#include <asm/siginfo.h>
41
Christoph Hellwig9f90b992007-04-30 11:56:46 +010042
Anil S Keshavamurthy4f9e87c2006-06-26 00:25:27 -070043#ifdef CONFIG_KPROBES
Christoph Hellwig9f90b992007-04-30 11:56:46 +010044static inline int notify_page_fault(struct pt_regs *regs)
Anil S Keshavamurthy4f9e87c2006-06-26 00:25:27 -070045{
Christoph Hellwig9f90b992007-04-30 11:56:46 +010046 int ret = 0;
Anil S Keshavamurthy4f9e87c2006-06-26 00:25:27 -070047
Christoph Hellwig9f90b992007-04-30 11:56:46 +010048 /* kprobe_running() needs smp_processor_id() */
49 if (!user_mode(regs)) {
50 preempt_disable();
51 if (kprobe_running() && kprobe_fault_handler(regs, 11))
52 ret = 1;
53 preempt_enable();
54 }
Anil S Keshavamurthy4f9e87c2006-06-26 00:25:27 -070055
Christoph Hellwig9f90b992007-04-30 11:56:46 +010056 return ret;
Anil S Keshavamurthy4f9e87c2006-06-26 00:25:27 -070057}
58#else
Christoph Hellwig9f90b992007-04-30 11:56:46 +010059static inline int notify_page_fault(struct pt_regs *regs)
Anil S Keshavamurthy4f9e87c2006-06-26 00:25:27 -070060{
Christoph Hellwig9f90b992007-04-30 11:56:46 +010061 return 0;
Anil S Keshavamurthy4f9e87c2006-06-26 00:25:27 -070062}
63#endif
64
Paul Mackerras14cf11a2005-09-26 16:04:21 +100065/*
66 * Check whether the instruction at regs->nip is a store using
67 * an update addressing form which will update r1.
68 */
69static int store_updates_sp(struct pt_regs *regs)
70{
71 unsigned int inst;
72
73 if (get_user(inst, (unsigned int __user *)regs->nip))
74 return 0;
75 /* check for 1 in the rA field */
76 if (((inst >> 16) & 0x1f) != 1)
77 return 0;
78 /* check major opcode */
79 switch (inst >> 26) {
80 case 37: /* stwu */
81 case 39: /* stbu */
82 case 45: /* sthu */
83 case 53: /* stfsu */
84 case 55: /* stfdu */
85 return 1;
86 case 62: /* std or stdu */
87 return (inst & 3) == 1;
88 case 31:
89 /* check minor opcode */
90 switch ((inst >> 1) & 0x3ff) {
91 case 181: /* stdux */
92 case 183: /* stwux */
93 case 247: /* stbux */
94 case 439: /* sthux */
95 case 695: /* stfsux */
96 case 759: /* stfdux */
97 return 1;
98 }
99 }
100 return 0;
101}
102
Kumar Galacffb09c2005-10-26 09:55:41 -0500103#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
Anton Blanchardbce6c5f2006-01-09 15:47:04 +1100104static void do_dabr(struct pt_regs *regs, unsigned long address,
105 unsigned long error_code)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000106{
107 siginfo_t info;
108
109 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
110 11, SIGSEGV) == NOTIFY_STOP)
111 return;
112
113 if (debugger_dabr_match(regs))
114 return;
115
116 /* Clear the DABR */
117 set_dabr(0);
118
119 /* Deliver the signal to userspace */
120 info.si_signo = SIGTRAP;
121 info.si_errno = 0;
122 info.si_code = TRAP_HWBKPT;
Anton Blanchardbce6c5f2006-01-09 15:47:04 +1100123 info.si_addr = (void __user *)address;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000124 force_sig_info(SIGTRAP, &info, current);
125}
Kumar Galacffb09c2005-10-26 09:55:41 -0500126#endif /* !(CONFIG_4xx || CONFIG_BOOKE)*/
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000127
128/*
129 * For 600- and 800-family processors, the error_code parameter is DSISR
130 * for a data fault, SRR1 for an instruction fault. For 400-family processors
131 * the error_code parameter is ESR for a data fault, 0 for an instruction
132 * fault.
133 * For 64-bit processors, the error_code parameter is
134 * - DSISR for a non-SLB data access fault,
135 * - SRR1 & 0x08000000 for a non-SLB instruction access fault
136 * - 0 any SLB fault.
137 *
138 * The return value is 0 if the fault was handled, or the signal
139 * number if this is a kernel fault that can't be handled here.
140 */
141int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
142 unsigned long error_code)
143{
144 struct vm_area_struct * vma;
145 struct mm_struct *mm = current->mm;
146 siginfo_t info;
147 int code = SEGV_MAPERR;
Nick Piggin83c54072007-07-19 01:47:05 -0700148 int is_write = 0, ret;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000149 int trap = TRAP(regs);
150 int is_exec = trap == 0x400;
151
152#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
153 /*
154 * Fortunately the bit assignments in SRR1 for an instruction
155 * fault and DSISR for a data fault are mostly the same for the
156 * bits we are interested in. But there are some bits which
157 * indicate errors in DSISR but can validly be set in SRR1.
158 */
159 if (trap == 0x400)
160 error_code &= 0x48200000;
161 else
162 is_write = error_code & DSISR_ISSTORE;
163#else
164 is_write = error_code & ESR_DST;
165#endif /* CONFIG_4xx || CONFIG_BOOKE */
166
Christoph Hellwig9f90b992007-04-30 11:56:46 +0100167 if (notify_page_fault(regs))
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000168 return 0;
169
170 if (trap == 0x300) {
171 if (debugger_fault_handler(regs))
172 return 0;
173 }
174
175 /* On a kernel SLB miss we can only check for a valid exception entry */
176 if (!user_mode(regs) && (address >= TASK_SIZE))
177 return SIGSEGV;
178
179#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
180 if (error_code & DSISR_DABRMATCH) {
181 /* DABR match */
Anton Blanchardbce6c5f2006-01-09 15:47:04 +1100182 do_dabr(regs, address, error_code);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000183 return 0;
184 }
185#endif /* !(CONFIG_4xx || CONFIG_BOOKE)*/
186
187 if (in_atomic() || mm == NULL) {
188 if (!user_mode(regs))
189 return SIGSEGV;
190 /* in_atomic() in user mode is really bad,
191 as is current->mm == NULL. */
192 printk(KERN_EMERG "Page fault in user mode with"
193 "in_atomic() = %d mm = %p\n", in_atomic(), mm);
194 printk(KERN_EMERG "NIP = %lx MSR = %lx\n",
195 regs->nip, regs->msr);
196 die("Weird page fault", regs, SIGSEGV);
197 }
198
199 /* When running in the kernel we expect faults to occur only to
200 * addresses in user space. All other faults represent errors in the
Anton Blanchardfc5266e2006-04-01 11:33:12 +1100201 * kernel and should generate an OOPS. Unfortunately, in the case of an
202 * erroneous fault occurring in a code path which already holds mmap_sem
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000203 * we will deadlock attempting to validate the fault against the
204 * address space. Luckily the kernel only validly references user
205 * space from well defined areas of code, which are listed in the
206 * exceptions table.
207 *
208 * As the vast majority of faults will be valid we will only perform
Anton Blanchardfc5266e2006-04-01 11:33:12 +1100209 * the source reference check when there is a possibility of a deadlock.
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000210 * Attempt to lock the address space, if we cannot we then validate the
211 * source. If this is invalid we can skip the address space check,
212 * thus avoiding the deadlock.
213 */
214 if (!down_read_trylock(&mm->mmap_sem)) {
215 if (!user_mode(regs) && !search_exception_tables(regs->nip))
216 goto bad_area_nosemaphore;
217
218 down_read(&mm->mmap_sem);
219 }
220
221 vma = find_vma(mm, address);
222 if (!vma)
223 goto bad_area;
224 if (vma->vm_start <= address)
225 goto good_area;
226 if (!(vma->vm_flags & VM_GROWSDOWN))
227 goto bad_area;
228
229 /*
230 * N.B. The POWER/Open ABI allows programs to access up to
231 * 288 bytes below the stack pointer.
232 * The kernel signal delivery code writes up to about 1.5kB
233 * below the stack pointer (r1) before decrementing it.
234 * The exec code can write slightly over 640kB to the stack
235 * before setting the user r1. Thus we allow the stack to
236 * expand to 1MB without further checks.
237 */
238 if (address + 0x100000 < vma->vm_end) {
239 /* get user regs even if this fault is in kernel mode */
240 struct pt_regs *uregs = current->thread.regs;
241 if (uregs == NULL)
242 goto bad_area;
243
244 /*
245 * A user-mode access to an address a long way below
246 * the stack pointer is only valid if the instruction
247 * is one which would update the stack pointer to the
248 * address accessed if the instruction completed,
249 * i.e. either stwu rs,n(r1) or stwux rs,r1,rb
250 * (or the byte, halfword, float or double forms).
251 *
252 * If we don't check this then any write to the area
253 * between the last mapped region and the stack will
254 * expand the stack rather than segfaulting.
255 */
256 if (address + 2048 < uregs->gpr[1]
257 && (!user_mode(regs) || !store_updates_sp(regs)))
258 goto bad_area;
259 }
260 if (expand_stack(vma, address))
261 goto bad_area;
262
263good_area:
264 code = SEGV_ACCERR;
265#if defined(CONFIG_6xx)
266 if (error_code & 0x95700000)
267 /* an error such as lwarx to I/O controller space,
268 address matching DABR, eciwx, etc. */
269 goto bad_area;
270#endif /* CONFIG_6xx */
271#if defined(CONFIG_8xx)
272 /* The MPC8xx seems to always set 0x80000000, which is
273 * "undefined". Of those that can be set, this is the only
274 * one which seems bad.
275 */
276 if (error_code & 0x10000000)
277 /* Guarded storage error. */
278 goto bad_area;
279#endif /* CONFIG_8xx */
280
281 if (is_exec) {
Segher Boessenkool9ba4ace2007-06-20 01:07:04 +1000282#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000283 /* protection fault */
284 if (error_code & DSISR_PROTFAULT)
285 goto bad_area;
Paul Mackerras08ae6cc2007-07-19 10:00:20 +1000286 /*
287 * Allow execution from readable areas if the MMU does not
288 * provide separate controls over reading and executing.
289 */
290 if (!(vma->vm_flags & VM_EXEC) &&
291 (cpu_has_feature(CPU_FTR_NOEXECUTE) ||
292 !(vma->vm_flags & (VM_READ | VM_WRITE))))
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000293 goto bad_area;
Segher Boessenkool9ba4ace2007-06-20 01:07:04 +1000294#else
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000295 pte_t *ptep;
Eugene Suroveginbab70a42006-03-28 10:13:12 -0800296 pmd_t *pmdp;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000297
298 /* Since 4xx/Book-E supports per-page execute permission,
299 * we lazily flush dcache to icache. */
300 ptep = NULL;
Eugene Suroveginbab70a42006-03-28 10:13:12 -0800301 if (get_pteptr(mm, address, &ptep, &pmdp)) {
302 spinlock_t *ptl = pte_lockptr(mm, pmdp);
303 spin_lock(ptl);
304 if (pte_present(*ptep)) {
305 struct page *page = pte_page(*ptep);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000306
Eugene Suroveginbab70a42006-03-28 10:13:12 -0800307 if (!test_bit(PG_arch_1, &page->flags)) {
308 flush_dcache_icache_page(page);
309 set_bit(PG_arch_1, &page->flags);
310 }
311 pte_update(ptep, 0, _PAGE_HWEXEC);
Benjamin Herrenschmidte701d262007-10-30 09:46:06 +1100312 _tlbie(address, mm->context.id);
Eugene Suroveginbab70a42006-03-28 10:13:12 -0800313 pte_unmap_unlock(ptep, ptl);
314 up_read(&mm->mmap_sem);
315 return 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000316 }
Eugene Suroveginbab70a42006-03-28 10:13:12 -0800317 pte_unmap_unlock(ptep, ptl);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000318 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000319#endif
320 /* a write */
321 } else if (is_write) {
322 if (!(vma->vm_flags & VM_WRITE))
323 goto bad_area;
324 /* a read */
325 } else {
326 /* protection fault */
327 if (error_code & 0x08000000)
328 goto bad_area;
Jason Barondf67b3d2006-09-29 01:58:58 -0700329 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000330 goto bad_area;
331 }
332
333 /*
334 * If for any reason at all we couldn't handle the fault,
335 * make sure we exit gracefully rather than endlessly redo
336 * the fault.
337 */
338 survive:
Nick Piggin83c54072007-07-19 01:47:05 -0700339 ret = handle_mm_fault(mm, vma, address, is_write);
340 if (unlikely(ret & VM_FAULT_ERROR)) {
341 if (ret & VM_FAULT_OOM)
342 goto out_of_memory;
343 else if (ret & VM_FAULT_SIGBUS)
344 goto do_sigbus;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000345 BUG();
346 }
Nick Piggin83c54072007-07-19 01:47:05 -0700347 if (ret & VM_FAULT_MAJOR)
348 current->maj_flt++;
349 else
350 current->min_flt++;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000351 up_read(&mm->mmap_sem);
352 return 0;
353
354bad_area:
355 up_read(&mm->mmap_sem);
356
357bad_area_nosemaphore:
358 /* User mode accesses cause a SIGSEGV */
359 if (user_mode(regs)) {
360 _exception(SIGSEGV, regs, code, address);
361 return 0;
362 }
363
364 if (is_exec && (error_code & DSISR_PROTFAULT)
365 && printk_ratelimit())
366 printk(KERN_CRIT "kernel tried to execute NX-protected"
367 " page (%lx) - exploit attempt? (uid: %d)\n",
368 address, current->uid);
369
370 return SIGSEGV;
371
372/*
373 * We ran out of memory, or some other thing happened to us that made
374 * us unable to handle the page fault gracefully.
375 */
376out_of_memory:
377 up_read(&mm->mmap_sem);
Serge E. Hallynb460cbc2007-10-18 23:39:52 -0700378 if (is_global_init(current)) {
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000379 yield();
380 down_read(&mm->mmap_sem);
381 goto survive;
382 }
383 printk("VM: killing process %s\n", current->comm);
384 if (user_mode(regs))
will schmidteffe24bd2007-06-13 01:19:01 +1000385 do_group_exit(SIGKILL);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000386 return SIGKILL;
387
388do_sigbus:
389 up_read(&mm->mmap_sem);
390 if (user_mode(regs)) {
391 info.si_signo = SIGBUS;
392 info.si_errno = 0;
393 info.si_code = BUS_ADRERR;
394 info.si_addr = (void __user *)address;
395 force_sig_info(SIGBUS, &info, current);
396 return 0;
397 }
398 return SIGBUS;
399}
400
401/*
402 * bad_page_fault is called when we have a bad access from the kernel.
403 * It is called from the DSI and ISI handlers in head.S and from some
404 * of the procedures in traps.c.
405 */
406void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
407{
408 const struct exception_table_entry *entry;
409
410 /* Are we prepared to handle this fault? */
411 if ((entry = search_exception_tables(regs->nip)) != NULL) {
412 regs->nip = entry->fixup;
413 return;
414 }
415
416 /* kernel has accessed a bad area */
Olof Johansson723925b2005-11-06 14:54:36 -0800417
Olof Johansson723925b2005-11-06 14:54:36 -0800418 switch (regs->trap) {
Michael Ellermana416dd82006-11-08 10:22:59 +1100419 case 0x300:
420 case 0x380:
421 printk(KERN_ALERT "Unable to handle kernel paging request for "
422 "data at address 0x%08lx\n", regs->dar);
423 break;
424 case 0x400:
425 case 0x480:
426 printk(KERN_ALERT "Unable to handle kernel paging request for "
427 "instruction fetch\n");
428 break;
429 default:
430 printk(KERN_ALERT "Unable to handle kernel paging request for "
431 "unknown fault\n");
432 break;
Olof Johansson723925b2005-11-06 14:54:36 -0800433 }
434 printk(KERN_ALERT "Faulting instruction address: 0x%08lx\n",
435 regs->nip);
436
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000437 die("Kernel access of bad area", regs, sig);
438}