| /* SPDX-License-Identifier: GPL-2.0-or-later */ | |
| /* | |
| * PowerPC version | |
| * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) | |
| * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP | |
| * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com> | |
| * Adapted for Power Macintosh by Paul Mackerras. | |
| * Low-level exception handlers and MMU support | |
| * rewritten by Paul Mackerras. | |
| * Copyright (C) 1996 Paul Mackerras. | |
| * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net). | |
| * | |
| * This file contains the system call entry code, context switch | |
| * code, and exception/interrupt return code for PowerPC. | |
| */ | |
| #include <linux/errno.h> | |
| #include <linux/err.h> | |
| #include <linux/sys.h> | |
| #include <linux/threads.h> | |
| #include <linux/linkage.h> | |
| #include <asm/reg.h> | |
| #include <asm/page.h> | |
| #include <asm/mmu.h> | |
| #include <asm/cputable.h> | |
| #include <asm/thread_info.h> | |
| #include <asm/ppc_asm.h> | |
| #include <asm/asm-offsets.h> | |
| #include <asm/unistd.h> | |
| #include <asm/ptrace.h> | |
| #include <asm/feature-fixups.h> | |
| #include <asm/barrier.h> | |
| #include <asm/kup.h> | |
| #include <asm/bug.h> | |
| #include <asm/interrupt.h> | |
| #include "head_32.h" | |
| /* | |
| * powerpc relies on return from interrupt/syscall being context synchronising | |
| * (which rfi is) to support ARCH_HAS_MEMBARRIER_SYNC_CORE without additional | |
| * synchronisation instructions. | |
| */ | |
| /* | |
| * Align to 4k in order to ensure that all functions modyfing srr0/srr1 | |
| * fit into one page in order to not encounter a TLB miss between the | |
| * modification of srr0/srr1 and the associated rfi. | |
| */ | |
| .align 12 | |
| #if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_E500) | |
| .globl prepare_transfer_to_handler | |
| prepare_transfer_to_handler: | |
| /* if from kernel, check interrupted DOZE/NAP mode */ | |
| lwz r12,TI_LOCAL_FLAGS(r2) | |
| mtcrf 0x01,r12 | |
| bt- 31-TLF_NAPPING,4f | |
| bt- 31-TLF_SLEEPING,7f | |
| blr | |
| 4: rlwinm r12,r12,0,~_TLF_NAPPING | |
| stw r12,TI_LOCAL_FLAGS(r2) | |
| b power_save_ppc32_restore | |
| 7: rlwinm r12,r12,0,~_TLF_SLEEPING | |
| stw r12,TI_LOCAL_FLAGS(r2) | |
| lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */ | |
| rlwinm r9,r9,0,~MSR_EE | |
| lwz r12,_LINK(r11) /* and return to address in LR */ | |
| REST_GPR(2, r11) | |
| b fast_exception_return | |
| _ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler) | |
| #endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_PPC_E500 */ | |
| #if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32) | |
| SYM_FUNC_START(__kuep_lock) | |
| lwz r9, THREAD+THSR0(r2) | |
| update_user_segments_by_4 r9, r10, r11, r12 | |
| blr | |
| SYM_FUNC_END(__kuep_lock) | |
| SYM_FUNC_START_LOCAL(__kuep_unlock) | |
| lwz r9, THREAD+THSR0(r2) | |
| rlwinm r9,r9,0,~SR_NX | |
| update_user_segments_by_4 r9, r10, r11, r12 | |
| blr | |
| SYM_FUNC_END(__kuep_unlock) | |
| .macro kuep_lock | |
| bl __kuep_lock | |
| .endm | |
| .macro kuep_unlock | |
| bl __kuep_unlock | |
| .endm | |
| #else | |
| .macro kuep_lock | |
| .endm | |
| .macro kuep_unlock | |
| .endm | |
| #endif | |
| .globl transfer_to_syscall | |
| transfer_to_syscall: | |
| stw r3, ORIG_GPR3(r1) | |
| stw r11, GPR1(r1) | |
| stw r11, 0(r1) | |
| mflr r12 | |
| stw r12, _LINK(r1) | |
| #ifdef CONFIG_BOOKE | |
| rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ | |
| #endif | |
| lis r12,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ | |
| SAVE_GPR(2, r1) | |
| addi r12,r12,STACK_FRAME_REGS_MARKER@l | |
| stw r9,_MSR(r1) | |
| li r2, INTERRUPT_SYSCALL | |
| stw r12,STACK_INT_FRAME_MARKER(r1) | |
| stw r2,_TRAP(r1) | |
| SAVE_GPR(0, r1) | |
| SAVE_GPRS(3, 8, r1) | |
| addi r2,r10,-THREAD | |
| SAVE_NVGPRS(r1) | |
| kuep_lock | |
| /* Calling convention has r3 = regs, r4 = orig r0 */ | |
| addi r3,r1,STACK_INT_FRAME_REGS | |
| mr r4,r0 | |
| bl system_call_exception | |
| ret_from_syscall: | |
| addi r4,r1,STACK_INT_FRAME_REGS | |
| li r5,0 | |
| bl syscall_exit_prepare | |
| #ifdef CONFIG_PPC_47x | |
| lis r4,icache_44x_need_flush@ha | |
| lwz r5,icache_44x_need_flush@l(r4) | |
| cmplwi cr0,r5,0 | |
| bne- .L44x_icache_flush | |
| #endif /* CONFIG_PPC_47x */ | |
| .L44x_icache_flush_return: | |
| kuep_unlock | |
| lwz r4,_LINK(r1) | |
| lwz r5,_CCR(r1) | |
| mtlr r4 | |
| lwz r7,_NIP(r1) | |
| lwz r8,_MSR(r1) | |
| cmpwi r3,0 | |
| REST_GPR(3, r1) | |
| syscall_exit_finish: | |
| mtspr SPRN_SRR0,r7 | |
| mtspr SPRN_SRR1,r8 | |
| bne 3f | |
| mtcr r5 | |
| 1: REST_GPR(2, r1) | |
| REST_GPR(1, r1) | |
| rfi | |
| 3: mtcr r5 | |
| lwz r4,_CTR(r1) | |
| lwz r5,_XER(r1) | |
| REST_NVGPRS(r1) | |
| mtctr r4 | |
| mtxer r5 | |
| REST_GPR(0, r1) | |
| REST_GPRS(3, 12, r1) | |
| b 1b | |
| _ASM_NOKPROBE_SYMBOL(syscall_exit_finish) | |
| #ifdef CONFIG_44x | |
| .L44x_icache_flush: | |
| li r7,0 | |
| iccci r0,r0 | |
| stw r7,icache_44x_need_flush@l(r4) | |
| b .L44x_icache_flush_return | |
| #endif /* CONFIG_44x */ | |
| .globl ret_from_fork | |
| ret_from_fork: | |
| REST_NVGPRS(r1) | |
| bl schedule_tail | |
| li r3,0 /* fork() return value */ | |
| b ret_from_syscall | |
| .globl ret_from_kernel_user_thread | |
| ret_from_kernel_user_thread: | |
| bl schedule_tail | |
| mtctr r14 | |
| mr r3,r15 | |
| PPC440EP_ERR42 | |
| bctrl | |
| li r3,0 | |
| b ret_from_syscall | |
| .globl start_kernel_thread | |
| start_kernel_thread: | |
| bl schedule_tail | |
| mtctr r14 | |
| mr r3,r15 | |
| PPC440EP_ERR42 | |
| bctrl | |
| /* | |
| * This must not return. We actually want to BUG here, not WARN, | |
| * because BUG will exit the process which is what the kernel thread | |
| * should have done, which may give some hope of continuing. | |
| */ | |
| 100: trap | |
| EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 | |
| .globl fast_exception_return | |
| fast_exception_return: | |
| #ifndef CONFIG_BOOKE | |
| andi. r10,r9,MSR_RI /* check for recoverable interrupt */ | |
| beq 3f /* if not, we've got problems */ | |
| #endif | |
| lwz r10,_CCR(r11) | |
| REST_GPRS(1, 6, r11) | |
| mtcr r10 | |
| lwz r10,_LINK(r11) | |
| mtlr r10 | |
| /* Clear the exception marker on the stack to avoid confusing stacktrace */ | |
| li r10, 0 | |
| stw r10, 8(r11) | |
| mtspr SPRN_SRR1,r9 | |
| mtspr SPRN_SRR0,r12 | |
| REST_GPR(9, r11) | |
| REST_GPR(10, r11) | |
| REST_GPR(12, r11) | |
| REST_GPR(11, r11) | |
| rfi | |
| _ASM_NOKPROBE_SYMBOL(fast_exception_return) | |
| /* aargh, a nonrecoverable interrupt, panic */ | |
| /* aargh, we don't know which trap this is */ | |
| 3: | |
| li r10,-1 | |
| stw r10,_TRAP(r11) | |
| prepare_transfer_to_handler | |
| bl unrecoverable_exception | |
| trap /* should not get here */ | |
| .globl interrupt_return | |
| interrupt_return: | |
| lwz r4,_MSR(r1) | |
| addi r3,r1,STACK_INT_FRAME_REGS | |
| andi. r0,r4,MSR_PR | |
| beq .Lkernel_interrupt_return | |
| bl interrupt_exit_user_prepare | |
| cmpwi r3,0 | |
| kuep_unlock | |
| bne- .Lrestore_nvgprs | |
| .Lfast_user_interrupt_return: | |
| lwz r11,_NIP(r1) | |
| lwz r12,_MSR(r1) | |
| mtspr SPRN_SRR0,r11 | |
| mtspr SPRN_SRR1,r12 | |
| BEGIN_FTR_SECTION | |
| lwarx r0,0,r1 | |
| END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) | |
| stwcx. r0,0,r1 /* to clear the reservation */ | |
| lwz r3,_CCR(r1) | |
| lwz r4,_LINK(r1) | |
| lwz r5,_CTR(r1) | |
| lwz r6,_XER(r1) | |
| li r0,0 | |
| /* | |
| * Leaving a stale exception marker on the stack can confuse | |
| * the reliable stack unwinder later on. Clear it. | |
| */ | |
| stw r0,8(r1) | |
| REST_GPRS(7, 12, r1) | |
| mtcr r3 | |
| mtlr r4 | |
| mtctr r5 | |
| mtspr SPRN_XER,r6 | |
| REST_GPRS(2, 6, r1) | |
| REST_GPR(0, r1) | |
| REST_GPR(1, r1) | |
| rfi | |
| .Lrestore_nvgprs: | |
| REST_NVGPRS(r1) | |
| b .Lfast_user_interrupt_return | |
| .Lkernel_interrupt_return: | |
| bl interrupt_exit_kernel_prepare | |
| .Lfast_kernel_interrupt_return: | |
| cmpwi cr1,r3,0 | |
| lwz r11,_NIP(r1) | |
| lwz r12,_MSR(r1) | |
| mtspr SPRN_SRR0,r11 | |
| mtspr SPRN_SRR1,r12 | |
| BEGIN_FTR_SECTION | |
| lwarx r0,0,r1 | |
| END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) | |
| stwcx. r0,0,r1 /* to clear the reservation */ | |
| lwz r3,_LINK(r1) | |
| lwz r4,_CTR(r1) | |
| lwz r5,_XER(r1) | |
| lwz r6,_CCR(r1) | |
| li r0,0 | |
| REST_GPRS(7, 12, r1) | |
| mtlr r3 | |
| mtctr r4 | |
| mtspr SPRN_XER,r5 | |
| /* | |
| * Leaving a stale exception marker on the stack can confuse | |
| * the reliable stack unwinder later on. Clear it. | |
| */ | |
| stw r0,8(r1) | |
| REST_GPRS(2, 5, r1) | |
| bne- cr1,1f /* emulate stack store */ | |
| mtcr r6 | |
| REST_GPR(6, r1) | |
| REST_GPR(0, r1) | |
| REST_GPR(1, r1) | |
| rfi | |
| 1: /* | |
| * Emulate stack store with update. New r1 value was already calculated | |
| * and updated in our interrupt regs by emulate_loadstore, but we can't | |
| * store the previous value of r1 to the stack before re-loading our | |
| * registers from it, otherwise they could be clobbered. Use | |
| * SPRG Scratch0 as temporary storage to hold the store | |
| * data, as interrupts are disabled here so it won't be clobbered. | |
| */ | |
| mtcr r6 | |
| #ifdef CONFIG_BOOKE | |
| mtspr SPRN_SPRG_WSCRATCH0, r9 | |
| #else | |
| mtspr SPRN_SPRG_SCRATCH0, r9 | |
| #endif | |
| addi r9,r1,INT_FRAME_SIZE /* get original r1 */ | |
| REST_GPR(6, r1) | |
| REST_GPR(0, r1) | |
| REST_GPR(1, r1) | |
| stw r9,0(r1) /* perform store component of stwu */ | |
| #ifdef CONFIG_BOOKE | |
| mfspr r9, SPRN_SPRG_RSCRATCH0 | |
| #else | |
| mfspr r9, SPRN_SPRG_SCRATCH0 | |
| #endif | |
| rfi | |
| _ASM_NOKPROBE_SYMBOL(interrupt_return) | |
| #ifdef CONFIG_BOOKE | |
| /* | |
| * Returning from a critical interrupt in user mode doesn't need | |
| * to be any different from a normal exception. For a critical | |
| * interrupt in the kernel, we just return (without checking for | |
| * preemption) since the interrupt may have happened at some crucial | |
| * place (e.g. inside the TLB miss handler), and because we will be | |
| * running with r1 pointing into critical_stack, not the current | |
| * process's kernel stack (and therefore current_thread_info() will | |
| * give the wrong answer). | |
| * We have to restore various SPRs that may have been in use at the | |
| * time of the critical interrupt. | |
| * | |
| */ | |
| #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \ | |
| REST_NVGPRS(r1); \ | |
| lwz r3,_MSR(r1); \ | |
| andi. r3,r3,MSR_PR; \ | |
| bne interrupt_return; \ | |
| REST_GPR(0, r1); \ | |
| REST_GPRS(2, 8, r1); \ | |
| lwz r10,_XER(r1); \ | |
| lwz r11,_CTR(r1); \ | |
| mtspr SPRN_XER,r10; \ | |
| mtctr r11; \ | |
| stwcx. r0,0,r1; /* to clear the reservation */ \ | |
| lwz r11,_LINK(r1); \ | |
| mtlr r11; \ | |
| lwz r10,_CCR(r1); \ | |
| mtcrf 0xff,r10; \ | |
| lwz r9,_DEAR(r1); \ | |
| lwz r10,_ESR(r1); \ | |
| mtspr SPRN_DEAR,r9; \ | |
| mtspr SPRN_ESR,r10; \ | |
| lwz r11,_NIP(r1); \ | |
| lwz r12,_MSR(r1); \ | |
| mtspr exc_lvl_srr0,r11; \ | |
| mtspr exc_lvl_srr1,r12; \ | |
| REST_GPRS(9, 12, r1); \ | |
| REST_GPR(1, r1); \ | |
| exc_lvl_rfi; \ | |
| b .; /* prevent prefetch past exc_lvl_rfi */ | |
| #define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \ | |
| lwz r9,_##exc_lvl_srr0(r1); \ | |
| lwz r10,_##exc_lvl_srr1(r1); \ | |
| mtspr SPRN_##exc_lvl_srr0,r9; \ | |
| mtspr SPRN_##exc_lvl_srr1,r10; | |
| #if defined(CONFIG_PPC_E500) | |
| #ifdef CONFIG_PHYS_64BIT | |
| #define RESTORE_MAS7 \ | |
| lwz r11,MAS7(r1); \ | |
| mtspr SPRN_MAS7,r11; | |
| #else | |
| #define RESTORE_MAS7 | |
| #endif /* CONFIG_PHYS_64BIT */ | |
| #define RESTORE_MMU_REGS \ | |
| lwz r9,MAS0(r1); \ | |
| lwz r10,MAS1(r1); \ | |
| lwz r11,MAS2(r1); \ | |
| mtspr SPRN_MAS0,r9; \ | |
| lwz r9,MAS3(r1); \ | |
| mtspr SPRN_MAS1,r10; \ | |
| lwz r10,MAS6(r1); \ | |
| mtspr SPRN_MAS2,r11; \ | |
| mtspr SPRN_MAS3,r9; \ | |
| mtspr SPRN_MAS6,r10; \ | |
| RESTORE_MAS7; | |
| #elif defined(CONFIG_44x) | |
| #define RESTORE_MMU_REGS \ | |
| lwz r9,MMUCR(r1); \ | |
| mtspr SPRN_MMUCR,r9; | |
| #else | |
| #define RESTORE_MMU_REGS | |
| #endif | |
| .globl ret_from_crit_exc | |
| ret_from_crit_exc: | |
| RESTORE_xSRR(SRR0,SRR1); | |
| RESTORE_MMU_REGS; | |
| RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI) | |
| _ASM_NOKPROBE_SYMBOL(ret_from_crit_exc) | |
| .globl ret_from_debug_exc | |
| ret_from_debug_exc: | |
| RESTORE_xSRR(SRR0,SRR1); | |
| RESTORE_xSRR(CSRR0,CSRR1); | |
| RESTORE_MMU_REGS; | |
| RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI) | |
| _ASM_NOKPROBE_SYMBOL(ret_from_debug_exc) | |
| .globl ret_from_mcheck_exc | |
| ret_from_mcheck_exc: | |
| RESTORE_xSRR(SRR0,SRR1); | |
| RESTORE_xSRR(CSRR0,CSRR1); | |
| RESTORE_xSRR(DSRR0,DSRR1); | |
| RESTORE_MMU_REGS; | |
| RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI) | |
| _ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc) | |
| #endif /* CONFIG_BOOKE */ |