| /* SPDX-License-Identifier: GPL-2.0 */ |
| #include <linux/init.h> |
| #include <linux/linkage.h> |
| |
| #include <asm/assembler.h> |
| #include <asm/asm-offsets.h> |
| #include <asm/errno.h> |
| #include <asm/thread_info.h> |
| #include <asm/uaccess-asm.h> |
| #include <asm/v7m.h> |
| |
| @ Bad Abort numbers |
| @ ----------------- |
| @ |
| #define BAD_PREFETCH 0 |
| #define BAD_DATA 1 |
| #define BAD_ADDREXCPTN 2 |
| #define BAD_IRQ 3 |
| #define BAD_UNDEFINSTR 4 |
| |
| @ |
| @ Most of the stack format comes from struct pt_regs, but with |
| @ the addition of 8 bytes for storing syscall args 5 and 6. |
| @ This _must_ remain a multiple of 8 for EABI. |
| @ |
| #define S_OFF 8 |
| |
| /* |
| * The SWI code relies on the fact that R0 is at the bottom of the stack |
| * (due to slow/fast restore user regs). |
| */ |
| #if S_R0 != 0 |
| #error "Please fix" |
| #endif |
| |
| .macro zero_fp |
| #ifdef CONFIG_FRAME_POINTER |
| mov fp, #0 |
| #endif |
| .endm |
| |
| #ifdef CONFIG_ALIGNMENT_TRAP |
| #define ATRAP(x...) x |
| #else |
| #define ATRAP(x...) |
| #endif |
| |
| .macro alignment_trap, rtmp1, rtmp2, label |
| #ifdef CONFIG_ALIGNMENT_TRAP |
| mrc p15, 0, \rtmp2, c1, c0, 0 |
| ldr_va \rtmp1, \label |
| teq \rtmp1, \rtmp2 |
| mcrne p15, 0, \rtmp1, c1, c0, 0 |
| #endif |
| .endm |
| |
| #ifdef CONFIG_CPU_V7M |
| /* |
| * ARMv7-M exception entry/exit macros. |
| * |
| * xPSR, ReturnAddress(), LR (R14), R12, R3, R2, R1, and R0 are |
| * automatically saved on the current stack (32 words) before |
| * switching to the exception stack (SP_main). |
| * |
| * If exception is taken while in user mode, SP_main is |
| * empty. Otherwise, SP_main is aligned to 64 bit automatically |
| * (CCR.STKALIGN set). |
| * |
| * Linux assumes that the interrupts are disabled when entering an |
| * exception handler and it may BUG if this is not the case. Interrupts |
| * are disabled during entry and reenabled in the exit macro. |
| * |
| * v7m_exception_slow_exit is used when returning from SVC or PendSV. |
| * When returning to kernel mode, we don't return from exception. |
| */ |
| .macro v7m_exception_entry |
| @ determine the location of the registers saved by the core during |
| @ exception entry. Depending on the mode the cpu was in when the |
| @ exception happend that is either on the main or the process stack. |
| @ Bit 2 of EXC_RETURN stored in the lr register specifies which stack |
| @ was used. |
| tst lr, #EXC_RET_STACK_MASK |
| mrsne r12, psp |
| moveq r12, sp |
| |
| @ we cannot rely on r0-r3 and r12 matching the value saved in the |
| @ exception frame because of tail-chaining. So these have to be |
| @ reloaded. |
| ldmia r12!, {r0-r3} |
| |
| @ Linux expects to have irqs off. Do it here before taking stack space |
| cpsid i |
| |
| sub sp, #PT_REGS_SIZE-S_IP |
| stmdb sp!, {r0-r11} |
| |
| @ load saved r12, lr, return address and xPSR. |
| @ r0-r7 are used for signals and never touched from now on. Clobbering |
| @ r8-r12 is OK. |
| mov r9, r12 |
| ldmia r9!, {r8, r10-r12} |
| |
| @ calculate the original stack pointer value. |
| @ r9 currently points to the memory location just above the auto saved |
| @ xPSR. |
| @ The cpu might automatically 8-byte align the stack. Bit 9 |
| @ of the saved xPSR specifies if stack aligning took place. In this case |
| @ another 32-bit value is included in the stack. |
| |
| tst r12, V7M_xPSR_FRAMEPTRALIGN |
| addne r9, r9, #4 |
| |
| @ store saved r12 using str to have a register to hold the base for stm |
| str r8, [sp, #S_IP] |
| add r8, sp, #S_SP |
| @ store r13-r15, xPSR |
| stmia r8!, {r9-r12} |
| @ store old_r0 |
| str r0, [r8] |
| .endm |
| |
| /* |
| * PENDSV and SVCALL are configured to have the same exception |
| * priorities. As a kernel thread runs at SVCALL execution priority it |
| * can never be preempted and so we will never have to return to a |
| * kernel thread here. |
| */ |
| .macro v7m_exception_slow_exit ret_r0 |
| cpsid i |
| ldr lr, =exc_ret |
| ldr lr, [lr] |
| |
| @ read original r12, sp, lr, pc and xPSR |
| add r12, sp, #S_IP |
| ldmia r12, {r1-r5} |
| |
| @ an exception frame is always 8-byte aligned. To tell the hardware if |
| @ the sp to be restored is aligned or not set bit 9 of the saved xPSR |
| @ accordingly. |
| tst r2, #4 |
| subne r2, r2, #4 |
| orrne r5, V7M_xPSR_FRAMEPTRALIGN |
| biceq r5, V7M_xPSR_FRAMEPTRALIGN |
| |
| @ ensure bit 0 is cleared in the PC, otherwise behaviour is |
| @ unpredictable |
| bic r4, #1 |
| |
| @ write basic exception frame |
| stmdb r2!, {r1, r3-r5} |
| ldmia sp, {r1, r3-r5} |
| .if \ret_r0 |
| stmdb r2!, {r0, r3-r5} |
| .else |
| stmdb r2!, {r1, r3-r5} |
| .endif |
| |
| @ restore process sp |
| msr psp, r2 |
| |
| @ restore original r4-r11 |
| ldmia sp!, {r0-r11} |
| |
| @ restore main sp |
| add sp, sp, #PT_REGS_SIZE-S_IP |
| |
| cpsie i |
| bx lr |
| .endm |
| #endif /* CONFIG_CPU_V7M */ |
| |
| @ |
| @ Store/load the USER SP and LR registers by switching to the SYS |
| @ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not |
| @ available. Should only be called from SVC mode |
| @ |
| .macro store_user_sp_lr, rd, rtemp, offset = 0 |
| mrs \rtemp, cpsr |
| eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) |
| msr cpsr_c, \rtemp @ switch to the SYS mode |
| |
| str sp, [\rd, #\offset] @ save sp_usr |
| str lr, [\rd, #\offset + 4] @ save lr_usr |
| |
| eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) |
| msr cpsr_c, \rtemp @ switch back to the SVC mode |
| .endm |
| |
| .macro load_user_sp_lr, rd, rtemp, offset = 0 |
| mrs \rtemp, cpsr |
| eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) |
| msr cpsr_c, \rtemp @ switch to the SYS mode |
| |
| ldr sp, [\rd, #\offset] @ load sp_usr |
| ldr lr, [\rd, #\offset + 4] @ load lr_usr |
| |
| eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) |
| msr cpsr_c, \rtemp @ switch back to the SVC mode |
| .endm |
| |
| |
| .macro svc_exit, rpsr, irq = 0 |
| .if \irq != 0 |
| @ IRQs already off |
| #ifdef CONFIG_TRACE_IRQFLAGS |
| @ The parent context IRQs must have been enabled to get here in |
| @ the first place, so there's no point checking the PSR I bit. |
| bl trace_hardirqs_on |
| #endif |
| .else |
| @ IRQs off again before pulling preserved data off the stack |
| disable_irq_notrace |
| #ifdef CONFIG_TRACE_IRQFLAGS |
| tst \rpsr, #PSR_I_BIT |
| bleq trace_hardirqs_on |
| tst \rpsr, #PSR_I_BIT |
| blne trace_hardirqs_off |
| #endif |
| .endif |
| uaccess_exit tsk, r0, r1 |
| |
| #ifndef CONFIG_THUMB2_KERNEL |
| @ ARM mode SVC restore |
| msr spsr_cxsf, \rpsr |
| #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) |
| @ We must avoid clrex due to Cortex-A15 erratum #830321 |
| sub r0, sp, #4 @ uninhabited address |
| strex r1, r2, [r0] @ clear the exclusive monitor |
| #endif |
| ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr |
| #else |
| @ Thumb mode SVC restore |
| ldr lr, [sp, #S_SP] @ top of the stack |
| ldrd r0, r1, [sp, #S_LR] @ calling lr and pc |
| |
| @ We must avoid clrex due to Cortex-A15 erratum #830321 |
| strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor |
| |
| stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context |
| ldmia sp, {r0 - r12} |
| mov sp, lr |
| ldr lr, [sp], #4 |
| rfeia sp! |
| #endif |
| .endm |
| |
| @ |
| @ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit |
| @ |
| @ This macro acts in a similar manner to svc_exit but switches to FIQ |
| @ mode to restore the final part of the register state. |
| @ |
| @ We cannot use the normal svc_exit procedure because that would |
| @ clobber spsr_svc (FIQ could be delivered during the first few |
| @ instructions of vector_swi meaning its contents have not been |
| @ saved anywhere). |
| @ |
| @ Note that, unlike svc_exit, this macro also does not allow a caller |
| @ supplied rpsr. This is because the FIQ exceptions are not re-entrant |
| @ and the handlers cannot call into the scheduler (meaning the value |
| @ on the stack remains correct). |
| @ |
| .macro svc_exit_via_fiq |
| uaccess_exit tsk, r0, r1 |
| #ifndef CONFIG_THUMB2_KERNEL |
| @ ARM mode restore |
| mov r0, sp |
| ldmib r0, {r1 - r14} @ abort is deadly from here onward (it will |
| @ clobber state restored below) |
| msr cpsr_c, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT |
| add r8, r0, #S_PC |
| ldr r9, [r0, #S_PSR] |
| msr spsr_cxsf, r9 |
| ldr r0, [r0, #S_R0] |
| ldmia r8, {pc}^ |
| #else |
| @ Thumb mode restore |
| add r0, sp, #S_R2 |
| ldr lr, [sp, #S_LR] |
| ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will |
| @ clobber state restored below) |
| ldmia r0, {r2 - r12} |
| mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT |
| msr cpsr_c, r1 |
| sub r0, #S_R2 |
| add r8, r0, #S_PC |
| ldmia r0, {r0 - r1} |
| rfeia r8 |
| #endif |
| .endm |
| |
| |
| .macro restore_user_regs, fast = 0, offset = 0 |
| #if defined(CONFIG_CPU_32v6K) && \ |
| (!defined(CONFIG_CPU_V6) || defined(CONFIG_SMP)) |
| #ifdef CONFIG_CPU_V6 |
| ALT_SMP(nop) |
| ALT_UP_B(.L1_\@) |
| #endif |
| @ The TLS register update is deferred until return to user space so we |
| @ can use it for other things while running in the kernel |
| mrc p15, 0, r1, c13, c0, 3 @ get current_thread_info pointer |
| ldr r1, [r1, #TI_TP_VALUE] |
| mcr p15, 0, r1, c13, c0, 3 @ set TLS register |
| .L1_\@: |
| #endif |
| |
| uaccess_enable r1, isb=0 |
| #ifndef CONFIG_THUMB2_KERNEL |
| @ ARM mode restore |
| mov r2, sp |
| ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr |
| ldr lr, [r2, #\offset + S_PC]! @ get pc |
| tst r1, #PSR_I_BIT | 0x0f |
| bne 1f |
| msr spsr_cxsf, r1 @ save in spsr_svc |
| #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) |
| @ We must avoid clrex due to Cortex-A15 erratum #830321 |
| strex r1, r2, [r2] @ clear the exclusive monitor |
| #endif |
| .if \fast |
| ldmdb r2, {r1 - lr}^ @ get calling r1 - lr |
| .else |
| ldmdb r2, {r0 - lr}^ @ get calling r0 - lr |
| .endif |
| mov r0, r0 @ ARMv5T and earlier require a nop |
| @ after ldm {}^ |
| add sp, sp, #\offset + PT_REGS_SIZE |
| movs pc, lr @ return & move spsr_svc into cpsr |
| 1: bug "Returning to usermode but unexpected PSR bits set?", \@ |
| #elif defined(CONFIG_CPU_V7M) |
| @ V7M restore. |
| @ Note that we don't need to do clrex here as clearing the local |
| @ monitor is part of the exception entry and exit sequence. |
| .if \offset |
| add sp, #\offset |
| .endif |
| v7m_exception_slow_exit ret_r0 = \fast |
| #else |
| @ Thumb mode restore |
| mov r2, sp |
| load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr |
| ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr |
| ldr lr, [sp, #\offset + S_PC] @ get pc |
| add sp, sp, #\offset + S_SP |
| tst r1, #PSR_I_BIT | 0x0f |
| bne 1f |
| msr spsr_cxsf, r1 @ save in spsr_svc |
| |
| @ We must avoid clrex due to Cortex-A15 erratum #830321 |
| strex r1, r2, [sp] @ clear the exclusive monitor |
| |
| .if \fast |
| ldmdb sp, {r1 - r12} @ get calling r1 - r12 |
| .else |
| ldmdb sp, {r0 - r12} @ get calling r0 - r12 |
| .endif |
| add sp, sp, #PT_REGS_SIZE - S_SP |
| movs pc, lr @ return & move spsr_svc into cpsr |
| 1: bug "Returning to usermode but unexpected PSR bits set?", \@ |
| #endif /* !CONFIG_THUMB2_KERNEL */ |
| .endm |
| |
| /* |
| * Context tracking subsystem. Used to instrument transitions |
| * between user and kernel mode. |
| */ |
| .macro ct_user_exit, save = 1 |
| #ifdef CONFIG_CONTEXT_TRACKING_USER |
| .if \save |
| stmdb sp!, {r0-r3, ip, lr} |
| bl user_exit_callable |
| ldmia sp!, {r0-r3, ip, lr} |
| .else |
| bl user_exit_callable |
| .endif |
| #endif |
| .endm |
| |
| .macro ct_user_enter, save = 1 |
| #ifdef CONFIG_CONTEXT_TRACKING_USER |
| .if \save |
| stmdb sp!, {r0-r3, ip, lr} |
| bl user_enter_callable |
| ldmia sp!, {r0-r3, ip, lr} |
| .else |
| bl user_enter_callable |
| .endif |
| #endif |
| .endm |
| |
| .macro invoke_syscall, table, nr, tmp, ret, reload=0 |
| #ifdef CONFIG_CPU_SPECTRE |
| mov \tmp, \nr |
| cmp \tmp, #NR_syscalls @ check upper syscall limit |
| movcs \tmp, #0 |
| csdb |
| badr lr, \ret @ return address |
| .if \reload |
| add r1, sp, #S_R0 + S_OFF @ pointer to regs |
| ldmiacc r1, {r0 - r6} @ reload r0-r6 |
| stmiacc sp, {r4, r5} @ update stack arguments |
| .endif |
| ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine |
| #else |
| cmp \nr, #NR_syscalls @ check upper syscall limit |
| badr lr, \ret @ return address |
| .if \reload |
| add r1, sp, #S_R0 + S_OFF @ pointer to regs |
| ldmiacc r1, {r0 - r6} @ reload r0-r6 |
| stmiacc sp, {r4, r5} @ update stack arguments |
| .endif |
| ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine |
| #endif |
| .endm |
| |
| /* |
| * These are the registers used in the syscall handler, and allow us to |
| * have in theory up to 7 arguments to a function - r0 to r6. |
| * |
| * r7 is reserved for the system call number for thumb mode. |
| * |
| * Note that tbl == why is intentional. |
| * |
| * We must set at least "tsk" and "why" when calling ret_with_reschedule. |
| */ |
| scno .req r7 @ syscall number |
| tbl .req r8 @ syscall table pointer |
| why .req r8 @ Linux syscall (!= 0) |
| tsk .req r9 @ current thread_info |
| |
| .macro do_overflow_check, frame_size:req |
| #ifdef CONFIG_VMAP_STACK |
| @ |
| @ Test whether the SP has overflowed. Task and IRQ stacks are aligned |
| @ so that SP & BIT(THREAD_SIZE_ORDER + PAGE_SHIFT) should always be |
| @ zero. |
| @ |
| ARM( tst sp, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT) ) |
| THUMB( tst r1, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT) ) |
| THUMB( it ne ) |
| bne .Lstack_overflow_check\@ |
| |
| .pushsection .text |
| .Lstack_overflow_check\@: |
| @ |
| @ The stack pointer is not pointing to a valid vmap'ed stack, but it |
| @ may be pointing into the linear map instead, which may happen if we |
| @ are already running from the overflow stack. We cannot detect overflow |
| @ in such cases so just carry on. |
| @ |
| str ip, [r0, #12] @ Stash IP on the mode stack |
| ldr_va ip, high_memory @ Start of VMALLOC space |
| ARM( cmp sp, ip ) @ SP in vmalloc space? |
| THUMB( cmp r1, ip ) |
| THUMB( itt lo ) |
| ldrlo ip, [r0, #12] @ Restore IP |
| blo .Lout\@ @ Carry on |
| |
| THUMB( sub r1, sp, r1 ) @ Restore original R1 |
| THUMB( sub sp, r1 ) @ Restore original SP |
| add sp, sp, #\frame_size @ Undo svc_entry's SP change |
| b __bad_stack @ Handle VMAP stack overflow |
| .popsection |
| .Lout\@: |
| #endif |
| .endm |
