| /* SPDX-License-Identifier: GPL-2.0-only */ |
| /* |
| * |
| * Derived from book3s_hv_rmhandlers.S, which is: |
| * |
| * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
| */ |
| |
| #include <linux/export.h> |
| #include <asm/reg.h> |
| #include <asm/ppc_asm.h> |
| #include <asm/asm-offsets.h> |
| #include <asm/tm.h> |
| #include <asm/cputable.h> |
| |
| #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
| #define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM) |
| |
| /* |
| * Save transactional state and TM-related registers. |
| * Called with: |
| * - r3 pointing to the vcpu struct |
| * - r4 containing the MSR with current TS bits: |
| * (For HV KVM, it is VCPU_MSR ; For PR KVM, it is host MSR). |
| * - r5 containing a flag indicating that non-volatile registers |
| * must be preserved. |
| * If r5 == 0, this can modify all checkpointed registers, but |
| * restores r1, r2 before exit. If r5 != 0, this restores the |
| * MSR TM/FP/VEC/VSX bits to their state on entry. |
| */ |
| _GLOBAL(__kvmppc_save_tm) |
| mflr r0 |
| std r0, PPC_LR_STKOFF(r1) |
| stdu r1, -SWITCH_FRAME_SIZE(r1) |
| |
| mr r9, r3 |
| cmpdi cr7, r5, 0 |
| |
| /* Turn on TM. */ |
| mfmsr r8 |
| mr r10, r8 |
| li r0, 1 |
| rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG |
| ori r8, r8, MSR_FP |
| oris r8, r8, (MSR_VEC | MSR_VSX)@h |
| mtmsrd r8 |
| |
| rldicl. r4, r4, 64 - MSR_TS_S_LG, 62 |
| beq 1f /* TM not active in guest. */ |
| |
| std r1, HSTATE_SCRATCH2(r13) |
| std r3, HSTATE_SCRATCH1(r13) |
| |
| /* Save CR on the stack - even if r5 == 0 we need to get cr7 back. */ |
| mfcr r6 |
| SAVE_GPR(6, r1) |
| |
| /* Save DSCR so we can restore it to avoid running with user value */ |
| mfspr r7, SPRN_DSCR |
| SAVE_GPR(7, r1) |
| |
| /* |
| * We are going to do treclaim., which will modify all checkpointed |
| * registers. Save the non-volatile registers on the stack if |
| * preservation of non-volatile state has been requested. |
| */ |
| beq cr7, 3f |
| SAVE_NVGPRS(r1) |
| |
| /* MSR[TS] will be 0 (non-transactional) once we do treclaim. */ |
| li r0, 0 |
| rldimi r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG |
| SAVE_GPR(10, r1) /* final MSR value */ |
| 3: |
| #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
| BEGIN_FTR_SECTION |
| /* Emulation of the treclaim instruction needs TEXASR before treclaim */ |
| mfspr r6, SPRN_TEXASR |
| std r6, VCPU_ORIG_TEXASR(r3) |
| END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST) |
| #endif |
| |
| /* Clear the MSR RI since r1, r13 are all going to be foobar. */ |
| li r5, 0 |
| mtmsrd r5, 1 |
| |
| li r3, TM_CAUSE_KVM_RESCHED |
| |
| /* All GPRs are volatile at this point. */ |
| TRECLAIM(R3) |
| |
| /* Temporarily store r13 and r9 so we have some regs to play with */ |
| SET_SCRATCH0(r13) |
| GET_PACA(r13) |
| std r9, PACATMSCRATCH(r13) |
| ld r9, HSTATE_SCRATCH1(r13) |
| |
| /* Save away PPR soon so we don't run with user value. */ |
| std r0, VCPU_GPRS_TM(0)(r9) |
| mfspr r0, SPRN_PPR |
| HMT_MEDIUM |
| |
| /* Reload stack pointer. */ |
| std r1, VCPU_GPRS_TM(1)(r9) |
| ld r1, HSTATE_SCRATCH2(r13) |
| |
| /* Set MSR RI now we have r1 and r13 back. */ |
| std r2, VCPU_GPRS_TM(2)(r9) |
| li r2, MSR_RI |
| mtmsrd r2, 1 |
| |
| /* Reload TOC pointer. */ |
| LOAD_PACA_TOC() |
| |
| /* Save all but r0-r2, r9 & r13 */ |
| reg = 3 |
| .rept 29 |
| .if (reg != 9) && (reg != 13) |
| std reg, VCPU_GPRS_TM(reg)(r9) |
| .endif |
| reg = reg + 1 |
| .endr |
| /* ... now save r13 */ |
| GET_SCRATCH0(r4) |
| std r4, VCPU_GPRS_TM(13)(r9) |
| /* ... and save r9 */ |
| ld r4, PACATMSCRATCH(r13) |
| std r4, VCPU_GPRS_TM(9)(r9) |
| |
| /* Restore host DSCR and CR values, after saving guest values */ |
| mfcr r6 |
| mfspr r7, SPRN_DSCR |
| stw r6, VCPU_CR_TM(r9) |
| std r7, VCPU_DSCR_TM(r9) |
| REST_GPR(6, r1) |
| REST_GPR(7, r1) |
| mtcr r6 |
| mtspr SPRN_DSCR, r7 |
| |
| /* Save away checkpointed SPRs. */ |
| std r0, VCPU_PPR_TM(r9) |
| mflr r5 |
| mfctr r7 |
| mfspr r8, SPRN_AMR |
| mfspr r10, SPRN_TAR |
| mfxer r11 |
| std r5, VCPU_LR_TM(r9) |
| std r7, VCPU_CTR_TM(r9) |
| std r8, VCPU_AMR_TM(r9) |
| std r10, VCPU_TAR_TM(r9) |
| std r11, VCPU_XER_TM(r9) |
| |
| /* Save FP/VSX. */ |
| addi r3, r9, VCPU_FPRS_TM |
| bl store_fp_state |
| addi r3, r9, VCPU_VRS_TM |
| bl store_vr_state |
| mfspr r6, SPRN_VRSAVE |
| stw r6, VCPU_VRSAVE_TM(r9) |
| |
| /* Restore non-volatile registers if requested to */ |
| beq cr7, 1f |
| REST_NVGPRS(r1) |
| REST_GPR(10, r1) |
| 1: |
| /* |
| * We need to save these SPRs after the treclaim so that the software |
| * error code is recorded correctly in the TEXASR. Also the user may |
| * change these outside of a transaction, so they must always be |
| * context switched. |
| */ |
| mfspr r7, SPRN_TEXASR |
| std r7, VCPU_TEXASR(r9) |
| mfspr r5, SPRN_TFHAR |
| mfspr r6, SPRN_TFIAR |
| std r5, VCPU_TFHAR(r9) |
| std r6, VCPU_TFIAR(r9) |
| |
| /* Restore MSR state if requested */ |
| beq cr7, 2f |
| mtmsrd r10, 0 |
| 2: |
| addi r1, r1, SWITCH_FRAME_SIZE |
| ld r0, PPC_LR_STKOFF(r1) |
| mtlr r0 |
| blr |
| |
| /* |
| * _kvmppc_save_tm_pr() is a wrapper around __kvmppc_save_tm(), so that it can |
| * be invoked from C function by PR KVM only. |
| */ |
| _GLOBAL(_kvmppc_save_tm_pr) |
| mflr r0 |
| std r0, PPC_LR_STKOFF(r1) |
| stdu r1, -PPC_MIN_STKFRM(r1) |
| |
| mfspr r8, SPRN_TAR |
| std r8, PPC_MIN_STKFRM-8(r1) |
| |
| li r5, 1 /* preserve non-volatile registers */ |
| bl __kvmppc_save_tm |
| |
| ld r8, PPC_MIN_STKFRM-8(r1) |
| mtspr SPRN_TAR, r8 |
| |
| addi r1, r1, PPC_MIN_STKFRM |
| ld r0, PPC_LR_STKOFF(r1) |
| mtlr r0 |
| blr |
| |
| EXPORT_SYMBOL_GPL(_kvmppc_save_tm_pr); |
| |
| /* |
| * Restore transactional state and TM-related registers. |
| * Called with: |
| * - r3 pointing to the vcpu struct. |
| * - r4 is the guest MSR with desired TS bits: |
| * For HV KVM, it is VCPU_MSR |
| * For PR KVM, it is provided by caller |
| * - r5 containing a flag indicating that non-volatile registers |
| * must be preserved. |
| * If r5 == 0, this potentially modifies all checkpointed registers, but |
| * restores r1, r2 from the PACA before exit. |
| * If r5 != 0, this restores the MSR TM/FP/VEC/VSX bits to their state on entry. |
| */ |
| _GLOBAL(__kvmppc_restore_tm) |
| mflr r0 |
| std r0, PPC_LR_STKOFF(r1) |
| |
| cmpdi cr7, r5, 0 |
| |
| /* Turn on TM/FP/VSX/VMX so we can restore them. */ |
| mfmsr r5 |
| mr r10, r5 |
| li r6, MSR_TM >> 32 |
| sldi r6, r6, 32 |
| or r5, r5, r6 |
| ori r5, r5, MSR_FP |
| oris r5, r5, (MSR_VEC | MSR_VSX)@h |
| mtmsrd r5 |
| |
| /* |
| * The user may change these outside of a transaction, so they must |
| * always be context switched. |
| */ |
| ld r5, VCPU_TFHAR(r3) |
| ld r6, VCPU_TFIAR(r3) |
| ld r7, VCPU_TEXASR(r3) |
| mtspr SPRN_TFHAR, r5 |
| mtspr SPRN_TFIAR, r6 |
| mtspr SPRN_TEXASR, r7 |
| |
| mr r5, r4 |
| rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 |
| beq 9f /* TM not active in guest */ |
| |
| /* Make sure the failure summary is set, otherwise we'll program check |
| * when we trechkpt. It's possible that this might have been not set |
| * on a kvmppc_set_one_reg() call but we shouldn't let this crash the |
| * host. |
| */ |
| oris r7, r7, (TEXASR_FS)@h |
| mtspr SPRN_TEXASR, r7 |
| |
| /* |
| * Make a stack frame and save non-volatile registers if requested. |
| */ |
| stdu r1, -SWITCH_FRAME_SIZE(r1) |
| std r1, HSTATE_SCRATCH2(r13) |
| |
| mfcr r6 |
| mfspr r7, SPRN_DSCR |
| SAVE_GPR(2, r1) |
| SAVE_GPR(6, r1) |
| SAVE_GPR(7, r1) |
| |
| beq cr7, 4f |
| SAVE_NVGPRS(r1) |
| |
| /* MSR[TS] will be 1 (suspended) once we do trechkpt */ |
| li r0, 1 |
| rldimi r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG |
| SAVE_GPR(10, r1) /* final MSR value */ |
| 4: |
| /* |
| * We need to load up the checkpointed state for the guest. |
| * We need to do this early as it will blow away any GPRs, VSRs and |
| * some SPRs. |
| */ |
| |
| mr r31, r3 |
| addi r3, r31, VCPU_FPRS_TM |
| bl load_fp_state |
| addi r3, r31, VCPU_VRS_TM |
| bl load_vr_state |
| mr r3, r31 |
| lwz r7, VCPU_VRSAVE_TM(r3) |
| mtspr SPRN_VRSAVE, r7 |
| |
| ld r5, VCPU_LR_TM(r3) |
| lwz r6, VCPU_CR_TM(r3) |
| ld r7, VCPU_CTR_TM(r3) |
| ld r8, VCPU_AMR_TM(r3) |
| ld r9, VCPU_TAR_TM(r3) |
| ld r10, VCPU_XER_TM(r3) |
| mtlr r5 |
| mtcr r6 |
| mtctr r7 |
| mtspr SPRN_AMR, r8 |
| mtspr SPRN_TAR, r9 |
| mtxer r10 |
| |
| /* |
| * Load up PPR and DSCR values but don't put them in the actual SPRs |
| * till the last moment to avoid running with userspace PPR and DSCR for |
| * too long. |
| */ |
| ld r29, VCPU_DSCR_TM(r3) |
| ld r30, VCPU_PPR_TM(r3) |
| |
| /* Clear the MSR RI since r1, r13 are all going to be foobar. */ |
| li r5, 0 |
| mtmsrd r5, 1 |
| |
| /* Load GPRs r0-r28 */ |
| reg = 0 |
| .rept 29 |
| ld reg, VCPU_GPRS_TM(reg)(r31) |
| reg = reg + 1 |
| .endr |
| |
| mtspr SPRN_DSCR, r29 |
| mtspr SPRN_PPR, r30 |
| |
| /* Load final GPRs */ |
| ld 29, VCPU_GPRS_TM(29)(r31) |
| ld 30, VCPU_GPRS_TM(30)(r31) |
| ld 31, VCPU_GPRS_TM(31)(r31) |
| |
| /* TM checkpointed state is now setup. All GPRs are now volatile. */ |
| TRECHKPT |
| |
| /* Now let's get back the state we need. */ |
| HMT_MEDIUM |
| GET_PACA(r13) |
| ld r1, HSTATE_SCRATCH2(r13) |
| REST_GPR(7, r1) |
| mtspr SPRN_DSCR, r7 |
| |
| /* Set the MSR RI since we have our registers back. */ |
| li r5, MSR_RI |
| mtmsrd r5, 1 |
| |
| /* Restore TOC pointer and CR */ |
| REST_GPR(2, r1) |
| REST_GPR(6, r1) |
| mtcr r6 |
| |
| /* Restore non-volatile registers if requested to. */ |
| beq cr7, 5f |
| REST_GPR(10, r1) |
| REST_NVGPRS(r1) |
| |
| 5: addi r1, r1, SWITCH_FRAME_SIZE |
| ld r0, PPC_LR_STKOFF(r1) |
| mtlr r0 |
| |
| 9: /* Restore MSR bits if requested */ |
| beqlr cr7 |
| mtmsrd r10, 0 |
| blr |
| |
| /* |
| * _kvmppc_restore_tm_pr() is a wrapper around __kvmppc_restore_tm(), so that it |
| * can be invoked from C function by PR KVM only. |
| */ |
| _GLOBAL(_kvmppc_restore_tm_pr) |
| mflr r0 |
| std r0, PPC_LR_STKOFF(r1) |
| stdu r1, -PPC_MIN_STKFRM(r1) |
| |
| /* save TAR so that it can be recovered later */ |
| mfspr r8, SPRN_TAR |
| std r8, PPC_MIN_STKFRM-8(r1) |
| |
| li r5, 1 |
| bl __kvmppc_restore_tm |
| |
| ld r8, PPC_MIN_STKFRM-8(r1) |
| mtspr SPRN_TAR, r8 |
| |
| addi r1, r1, PPC_MIN_STKFRM |
| ld r0, PPC_LR_STKOFF(r1) |
| mtlr r0 |
| blr |
| |
| EXPORT_SYMBOL_GPL(_kvmppc_restore_tm_pr); |
| #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ |
