Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 2 | /* |
| 3 | * Transactional memory support routines to reclaim and recheckpoint |
| 4 | * transactional process state. |
| 5 | * |
| 6 | * Copyright 2012 Matt Evans & Michael Neuling, IBM Corporation. |
| 7 | */ |
| 8 | |
Masahiro Yamada | 3932618 | 2023-08-07 00:09:53 +0900 | [diff] [blame] | 9 | #include <linux/export.h> |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 10 | #include <asm/asm-offsets.h> |
| 11 | #include <asm/ppc_asm.h> |
| 12 | #include <asm/ppc-opcode.h> |
| 13 | #include <asm/ptrace.h> |
| 14 | #include <asm/reg.h> |
Michael Neuling | 7f06f21 | 2014-03-28 16:40:34 +1100 | [diff] [blame] | 15 | #include <asm/bug.h> |
Christophe Leroy | 2c86cd1 | 2018-07-05 16:25:01 +0000 | [diff] [blame] | 16 | #include <asm/feature-fixups.h> |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 17 | |
| 18 | #ifdef CONFIG_VSX |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 19 | /* See fpu.S, this is borrowed from there */ |
| 20 | #define __SAVE_32FPRS_VSRS(n,c,base) \ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 21 | BEGIN_FTR_SECTION \ |
| 22 | b 2f; \ |
| 23 | END_FTR_SECTION_IFSET(CPU_FTR_VSX); \ |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 24 | SAVE_32FPRS(n,base); \ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 25 | b 3f; \ |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 26 | 2: SAVE_32VSRS(n,c,base); \ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 27 | 3: |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 28 | #define __REST_32FPRS_VSRS(n,c,base) \ |
| 29 | BEGIN_FTR_SECTION \ |
| 30 | b 2f; \ |
| 31 | END_FTR_SECTION_IFSET(CPU_FTR_VSX); \ |
| 32 | REST_32FPRS(n,base); \ |
| 33 | b 3f; \ |
| 34 | 2: REST_32VSRS(n,c,base); \ |
| 35 | 3: |
| 36 | #else |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 37 | #define __SAVE_32FPRS_VSRS(n,c,base) SAVE_32FPRS(n, base) |
| 38 | #define __REST_32FPRS_VSRS(n,c,base) REST_32FPRS(n, base) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 39 | #endif |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 40 | #define SAVE_32FPRS_VSRS(n,c,base) \ |
| 41 | __SAVE_32FPRS_VSRS(n,__REG_##c,__REG_##base) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 42 | #define REST_32FPRS_VSRS(n,c,base) \ |
| 43 | __REST_32FPRS_VSRS(n,__REG_##c,__REG_##base) |
| 44 | |
| 45 | /* Stack frame offsets for local variables. */ |
| 46 | #define TM_FRAME_L0 TM_FRAME_SIZE-16 |
| 47 | #define TM_FRAME_L1 TM_FRAME_SIZE-8 |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 48 | |
| 49 | |
| 50 | /* In order to access the TM SPRs, TM must be enabled. So, do so: */ |
| 51 | _GLOBAL(tm_enable) |
| 52 | mfmsr r4 |
| 53 | li r3, MSR_TM >> 32 |
| 54 | sldi r3, r3, 32 |
| 55 | and. r0, r4, r3 |
| 56 | bne 1f |
| 57 | or r4, r4, r3 |
| 58 | mtmsrd r4 |
| 59 | 1: blr |
Simon Guo | eacbb21 | 2018-05-23 15:01:46 +0800 | [diff] [blame] | 60 | EXPORT_SYMBOL_GPL(tm_enable); |
| 61 | |
| 62 | _GLOBAL(tm_disable) |
| 63 | mfmsr r4 |
| 64 | li r3, MSR_TM >> 32 |
| 65 | sldi r3, r3, 32 |
| 66 | andc r4, r4, r3 |
| 67 | mtmsrd r4 |
| 68 | blr |
| 69 | EXPORT_SYMBOL_GPL(tm_disable); |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 70 | |
| 71 | _GLOBAL(tm_save_sprs) |
| 72 | mfspr r0, SPRN_TFHAR |
| 73 | std r0, THREAD_TM_TFHAR(r3) |
| 74 | mfspr r0, SPRN_TEXASR |
| 75 | std r0, THREAD_TM_TEXASR(r3) |
| 76 | mfspr r0, SPRN_TFIAR |
| 77 | std r0, THREAD_TM_TFIAR(r3) |
| 78 | blr |
| 79 | |
| 80 | _GLOBAL(tm_restore_sprs) |
| 81 | ld r0, THREAD_TM_TFHAR(r3) |
| 82 | mtspr SPRN_TFHAR, r0 |
| 83 | ld r0, THREAD_TM_TEXASR(r3) |
| 84 | mtspr SPRN_TEXASR, r0 |
| 85 | ld r0, THREAD_TM_TFIAR(r3) |
| 86 | mtspr SPRN_TFIAR, r0 |
| 87 | blr |
| 88 | |
| 89 | /* Passed an 8-bit failure cause as first argument. */ |
| 90 | _GLOBAL(tm_abort) |
| 91 | TABORT(R3) |
| 92 | blr |
Simon Guo | eacbb21 | 2018-05-23 15:01:46 +0800 | [diff] [blame] | 93 | EXPORT_SYMBOL_GPL(tm_abort); |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 94 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 95 | /* |
| 96 | * void tm_reclaim(struct thread_struct *thread, |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 97 | * uint8_t cause) |
| 98 | * |
| 99 | * - Performs a full reclaim. This destroys outstanding |
Breno Leitao | 9669556 | 2018-06-18 19:59:42 -0300 | [diff] [blame] | 100 | * transactions and updates thread.ckpt_regs, thread.ckfp_state and |
| 101 | * thread.ckvr_state with the original checkpointed state. Note that |
| 102 | * thread->regs is unchanged. |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 103 | * |
| 104 | * Purpose is to both abort transactions of, and preserve the state of, |
| 105 | * a transactions at a context switch. We preserve/restore both sets of process |
| 106 | * state to restore them when the thread's scheduled again. We continue in |
| 107 | * userland as though nothing happened, but when the transaction is resumed |
| 108 | * they will abort back to the checkpointed state we save out here. |
| 109 | * |
| 110 | * Call with IRQs off, stacks get all out of sync for some periods in here! |
| 111 | */ |
| 112 | _GLOBAL(tm_reclaim) |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 113 | mfcr r5 |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 114 | mflr r0 |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 115 | stw r5, 8(r1) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 116 | std r0, 16(r1) |
Anton Blanchard | 6403105 | 2014-03-10 10:52:17 +1100 | [diff] [blame] | 117 | std r2, STK_GOT(r1) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 118 | stdu r1, -TM_FRAME_SIZE(r1) |
| 119 | |
Nicholas Piggin | c03be0a | 2022-11-27 22:49:32 +1000 | [diff] [blame] | 120 | /* We've a struct pt_regs at [r1+STACK_INT_FRAME_REGS]. */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 121 | |
Anton Blanchard | c2e31bd | 2014-03-10 10:48:44 +1100 | [diff] [blame] | 122 | std r3, STK_PARAM(R3)(r1) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 123 | SAVE_NVGPRS(r1) |
| 124 | |
Gustavo Romero | d0ffdee | 2020-09-19 12:00:25 -0300 | [diff] [blame] | 125 | /* |
| 126 | * Save kernel live AMR since it will be clobbered by treclaim |
| 127 | * but can be used elsewhere later in kernel space. |
| 128 | */ |
| 129 | mfspr r3, SPRN_AMR |
| 130 | std r3, TM_FRAME_L1(r1) |
| 131 | |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 132 | /* We need to setup MSR for VSX register save instructions. */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 133 | mfmsr r14 |
| 134 | mr r15, r14 |
| 135 | ori r15, r15, MSR_FP |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 136 | li r16, 0 |
Michael Neuling | c69e63b | 2013-10-02 17:15:15 +1000 | [diff] [blame] | 137 | ori r16, r16, MSR_EE /* IRQs hard off */ |
Michael Neuling | 090b928 | 2013-06-28 18:17:09 +1000 | [diff] [blame] | 138 | andc r15, r15, r16 |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 139 | oris r15, r15, MSR_VEC@h |
| 140 | #ifdef CONFIG_VSX |
| 141 | BEGIN_FTR_SECTION |
| 142 | oris r15,r15, MSR_VSX@h |
| 143 | END_FTR_SECTION_IFSET(CPU_FTR_VSX) |
| 144 | #endif |
| 145 | mtmsrd r15 |
| 146 | std r14, TM_FRAME_L0(r1) |
| 147 | |
Michael Neuling | 7f06f21 | 2014-03-28 16:40:34 +1100 | [diff] [blame] | 148 | /* Do sanity check on MSR to make sure we are suspended */ |
| 149 | li r7, (MSR_TS_S)@higher |
| 150 | srdi r6, r14, 32 |
| 151 | and r6, r6, r7 |
| 152 | 1: tdeqi r6, 0 |
| 153 | EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0 |
| 154 | |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 155 | /* Stash the stack pointer away for use after reclaim */ |
| 156 | std r1, PACAR1(r13) |
| 157 | |
Nicholas Piggin | f30a5e6 | 2019-06-28 15:33:32 +1000 | [diff] [blame] | 158 | /* Clear MSR RI since we are about to use SCRATCH0, EE is already off */ |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 159 | li r5, 0 |
| 160 | mtmsrd r5, 1 |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 161 | |
| 162 | /* |
| 163 | * BE CAREFUL HERE: |
| 164 | * At this point we can't take an SLB miss since we have MSR_RI |
| 165 | * off. Load only to/from the stack/paca which are in SLB bolted regions |
| 166 | * until we turn MSR RI back on. |
| 167 | * |
| 168 | * The moment we treclaim, ALL of our GPRs will switch |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 169 | * to user register state. (FPRs, CCR etc. also!) |
| 170 | * Use an sprg and a tm_scratch in the PACA to shuffle. |
| 171 | */ |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 172 | TRECLAIM(R4) /* Cause in r4 */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 173 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 174 | /* |
| 175 | * ******************** GPRs ******************** |
| 176 | * Stash the checkpointed r13 in the scratch SPR and get the real paca. |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 177 | */ |
| 178 | SET_SCRATCH0(r13) |
| 179 | GET_PACA(r13) |
| 180 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 181 | /* |
| 182 | * Stash the checkpointed r1 away in paca->tm_scratch and get the real |
| 183 | * stack pointer back into r1. |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 184 | */ |
| 185 | std r1, PACATMSCRATCH(r13) |
| 186 | ld r1, PACAR1(r13) |
| 187 | |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 188 | std r11, GPR11(r1) /* Temporary stash */ |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 189 | |
Michael Neuling | cf13435 | 2018-09-24 17:27:04 +1000 | [diff] [blame] | 190 | /* |
Michael Neuling | 96dc89d | 2018-09-25 19:36:47 +1000 | [diff] [blame] | 191 | * Move the saved user r1 to the kernel stack in case PACATMSCRATCH is |
| 192 | * clobbered by an exception once we turn on MSR_RI below. |
| 193 | */ |
| 194 | ld r11, PACATMSCRATCH(r13) |
| 195 | std r11, GPR1(r1) |
| 196 | |
| 197 | /* |
Michael Neuling | cf13435 | 2018-09-24 17:27:04 +1000 | [diff] [blame] | 198 | * Store r13 away so we can free up the scratch SPR for the SLB fault |
| 199 | * handler (needed once we start accessing the thread_struct). |
| 200 | */ |
| 201 | GET_SCRATCH0(r11) |
| 202 | std r11, GPR13(r1) |
| 203 | |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 204 | /* Reset MSR RI so we can take SLB faults again */ |
| 205 | li r11, MSR_RI |
| 206 | mtmsrd r11, 1 |
| 207 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 208 | /* Store the PPR in r11 and reset to decent value */ |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 209 | mfspr r11, SPRN_PPR |
| 210 | HMT_MEDIUM |
| 211 | |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 212 | /* Now get some more GPRS free */ |
| 213 | std r7, GPR7(r1) /* Temporary stash */ |
| 214 | std r12, GPR12(r1) /* '' '' '' */ |
Anton Blanchard | c2e31bd | 2014-03-10 10:48:44 +1100 | [diff] [blame] | 215 | ld r12, STK_PARAM(R3)(r1) /* Param 0, thread_struct * */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 216 | |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 217 | std r11, THREAD_TM_PPR(r12) /* Store PPR and free r11 */ |
| 218 | |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 219 | addi r7, r12, PT_CKPT_REGS /* Thread's ckpt_regs */ |
| 220 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 221 | /* |
| 222 | * Make r7 look like an exception frame so that we can use the neat |
| 223 | * GPRx(n) macros. r7 is NOT a pt_regs ptr! |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 224 | */ |
Nicholas Piggin | c03be0a | 2022-11-27 22:49:32 +1000 | [diff] [blame] | 225 | subi r7, r7, STACK_INT_FRAME_REGS |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 226 | |
| 227 | /* Sync the userland GPRs 2-12, 14-31 to thread->regs: */ |
| 228 | SAVE_GPR(0, r7) /* user r0 */ |
Nicholas Piggin | aebd1fb | 2021-10-22 16:13:22 +1000 | [diff] [blame] | 229 | SAVE_GPRS(2, 6, r7) /* user r2-r6 */ |
| 230 | SAVE_GPRS(8, 10, r7) /* user r8-r10 */ |
Michael Neuling | 96dc89d | 2018-09-25 19:36:47 +1000 | [diff] [blame] | 231 | ld r3, GPR1(r1) /* user r1 */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 232 | ld r4, GPR7(r1) /* user r7 */ |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 233 | ld r5, GPR11(r1) /* user r11 */ |
| 234 | ld r6, GPR12(r1) /* user r12 */ |
Michael Neuling | cf13435 | 2018-09-24 17:27:04 +1000 | [diff] [blame] | 235 | ld r8, GPR13(r1) /* user r13 */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 236 | std r3, GPR1(r7) |
| 237 | std r4, GPR7(r7) |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 238 | std r5, GPR11(r7) |
| 239 | std r6, GPR12(r7) |
| 240 | std r8, GPR13(r7) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 241 | |
| 242 | SAVE_NVGPRS(r7) /* user r14-r31 */ |
| 243 | |
| 244 | /* ******************** NIP ******************** */ |
| 245 | mfspr r3, SPRN_TFHAR |
| 246 | std r3, _NIP(r7) /* Returns to failhandler */ |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 247 | /* |
| 248 | * The checkpointed NIP is ignored when rescheduling/rechkpting, |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 249 | * but is used in signal return to 'wind back' to the abort handler. |
| 250 | */ |
| 251 | |
Gustavo Romero | d0ffdee | 2020-09-19 12:00:25 -0300 | [diff] [blame] | 252 | /* ***************** CTR, LR, CR, XER ********** */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 253 | mfctr r3 |
| 254 | mflr r4 |
| 255 | mfcr r5 |
| 256 | mfxer r6 |
| 257 | |
| 258 | std r3, _CTR(r7) |
| 259 | std r4, _LINK(r7) |
| 260 | std r5, _CCR(r7) |
| 261 | std r6, _XER(r7) |
| 262 | |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 263 | /* ******************** TAR, DSCR ********** */ |
Michael Neuling | 28e61cc | 2013-08-09 17:29:31 +1000 | [diff] [blame] | 264 | mfspr r3, SPRN_TAR |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 265 | mfspr r4, SPRN_DSCR |
Michael Neuling | 28e61cc | 2013-08-09 17:29:31 +1000 | [diff] [blame] | 266 | |
| 267 | std r3, THREAD_TM_TAR(r12) |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 268 | std r4, THREAD_TM_DSCR(r12) |
Michael Neuling | 28e61cc | 2013-08-09 17:29:31 +1000 | [diff] [blame] | 269 | |
Gustavo Romero | d0ffdee | 2020-09-19 12:00:25 -0300 | [diff] [blame] | 270 | /* ******************** AMR **************** */ |
| 271 | mfspr r3, SPRN_AMR |
| 272 | std r3, THREAD_TM_AMR(r12) |
| 273 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 274 | /* |
| 275 | * MSR and flags: We don't change CRs, and we don't need to alter MSR. |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 276 | */ |
| 277 | |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 278 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 279 | /* |
| 280 | * ******************** FPR/VR/VSRs ************ |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 281 | * After reclaiming, capture the checkpointed FPRs/VRs. |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 282 | * |
| 283 | * We enabled VEC/FP/VSX in the msr above, so we can execute these |
| 284 | * instructions! |
| 285 | */ |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 286 | mr r3, r12 |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 287 | |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 288 | /* Altivec (VEC/VMX/VR)*/ |
Cyril Bur | 000ec28 | 2016-09-23 16:18:25 +1000 | [diff] [blame] | 289 | addi r7, r3, THREAD_CKVRSTATE |
Breno Leitao | 9669556 | 2018-06-18 19:59:42 -0300 | [diff] [blame] | 290 | SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 ckvr_state */ |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 291 | mfvscr v0 |
| 292 | li r6, VRSTATE_VSCR |
| 293 | stvx v0, r7, r6 |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 294 | |
| 295 | /* VRSAVE */ |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 296 | mfspr r0, SPRN_VRSAVE |
Cyril Bur | 000ec28 | 2016-09-23 16:18:25 +1000 | [diff] [blame] | 297 | std r0, THREAD_CKVRSAVE(r3) |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 298 | |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 299 | /* Floating Point (FP) */ |
Cyril Bur | 000ec28 | 2016-09-23 16:18:25 +1000 | [diff] [blame] | 300 | addi r7, r3, THREAD_CKFPSTATE |
Breno Leitao | 9669556 | 2018-06-18 19:59:42 -0300 | [diff] [blame] | 301 | SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 ckfp_state */ |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 302 | mffs fr0 |
| 303 | stfd fr0,FPSTATE_FPSCR(r7) |
| 304 | |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 305 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 306 | /* |
| 307 | * TM regs, incl TEXASR -- these live in thread_struct. Note they've |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 308 | * been updated by the treclaim, to explain to userland the failure |
| 309 | * cause (aborted). |
| 310 | */ |
| 311 | mfspr r0, SPRN_TEXASR |
| 312 | mfspr r3, SPRN_TFHAR |
| 313 | mfspr r4, SPRN_TFIAR |
| 314 | std r0, THREAD_TM_TEXASR(r12) |
| 315 | std r3, THREAD_TM_TFHAR(r12) |
| 316 | std r4, THREAD_TM_TFIAR(r12) |
| 317 | |
Gustavo Romero | d0ffdee | 2020-09-19 12:00:25 -0300 | [diff] [blame] | 318 | /* Restore kernel live AMR */ |
| 319 | ld r8, TM_FRAME_L1(r1) |
| 320 | mtspr SPRN_AMR, r8 |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 321 | |
| 322 | /* Restore original MSR/IRQ state & clear TM mode */ |
| 323 | ld r14, TM_FRAME_L0(r1) /* Orig MSR */ |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 324 | |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 325 | li r15, 0 |
| 326 | rldimi r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1 |
| 327 | mtmsrd r14 |
| 328 | |
| 329 | REST_NVGPRS(r1) |
| 330 | |
| 331 | addi r1, r1, TM_FRAME_SIZE |
Anton Blanchard | bbe30b3 | 2013-10-15 14:36:31 +1100 | [diff] [blame] | 332 | lwz r4, 8(r1) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 333 | ld r0, 16(r1) |
| 334 | mtcr r4 |
| 335 | mtlr r0 |
Anton Blanchard | 6403105 | 2014-03-10 10:52:17 +1100 | [diff] [blame] | 336 | ld r2, STK_GOT(r1) |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 337 | |
Sam bobroff | 1739ea9 | 2014-05-21 16:32:38 +1000 | [diff] [blame] | 338 | /* Load CPU's default DSCR */ |
Anshuman Khandual | 1db3652 | 2015-05-21 12:13:03 +0530 | [diff] [blame] | 339 | ld r0, PACA_DSCR_DEFAULT(r13) |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 340 | mtspr SPRN_DSCR, r0 |
| 341 | |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 342 | blr |
| 343 | |
| 344 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 345 | /* |
Cyril Bur | a596a7e | 2018-02-05 16:17:16 +1100 | [diff] [blame] | 346 | * void __tm_recheckpoint(struct thread_struct *thread) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 347 | * - Restore the checkpointed register state saved by tm_reclaim |
| 348 | * when we switch_to a process. |
| 349 | * |
| 350 | * Call with IRQs off, stacks get all out of sync for |
| 351 | * some periods in here! |
| 352 | */ |
Michael Neuling | e6b8fd0 | 2014-04-04 20:19:48 +1100 | [diff] [blame] | 353 | _GLOBAL(__tm_recheckpoint) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 354 | mfcr r5 |
| 355 | mflr r0 |
Anton Blanchard | bbe30b3 | 2013-10-15 14:36:31 +1100 | [diff] [blame] | 356 | stw r5, 8(r1) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 357 | std r0, 16(r1) |
Anton Blanchard | 6403105 | 2014-03-10 10:52:17 +1100 | [diff] [blame] | 358 | std r2, STK_GOT(r1) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 359 | stdu r1, -TM_FRAME_SIZE(r1) |
| 360 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 361 | /* |
Nicholas Piggin | c03be0a | 2022-11-27 22:49:32 +1000 | [diff] [blame] | 362 | * We've a struct pt_regs at [r1+STACK_INT_FRAME_REGS]. |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 363 | * This is used for backing up the NVGPRs: |
| 364 | */ |
| 365 | SAVE_NVGPRS(r1) |
| 366 | |
Gustavo Romero | d0ffdee | 2020-09-19 12:00:25 -0300 | [diff] [blame] | 367 | /* |
| 368 | * Save kernel live AMR since it will be clobbered for trechkpt |
| 369 | * but can be used elsewhere later in kernel space. |
| 370 | */ |
| 371 | mfspr r8, SPRN_AMR |
| 372 | std r8, TM_FRAME_L0(r1) |
| 373 | |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 374 | /* Load complete register state from ts_ckpt* registers */ |
| 375 | |
| 376 | addi r7, r3, PT_CKPT_REGS /* Thread's ckpt_regs */ |
| 377 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 378 | /* |
| 379 | * Make r7 look like an exception frame so that we can use the neat |
| 380 | * GPRx(n) macros. r7 is now NOT a pt_regs ptr! |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 381 | */ |
Nicholas Piggin | c03be0a | 2022-11-27 22:49:32 +1000 | [diff] [blame] | 382 | subi r7, r7, STACK_INT_FRAME_REGS |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 383 | |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 384 | /* We need to setup MSR for FP/VMX/VSX register save instructions. */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 385 | mfmsr r6 |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 386 | mr r5, r6 |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 387 | ori r5, r5, MSR_FP |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 388 | #ifdef CONFIG_ALTIVEC |
| 389 | oris r5, r5, MSR_VEC@h |
| 390 | #endif |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 391 | #ifdef CONFIG_VSX |
| 392 | BEGIN_FTR_SECTION |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 393 | oris r5,r5, MSR_VSX@h |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 394 | END_FTR_SECTION_IFSET(CPU_FTR_VSX) |
| 395 | #endif |
Cyril Bur | eb5c3f1 | 2017-11-02 14:09:05 +1100 | [diff] [blame] | 396 | mtmsrd r5 |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 397 | |
Michael Neuling | f110c0c | 2013-04-09 16:18:55 +1000 | [diff] [blame] | 398 | #ifdef CONFIG_ALTIVEC |
Cyril Bur | dc31066 | 2016-09-23 16:18:24 +1000 | [diff] [blame] | 399 | /* |
| 400 | * FP and VEC registers: These are recheckpointed from |
| 401 | * thread.ckfp_state and thread.ckvr_state respectively. The |
| 402 | * thread.fp_state[] version holds the 'live' (transactional) |
| 403 | * and will be loaded subsequently by any FPUnavailable trap. |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 404 | */ |
Cyril Bur | 000ec28 | 2016-09-23 16:18:25 +1000 | [diff] [blame] | 405 | addi r8, r3, THREAD_CKVRSTATE |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 406 | li r5, VRSTATE_VSCR |
Anton Blanchard | c2ce6f9 | 2015-02-10 09:51:22 +1100 | [diff] [blame] | 407 | lvx v0, r8, r5 |
| 408 | mtvscr v0 |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 409 | REST_32VRS(0, r5, r8) /* r5 scratch, r8 ptr */ |
Cyril Bur | 000ec28 | 2016-09-23 16:18:25 +1000 | [diff] [blame] | 410 | ld r5, THREAD_CKVRSAVE(r3) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 411 | mtspr SPRN_VRSAVE, r5 |
Michael Neuling | f110c0c | 2013-04-09 16:18:55 +1000 | [diff] [blame] | 412 | #endif |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 413 | |
Cyril Bur | 000ec28 | 2016-09-23 16:18:25 +1000 | [diff] [blame] | 414 | addi r8, r3, THREAD_CKFPSTATE |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 415 | lfd fr0, FPSTATE_FPSCR(r8) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 416 | MTFSF_L(fr0) |
Paul Mackerras | de79f7b | 2013-09-10 20:20:42 +1000 | [diff] [blame] | 417 | REST_32FPRS_VSRS(0, R4, R8) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 418 | |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 419 | mtmsr r6 /* FP/Vec off again! */ |
| 420 | |
| 421 | restore_gprs: |
Michael Neuling | 28e61cc | 2013-08-09 17:29:31 +1000 | [diff] [blame] | 422 | |
Gustavo Romero | d0ffdee | 2020-09-19 12:00:25 -0300 | [diff] [blame] | 423 | /* ****************** CTR, LR, XER ************* */ |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 424 | ld r4, _CTR(r7) |
| 425 | ld r5, _LINK(r7) |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 426 | ld r8, _XER(r7) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 427 | |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 428 | mtctr r4 |
| 429 | mtlr r5 |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 430 | mtxer r8 |
| 431 | |
| 432 | /* ******************** TAR ******************** */ |
| 433 | ld r4, THREAD_TM_TAR(r3) |
| 434 | mtspr SPRN_TAR, r4 |
| 435 | |
Gustavo Romero | d0ffdee | 2020-09-19 12:00:25 -0300 | [diff] [blame] | 436 | /* ******************** AMR ******************** */ |
| 437 | ld r4, THREAD_TM_AMR(r3) |
| 438 | mtspr SPRN_AMR, r4 |
| 439 | |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 440 | /* Load up the PPR and DSCR in GPRs only at this stage */ |
| 441 | ld r5, THREAD_TM_DSCR(r3) |
| 442 | ld r6, THREAD_TM_PPR(r3) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 443 | |
Michael Neuling | 7f06f21 | 2014-03-28 16:40:34 +1100 | [diff] [blame] | 444 | REST_GPR(0, r7) /* GPR0 */ |
Nicholas Piggin | aebd1fb | 2021-10-22 16:13:22 +1000 | [diff] [blame] | 445 | REST_GPRS(2, 4, r7) /* GPR2-4 */ |
Nicholas Piggin | 9d71165 | 2022-03-11 12:47:33 +1000 | [diff] [blame] | 446 | REST_GPRS(8, 12, r7) /* GPR8-12 */ |
| 447 | REST_GPRS(14, 31, r7) /* GPR14-31 */ |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 448 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 449 | /* Load up PPR and DSCR here so we don't run with user values for long */ |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 450 | mtspr SPRN_DSCR, r5 |
| 451 | mtspr SPRN_PPR, r6 |
| 452 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 453 | /* |
| 454 | * Do final sanity check on TEXASR to make sure FS is set. Do this |
Michael Neuling | 7f06f21 | 2014-03-28 16:40:34 +1100 | [diff] [blame] | 455 | * here before we load up the userspace r1 so any bugs we hit will get |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 456 | * a call chain. |
| 457 | */ |
Michael Neuling | 7f06f21 | 2014-03-28 16:40:34 +1100 | [diff] [blame] | 458 | mfspr r5, SPRN_TEXASR |
| 459 | srdi r5, r5, 16 |
| 460 | li r6, (TEXASR_FS)@h |
| 461 | and r6, r6, r5 |
| 462 | 1: tdeqi r6, 0 |
| 463 | EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0 |
| 464 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 465 | /* |
| 466 | * Do final sanity check on MSR to make sure we are not transactional |
| 467 | * or suspended. |
Michael Neuling | 7f06f21 | 2014-03-28 16:40:34 +1100 | [diff] [blame] | 468 | */ |
| 469 | mfmsr r6 |
| 470 | li r5, (MSR_TS_MASK)@higher |
| 471 | srdi r6, r6, 32 |
| 472 | and r6, r6, r5 |
| 473 | 1: tdnei r6, 0 |
| 474 | EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0 |
| 475 | |
| 476 | /* Restore CR */ |
| 477 | ld r6, _CCR(r7) |
| 478 | mtcr r6 |
| 479 | |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 480 | REST_GPR(6, r7) |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 481 | |
| 482 | /* |
Nicholas Piggin | 9d71165 | 2022-03-11 12:47:33 +1000 | [diff] [blame] | 483 | * Store user r1 and r5 and r13 on the stack (in the unused save |
| 484 | * areas / compiler reserved areas), so that we can access them after |
| 485 | * we clear MSR RI. |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 486 | */ |
| 487 | |
| 488 | REST_GPR(5, r7) |
| 489 | std r5, -8(r1) |
Nicholas Piggin | 9d71165 | 2022-03-11 12:47:33 +1000 | [diff] [blame] | 490 | ld r5, GPR13(r7) |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 491 | std r5, -16(r1) |
Nicholas Piggin | 9d71165 | 2022-03-11 12:47:33 +1000 | [diff] [blame] | 492 | ld r5, GPR1(r7) |
| 493 | std r5, -24(r1) |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 494 | |
| 495 | REST_GPR(7, r7) |
| 496 | |
Nicholas Piggin | 9d71165 | 2022-03-11 12:47:33 +1000 | [diff] [blame] | 497 | /* Stash the stack pointer away for use after recheckpoint */ |
| 498 | std r1, PACAR1(r13) |
| 499 | |
| 500 | /* Clear MSR RI since we are about to clobber r13. EE is already off */ |
Michael Neuling | 190ce86 | 2016-06-28 13:01:04 +1000 | [diff] [blame] | 501 | li r5, 0 |
| 502 | mtmsrd r5, 1 |
| 503 | |
| 504 | /* |
| 505 | * BE CAREFUL HERE: |
| 506 | * At this point we can't take an SLB miss since we have MSR_RI |
| 507 | * off. Load only to/from the stack/paca which are in SLB bolted regions |
| 508 | * until we turn MSR RI back on. |
| 509 | */ |
| 510 | |
| 511 | ld r5, -8(r1) |
Nicholas Piggin | 9d71165 | 2022-03-11 12:47:33 +1000 | [diff] [blame] | 512 | ld r13, -16(r1) |
| 513 | ld r1, -24(r1) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 514 | |
| 515 | /* Commit register state as checkpointed state: */ |
| 516 | TRECHKPT |
| 517 | |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 518 | HMT_MEDIUM |
| 519 | |
Michael Neuling | 306b1c0 | 2018-09-27 15:05:15 +1000 | [diff] [blame] | 520 | /* |
| 521 | * Our transactional state has now changed. |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 522 | * |
| 523 | * Now just get out of here. Transactional (current) state will be |
| 524 | * updated once restore is called on the return path in the _switch-ed |
| 525 | * -to process. |
| 526 | */ |
| 527 | |
| 528 | GET_PACA(r13) |
Nicholas Piggin | 9d71165 | 2022-03-11 12:47:33 +1000 | [diff] [blame] | 529 | ld r1, PACAR1(r13) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 530 | |
Nicholas Piggin | 9d71165 | 2022-03-11 12:47:33 +1000 | [diff] [blame] | 531 | /* R13, R1 is restored, so we are recoverable again. EE is still off */ |
Michael Neuling | 090b928 | 2013-06-28 18:17:09 +1000 | [diff] [blame] | 532 | li r4, MSR_RI |
| 533 | mtmsrd r4, 1 |
| 534 | |
Gustavo Romero | d0ffdee | 2020-09-19 12:00:25 -0300 | [diff] [blame] | 535 | /* Restore kernel live AMR */ |
| 536 | ld r8, TM_FRAME_L0(r1) |
| 537 | mtspr SPRN_AMR, r8 |
| 538 | |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 539 | REST_NVGPRS(r1) |
| 540 | |
| 541 | addi r1, r1, TM_FRAME_SIZE |
Anton Blanchard | bbe30b3 | 2013-10-15 14:36:31 +1100 | [diff] [blame] | 542 | lwz r4, 8(r1) |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 543 | ld r0, 16(r1) |
| 544 | mtcr r4 |
| 545 | mtlr r0 |
Anton Blanchard | 6403105 | 2014-03-10 10:52:17 +1100 | [diff] [blame] | 546 | ld r2, STK_GOT(r1) |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 547 | |
Sam bobroff | 1739ea9 | 2014-05-21 16:32:38 +1000 | [diff] [blame] | 548 | /* Load CPU's default DSCR */ |
Anshuman Khandual | 1db3652 | 2015-05-21 12:13:03 +0530 | [diff] [blame] | 549 | ld r0, PACA_DSCR_DEFAULT(r13) |
Michael Neuling | e9bdc3d6 | 2013-09-26 13:29:09 +1000 | [diff] [blame] | 550 | mtspr SPRN_DSCR, r0 |
| 551 | |
Michael Neuling | 98ae22e | 2013-02-13 16:21:35 +0000 | [diff] [blame] | 552 | blr |
| 553 | |
| 554 | /* ****************************************************************** */ |