blob: 4ae39db70044e33d501434422636d9b3eff3319e [file] [log] [blame]
Thomas Gleixner2874c5f2019-05-27 08:55:01 +02001/* SPDX-License-Identifier: GPL-2.0-or-later */
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002/*
3 * FPU support code, moved here from head.S so that it can be used
4 * by chips which use other head-whatever.S files.
5 *
Paul Mackerrasfea23bf2006-08-30 14:45:35 +10006 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
7 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
8 * Copyright (C) 1996 Paul Mackerras.
9 * Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
Paul Mackerras14cf11a2005-09-26 16:04:21 +100010 */
11
Paul Mackerrasb3b8dc62005-10-10 22:20:10 +100012#include <asm/reg.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100013#include <asm/page.h>
14#include <asm/mmu.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100015#include <asm/cputable.h>
16#include <asm/cache.h>
17#include <asm/thread_info.h>
18#include <asm/ppc_asm.h>
19#include <asm/asm-offsets.h>
Stephen Rothwell46f52212010-11-18 15:06:17 +000020#include <asm/ptrace.h>
Al Viro9445aa12016-01-13 23:33:46 -050021#include <asm/export.h>
Christophe Leroyec0c4642018-07-05 16:24:57 +000022#include <asm/asm-compat.h>
Christophe Leroy2c86cd12018-07-05 16:25:01 +000023#include <asm/feature-fixups.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100024
Michael Neuling72ffff52008-06-25 14:07:18 +100025#ifdef CONFIG_VSX
Michael Neuling0b7673c2012-06-25 13:33:23 +000026#define __REST_32FPVSRS(n,c,base) \
Michael Neuling72ffff52008-06-25 14:07:18 +100027BEGIN_FTR_SECTION \
28 b 2f; \
29END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
30 REST_32FPRS(n,base); \
31 b 3f; \
322: REST_32VSRS(n,c,base); \
333:
34
Michael Neuling0b7673c2012-06-25 13:33:23 +000035#define __SAVE_32FPVSRS(n,c,base) \
Michael Neuling72ffff52008-06-25 14:07:18 +100036BEGIN_FTR_SECTION \
37 b 2f; \
38END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
39 SAVE_32FPRS(n,base); \
40 b 3f; \
412: SAVE_32VSRS(n,c,base); \
423:
43#else
Michael Neuling0b7673c2012-06-25 13:33:23 +000044#define __REST_32FPVSRS(n,b,base) REST_32FPRS(n, base)
45#define __SAVE_32FPVSRS(n,b,base) SAVE_32FPRS(n, base)
Michael Neuling72ffff52008-06-25 14:07:18 +100046#endif
Michael Neuling0b7673c2012-06-25 13:33:23 +000047#define REST_32FPVSRS(n,c,base) __REST_32FPVSRS(n,__REG_##c,__REG_##base)
48#define SAVE_32FPVSRS(n,c,base) __SAVE_32FPVSRS(n,__REG_##c,__REG_##base)
Michael Neuling72ffff52008-06-25 14:07:18 +100049
Paul Mackerras14cf11a2005-09-26 16:04:21 +100050/*
Paul Mackerras18461962013-09-10 20:21:10 +100051 * Load state from memory into FP registers including FPSCR.
52 * Assumes the caller has enabled FP in the MSR.
53 */
54_GLOBAL(load_fp_state)
55 lfd fr0,FPSTATE_FPSCR(r3)
56 MTFSF_L(fr0)
57 REST_32FPVSRS(0, R4, R3)
58 blr
Al Viro9445aa12016-01-13 23:33:46 -050059EXPORT_SYMBOL(load_fp_state)
Nicholas Piggine2b36d52019-05-02 15:21:07 +100060_ASM_NOKPROBE_SYMBOL(load_fp_state); /* used by restore_math */
Paul Mackerras18461962013-09-10 20:21:10 +100061
62/*
63 * Store FP state into memory, including FPSCR
64 * Assumes the caller has enabled FP in the MSR.
65 */
66_GLOBAL(store_fp_state)
67 SAVE_32FPVSRS(0, R4, R3)
68 mffs fr0
69 stfd fr0,FPSTATE_FPSCR(r3)
70 blr
Al Viro9445aa12016-01-13 23:33:46 -050071EXPORT_SYMBOL(store_fp_state)
Paul Mackerras18461962013-09-10 20:21:10 +100072
73/*
Paul Mackerras14cf11a2005-09-26 16:04:21 +100074 * This task wants to use the FPU now.
75 * On UP, disable FP for the task which had the FPU previously,
76 * and save its floating-point registers in its thread_struct.
77 * Load up this task's FP registers from its thread_struct,
78 * enable the FPU for the current task and return to the task.
Paul Mackerras955c1ca2013-10-23 09:40:02 +010079 * Note that on 32-bit this can only use registers that will be
80 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
Paul Mackerras14cf11a2005-09-26 16:04:21 +100081 */
Paul Mackerrasb85a0462005-10-06 10:59:19 +100082_GLOBAL(load_up_fpu)
Paul Mackerras14cf11a2005-09-26 16:04:21 +100083 mfmsr r5
84 ori r5,r5,MSR_FP
Michael Neulingce48b212008-06-25 14:07:18 +100085#ifdef CONFIG_VSX
86BEGIN_FTR_SECTION
87 oris r5,r5,MSR_VSX@h
88END_FTR_SECTION_IFSET(CPU_FTR_VSX)
89#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +100090 SYNC
91 MTMSRD(r5) /* enable use of fpu now */
92 isync
Paul Mackerras14cf11a2005-09-26 16:04:21 +100093 /* enable use of FP after return */
Paul Mackerrasb85a0462005-10-06 10:59:19 +100094#ifdef CONFIG_PPC32
Paul Mackerrasde79f7b2013-09-10 20:20:42 +100095 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
Christophe Leroycd08f102019-12-21 08:32:38 +000096#ifdef CONFIG_VMAP_STACK
97 tovirt(r5, r5)
98#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +100099 lwz r4,THREAD_FPEXC_MODE(r5)
100 ori r9,r9,MSR_FP /* enable FP for current */
101 or r9,r9,r4
Paul Mackerrasb85a0462005-10-06 10:59:19 +1000102#else
103 ld r4,PACACURRENT(r13)
104 addi r5,r4,THREAD /* Get THREAD */
Paul Mackerrase2f5a3c2006-02-07 13:55:30 +1100105 lwz r4,THREAD_FPEXC_MODE(r5)
Paul Mackerrasb85a0462005-10-06 10:59:19 +1000106 ori r12,r12,MSR_FP
107 or r12,r12,r4
108 std r12,_MSR(r1)
109#endif
Nicholas Pigginb2b46302020-06-24 09:41:39 +1000110 li r4,1
Cyril Bur70fe3d92016-02-29 17:53:47 +1100111 stb r4,THREAD_LOAD_FP(r5)
Paul Mackerras955c1ca2013-10-23 09:40:02 +0100112 addi r10,r5,THREAD_FPSTATE
113 lfd fr0,FPSTATE_FPSCR(r10)
Anton Blanchard3a2c48c2006-06-10 20:18:39 +1000114 MTFSF_L(fr0)
Paul Mackerras955c1ca2013-10-23 09:40:02 +0100115 REST_32FPVSRS(0, R4, R10)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000116 /* restore registers and return */
117 /* we haven't used ctr or xer or lr */
Michael Neuling6f3d8e62008-06-25 14:07:18 +1000118 blr
Christophe Leroy5f32e832020-03-31 16:03:44 +0000119_ASM_NOKPROBE_SYMBOL(load_up_fpu)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000120
121/*
Cyril Bur87924682016-02-29 17:53:49 +1100122 * save_fpu(tsk)
123 * Save the floating-point registers in its thread_struct.
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000124 * Enables the FPU for use in the kernel on return.
125 */
Cyril Bur87924682016-02-29 17:53:49 +1100126_GLOBAL(save_fpu)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000127 addi r3,r3,THREAD /* want THREAD of task */
Paul Mackerras18461962013-09-10 20:21:10 +1000128 PPC_LL r6,THREAD_FPSAVEAREA(r3)
David Gibson3ddfbcf2005-11-10 12:56:55 +1100129 PPC_LL r5,PT_REGS(r3)
Paul Mackerras18461962013-09-10 20:21:10 +1000130 PPC_LCMPI 0,r6,0
131 bne 2f
Paul Mackerrasde79f7b2013-09-10 20:20:42 +1000132 addi r6,r3,THREAD_FPSTATE
Cyril Bur87924682016-02-29 17:53:49 +11001332: SAVE_32FPVSRS(0, R4, R6)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000134 mffs fr0
Paul Mackerrasde79f7b2013-09-10 20:20:42 +1000135 stfd fr0,FPSTATE_FPSCR(r6)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000136 blr
David Gibson25c8a782005-10-27 16:27:25 +1000137
138/*
139 * These are used in the alignment trap handler when emulating
140 * single-precision loads and stores.
David Gibson25c8a782005-10-27 16:27:25 +1000141 */
142
143_GLOBAL(cvt_fd)
David Gibson25c8a782005-10-27 16:27:25 +1000144 lfs 0,0(r3)
145 stfd 0,0(r4)
David Gibson25c8a782005-10-27 16:27:25 +1000146 blr
147
148_GLOBAL(cvt_df)
David Gibson25c8a782005-10-27 16:27:25 +1000149 lfd 0,0(r3)
150 stfs 0,0(r4)
David Gibson25c8a782005-10-27 16:27:25 +1000151 blr