Thomas Gleixner | 41173ab | 2019-05-28 09:57:11 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 2 | /* |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 3 | * |
Ralf Baechle | 41c594a | 2006-04-05 09:45:45 +0100 | [diff] [blame] | 4 | * Copyright (C) 2004, 05, 06 MIPS Technologies, Inc. |
| 5 | * Elizabeth Clarke (beth@mips.com) |
| 6 | * Ralf Baechle (ralf@linux-mips.org) |
| 7 | * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 8 | */ |
| 9 | #include <linux/kernel.h> |
| 10 | #include <linux/sched.h> |
| 11 | #include <linux/cpumask.h> |
| 12 | #include <linux/interrupt.h> |
| 13 | #include <linux/compiler.h> |
Arnd Bergmann | fc69910 | 2017-03-08 08:29:31 +0100 | [diff] [blame] | 14 | #include <linux/sched/task_stack.h> |
Ralf Baechle | 0ab7aef | 2007-03-02 20:42:04 +0000 | [diff] [blame] | 15 | #include <linux/smp.h> |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 16 | |
Arun Sharma | 60063497 | 2011-07-26 16:09:06 -0700 | [diff] [blame] | 17 | #include <linux/atomic.h> |
Ralf Baechle | 41c594a | 2006-04-05 09:45:45 +0100 | [diff] [blame] | 18 | #include <asm/cacheflush.h> |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 19 | #include <asm/cpu.h> |
| 20 | #include <asm/processor.h> |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 21 | #include <asm/hardirq.h> |
| 22 | #include <asm/mmu_context.h> |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 23 | #include <asm/time.h> |
| 24 | #include <asm/mipsregs.h> |
| 25 | #include <asm/mipsmtregs.h> |
Ralf Baechle | 41c594a | 2006-04-05 09:45:45 +0100 | [diff] [blame] | 26 | #include <asm/mips_mt.h> |
Paul Burton | 72eb299 | 2017-08-12 21:36:34 -0700 | [diff] [blame] | 27 | #include <asm/mips-cps.h> |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 28 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 29 | static void __init smvp_copy_vpe_config(void) |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 30 | { |
| 31 | write_vpe_c0_status( |
| 32 | (read_c0_status() & ~(ST0_IM | ST0_IE | ST0_KSU)) | ST0_CU0); |
| 33 | |
| 34 | /* set config to be the same as vpe0, particularly kseg0 coherency alg */ |
| 35 | write_vpe_c0_config( read_c0_config()); |
| 36 | |
| 37 | /* make sure there are no software interrupts pending */ |
| 38 | write_vpe_c0_cause(0); |
| 39 | |
| 40 | /* Propagate Config7 */ |
| 41 | write_vpe_c0_config7(read_c0_config7()); |
Ralf Baechle | 70e46f4 | 2006-10-31 18:33:09 +0000 | [diff] [blame] | 42 | |
| 43 | write_vpe_c0_count(read_c0_count()); |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 44 | } |
| 45 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 46 | static unsigned int __init smvp_vpe_init(unsigned int tc, unsigned int mvpconf0, |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 47 | unsigned int ncpu) |
| 48 | { |
| 49 | if (tc > ((mvpconf0 & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT)) |
| 50 | return ncpu; |
| 51 | |
| 52 | /* Deactivate all but VPE 0 */ |
| 53 | if (tc != 0) { |
| 54 | unsigned long tmp = read_vpe_c0_vpeconf0(); |
| 55 | |
| 56 | tmp &= ~VPECONF0_VPA; |
| 57 | |
| 58 | /* master VPE */ |
| 59 | tmp |= VPECONF0_MVP; |
| 60 | write_vpe_c0_vpeconf0(tmp); |
| 61 | |
| 62 | /* Record this as available CPU */ |
Rusty Russell | 4037ac6 | 2009-09-24 09:34:47 -0600 | [diff] [blame] | 63 | set_cpu_possible(tc, true); |
Markos Chandras | c2c2a64 | 2013-10-09 16:16:25 +0100 | [diff] [blame] | 64 | set_cpu_present(tc, true); |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 65 | __cpu_number_map[tc] = ++ncpu; |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 66 | __cpu_logical_map[ncpu] = tc; |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 67 | } |
| 68 | |
| 69 | /* Disable multi-threading with TC's */ |
| 70 | write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() & ~VPECONTROL_TE); |
| 71 | |
| 72 | if (tc != 0) |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 73 | smvp_copy_vpe_config(); |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 74 | |
Paul Burton | f875a832 | 2017-08-12 19:49:35 -0700 | [diff] [blame] | 75 | cpu_set_vpe_id(&cpu_data[ncpu], tc); |
Paul Burton | 1eed400 | 2017-03-30 12:06:12 -0700 | [diff] [blame] | 76 | |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 77 | return ncpu; |
| 78 | } |
| 79 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 80 | static void __init smvp_tc_init(unsigned int tc, unsigned int mvpconf0) |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 81 | { |
| 82 | unsigned long tmp; |
| 83 | |
| 84 | if (!tc) |
| 85 | return; |
| 86 | |
| 87 | /* bind a TC to each VPE, May as well put all excess TC's |
| 88 | on the last VPE */ |
| 89 | if (tc >= (((mvpconf0 & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT)+1)) |
| 90 | write_tc_c0_tcbind(read_tc_c0_tcbind() | ((mvpconf0 & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT)); |
| 91 | else { |
| 92 | write_tc_c0_tcbind(read_tc_c0_tcbind() | tc); |
| 93 | |
| 94 | /* and set XTC */ |
| 95 | write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | (tc << VPECONF0_XTC_SHIFT)); |
| 96 | } |
| 97 | |
| 98 | tmp = read_tc_c0_tcstatus(); |
| 99 | |
| 100 | /* mark not allocated and not dynamically allocatable */ |
| 101 | tmp &= ~(TCSTATUS_A | TCSTATUS_DA); |
| 102 | tmp |= TCSTATUS_IXMT; /* interrupt exempt */ |
| 103 | write_tc_c0_tcstatus(tmp); |
| 104 | |
| 105 | write_tc_c0_tchalt(TCHALT_H); |
| 106 | } |
| 107 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 108 | static void vsmp_init_secondary(void) |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 109 | { |
Ralf Baechle | d002aaa | 2010-12-01 17:33:17 +0000 | [diff] [blame] | 110 | /* This is Malta specific: IPI,performance and timer interrupts */ |
Paul Burton | 72eb299 | 2017-08-12 21:36:34 -0700 | [diff] [blame] | 111 | if (mips_gic_present()) |
James Hogan | c3f134f | 2015-01-16 11:10:46 +0000 | [diff] [blame] | 112 | change_c0_status(ST0_IM, STATUSF_IP2 | STATUSF_IP3 | |
| 113 | STATUSF_IP4 | STATUSF_IP5 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 114 | STATUSF_IP6 | STATUSF_IP7); |
| 115 | else |
| 116 | change_c0_status(ST0_IM, STATUSF_IP0 | STATUSF_IP1 | |
| 117 | STATUSF_IP6 | STATUSF_IP7); |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 118 | } |
| 119 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 120 | static void vsmp_smp_finish(void) |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 121 | { |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 122 | /* CDFIXME: remove this? */ |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 123 | write_c0_compare(read_c0_count() + (8* mips_hpt_frequency/HZ)); |
| 124 | |
| 125 | #ifdef CONFIG_MIPS_MT_FPAFF |
| 126 | /* If we have an FPU, enroll ourselves in the FPU-full mask */ |
| 127 | if (cpu_has_fpu) |
Rusty Russell | 8dd9289 | 2015-03-05 10:49:17 +1030 | [diff] [blame] | 128 | cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask); |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 129 | #endif /* CONFIG_MIPS_MT_FPAFF */ |
| 130 | |
| 131 | local_irq_enable(); |
| 132 | } |
| 133 | |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 134 | /* |
| 135 | * Setup the PC, SP, and GP of a secondary processor and start it |
| 136 | * running! |
| 137 | * smp_bootstrap is the place to resume from |
| 138 | * __KSTK_TOS(idle) is apparently the stack pointer |
| 139 | * (unsigned long)idle->thread_info the gp |
| 140 | * assumes a 1:1 mapping of TC => VPE |
| 141 | */ |
Paul Burton | d595d42 | 2017-08-12 19:49:40 -0700 | [diff] [blame] | 142 | static int vsmp_boot_secondary(int cpu, struct task_struct *idle) |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 143 | { |
| 144 | struct thread_info *gp = task_thread_info(idle); |
| 145 | dvpe(); |
| 146 | set_c0_mvpcontrol(MVPCONTROL_VPC); |
| 147 | |
| 148 | settc(cpu); |
| 149 | |
| 150 | /* restart */ |
| 151 | write_tc_c0_tcrestart((unsigned long)&smp_bootstrap); |
| 152 | |
| 153 | /* enable the tc this vpe/cpu will be running */ |
| 154 | write_tc_c0_tcstatus((read_tc_c0_tcstatus() & ~TCSTATUS_IXMT) | TCSTATUS_A); |
| 155 | |
| 156 | write_tc_c0_tchalt(0); |
| 157 | |
| 158 | /* enable the VPE */ |
| 159 | write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); |
| 160 | |
| 161 | /* stack pointer */ |
| 162 | write_tc_gpr_sp( __KSTK_TOS(idle)); |
| 163 | |
| 164 | /* global pointer */ |
| 165 | write_tc_gpr_gp((unsigned long)gp); |
| 166 | |
| 167 | flush_icache_range((unsigned long)gp, |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 168 | (unsigned long)(gp + sizeof(struct thread_info))); |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 169 | |
| 170 | /* finally out of configuration and into chaos */ |
| 171 | clear_c0_mvpcontrol(MVPCONTROL_VPC); |
| 172 | |
| 173 | evpe(EVPE_ENABLE); |
Paul Burton | d595d42 | 2017-08-12 19:49:40 -0700 | [diff] [blame] | 174 | |
| 175 | return 0; |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 176 | } |
| 177 | |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 178 | /* |
| 179 | * Common setup before any secondaries are started |
| 180 | * Make sure all CPU's are in a sensible state before we boot any of the |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 181 | * secondaries |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 182 | */ |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 183 | static void __init vsmp_smp_setup(void) |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 184 | { |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 185 | unsigned int mvpconf0, ntc, tc, ncpu = 0; |
Ralf Baechle | 0ab7aef | 2007-03-02 20:42:04 +0000 | [diff] [blame] | 186 | unsigned int nvpe; |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 187 | |
Ralf Baechle | f088fc8 | 2006-04-05 09:45:47 +0100 | [diff] [blame] | 188 | #ifdef CONFIG_MIPS_MT_FPAFF |
| 189 | /* If we have an FPU, enroll ourselves in the FPU-full mask */ |
| 190 | if (cpu_has_fpu) |
Rusty Russell | 8dd9289 | 2015-03-05 10:49:17 +1030 | [diff] [blame] | 191 | cpumask_set_cpu(0, &mt_fpu_cpumask); |
Ralf Baechle | f088fc8 | 2006-04-05 09:45:47 +0100 | [diff] [blame] | 192 | #endif /* CONFIG_MIPS_MT_FPAFF */ |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 193 | if (!cpu_has_mipsmt) |
| 194 | return; |
| 195 | |
| 196 | /* disable MT so we can configure */ |
| 197 | dvpe(); |
| 198 | dmt(); |
| 199 | |
| 200 | /* Put MVPE's into 'configuration state' */ |
| 201 | set_c0_mvpcontrol(MVPCONTROL_VPC); |
| 202 | |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 203 | mvpconf0 = read_c0_mvpconf0(); |
| 204 | ntc = (mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT; |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 205 | |
Ralf Baechle | 0ab7aef | 2007-03-02 20:42:04 +0000 | [diff] [blame] | 206 | nvpe = ((mvpconf0 & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; |
| 207 | smp_num_siblings = nvpe; |
| 208 | |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 209 | /* we'll always have more TC's than VPE's, so loop setting everything |
| 210 | to a sensible state */ |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 211 | for (tc = 0; tc <= ntc; tc++) { |
| 212 | settc(tc); |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 213 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 214 | smvp_tc_init(tc, mvpconf0); |
| 215 | ncpu = smvp_vpe_init(tc, mvpconf0, ncpu); |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 216 | } |
| 217 | |
| 218 | /* Release config state */ |
| 219 | clear_c0_mvpcontrol(MVPCONTROL_VPC); |
| 220 | |
| 221 | /* We'll wait until starting the secondaries before starting MVPE */ |
| 222 | |
Ralf Baechle | 781b0f8 | 2006-10-31 18:25:10 +0000 | [diff] [blame] | 223 | printk(KERN_INFO "Detected %i available secondary CPU(s)\n", ncpu); |
Ralf Baechle | 41c594a | 2006-04-05 09:45:45 +0100 | [diff] [blame] | 224 | } |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 225 | |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 226 | static void __init vsmp_prepare_cpus(unsigned int max_cpus) |
Ralf Baechle | 41c594a | 2006-04-05 09:45:45 +0100 | [diff] [blame] | 227 | { |
Ralf Baechle | 8c976e3 | 2007-07-03 18:25:58 +0200 | [diff] [blame] | 228 | mips_mt_set_cpuoptions(); |
Ralf Baechle | 340ee4b | 2005-08-17 17:44:08 +0000 | [diff] [blame] | 229 | } |
| 230 | |
Matt Redfearn | ff2c825 | 2017-07-19 09:21:03 +0100 | [diff] [blame] | 231 | const struct plat_smp_ops vsmp_smp_ops = { |
Paul Burton | 1eed400 | 2017-03-30 12:06:12 -0700 | [diff] [blame] | 232 | .send_ipi_single = mips_smp_send_ipi_single, |
| 233 | .send_ipi_mask = mips_smp_send_ipi_mask, |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 234 | .init_secondary = vsmp_init_secondary, |
| 235 | .smp_finish = vsmp_smp_finish, |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 236 | .boot_secondary = vsmp_boot_secondary, |
| 237 | .smp_setup = vsmp_smp_setup, |
| 238 | .prepare_cpus = vsmp_prepare_cpus, |
| 239 | }; |
Ralf Baechle | d6d3c9a | 2013-10-16 17:10:07 +0200 | [diff] [blame] | 240 | |