blob: be450a3e9871673e22d56445dbe69f80522baac4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * SMP Support
3 *
4 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
5 * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com>
6 *
7 * Lots of stuff stolen from arch/alpha/kernel/smp.c
8 *
9 * 01/05/16 Rohit Seth <rohit.seth@intel.com> IA64-SMP functions. Reorganized
10 * the existing code (on the lines of x86 port).
11 * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy
12 * calibration on each CPU.
13 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id
14 * 00/03/31 Rohit Seth <rohit.seth@intel.com> Fixes for Bootstrap Processor
15 * & cpu_online_map now gets done here (instead of setup.c)
16 * 99/10/05 davidm Update to bring it in sync with new command-line processing
17 * scheme.
18 * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and
19 * smp_call_function_single to resend IPI on timeouts
20 */
21#include <linux/module.h>
22#include <linux/kernel.h>
23#include <linux/sched.h>
24#include <linux/init.h>
25#include <linux/interrupt.h>
26#include <linux/smp.h>
27#include <linux/kernel_stat.h>
28#include <linux/mm.h>
29#include <linux/cache.h>
30#include <linux/delay.h>
31#include <linux/efi.h>
32#include <linux/bitops.h>
Zou Nan haia79561132006-12-07 09:51:35 -080033#include <linux/kexec.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
35#include <asm/atomic.h>
36#include <asm/current.h>
37#include <asm/delay.h>
38#include <asm/machvec.h>
39#include <asm/io.h>
40#include <asm/irq.h>
41#include <asm/page.h>
42#include <asm/pgalloc.h>
43#include <asm/pgtable.h>
44#include <asm/processor.h>
45#include <asm/ptrace.h>
46#include <asm/sal.h>
47#include <asm/system.h>
48#include <asm/tlbflush.h>
49#include <asm/unistd.h>
50#include <asm/mca.h>
51
52/*
Jack Steiner3be44b92007-05-08 14:50:43 -070053 * Note: alignment of 4 entries/cacheline was empirically determined
54 * to be a good tradeoff between hot cachelines & spreading the array
55 * across too many cacheline.
56 */
57static struct local_tlb_flush_counts {
58 unsigned int count;
59} __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS];
60
Tejun Heob9bf3122009-06-24 15:13:47 +090061static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned short [NR_CPUS],
62 shadow_flush_counts);
Jack Steiner3be44b92007-05-08 14:50:43 -070063
Linus Torvalds1da177e2005-04-16 15:20:36 -070064#define IPI_CALL_FUNC 0
65#define IPI_CPU_STOP 1
Jens Axboef27b4332008-06-26 11:22:30 +020066#define IPI_CALL_FUNC_SINGLE 2
Zou Nan haia79561132006-12-07 09:51:35 -080067#define IPI_KDUMP_CPU_STOP 3
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
69/* This needs to be cacheline aligned because it is written to by *other* CPUs. */
Matthew Wilcoxe088a4a2009-05-22 13:49:49 -070070static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, ipi_operation);
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
72extern void cpu_halt (void);
73
Linus Torvalds1da177e2005-04-16 15:20:36 -070074static void
Hidetoshi Setoc0cd6612008-04-30 18:55:48 +090075stop_this_cpu(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070076{
77 /*
78 * Remove this CPU:
79 */
80 cpu_clear(smp_processor_id(), cpu_online_map);
81 max_xtp();
82 local_irq_disable();
83 cpu_halt();
84}
85
86void
87cpu_die(void)
88{
89 max_xtp();
90 local_irq_disable();
91 cpu_halt();
92 /* Should never be here */
93 BUG();
94 for (;;);
95}
96
97irqreturn_t
Keith Owens024e4f22006-10-18 15:36:49 +100098handle_IPI (int irq, void *dev_id)
Linus Torvalds1da177e2005-04-16 15:20:36 -070099{
100 int this_cpu = get_cpu();
101 unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation);
102 unsigned long ops;
103
104 mb(); /* Order interrupt and bit testing. */
105 while ((ops = xchg(pending_ipis, 0)) != 0) {
106 mb(); /* Order bit clearing and data access. */
107 do {
108 unsigned long which;
109
110 which = ffz(~ops);
111 ops &= ~(1 << which);
112
113 switch (which) {
Hidetoshi Setoc0cd6612008-04-30 18:55:48 +0900114 case IPI_CPU_STOP:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 stop_this_cpu();
116 break;
Jens Axboef27b4332008-06-26 11:22:30 +0200117 case IPI_CALL_FUNC:
118 generic_smp_call_function_interrupt();
119 break;
120 case IPI_CALL_FUNC_SINGLE:
121 generic_smp_call_function_single_interrupt();
122 break;
Horms45a98fc2006-12-12 17:49:03 +0900123#ifdef CONFIG_KEXEC
Hidetoshi Setoc0cd6612008-04-30 18:55:48 +0900124 case IPI_KDUMP_CPU_STOP:
Zou Nan haia79561132006-12-07 09:51:35 -0800125 unw_init_running(kdump_cpu_freeze, NULL);
126 break;
127#endif
Hidetoshi Setoc0cd6612008-04-30 18:55:48 +0900128 default:
129 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
130 this_cpu, which);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 break;
132 }
133 } while (ops);
134 mb(); /* Order data access and bit testing. */
135 }
136 put_cpu();
137 return IRQ_HANDLED;
138}
139
Jens Axboef27b4332008-06-26 11:22:30 +0200140
141
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142/*
Simon Arlott72fdbdc2007-05-11 14:55:43 -0700143 * Called with preemption disabled.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 */
145static inline void
146send_IPI_single (int dest_cpu, int op)
147{
148 set_bit(op, &per_cpu(ipi_operation, dest_cpu));
149 platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0);
150}
151
152/*
Simon Arlott72fdbdc2007-05-11 14:55:43 -0700153 * Called with preemption disabled.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154 */
155static inline void
156send_IPI_allbutself (int op)
157{
158 unsigned int i;
159
hawkes@sgi.comdc565b52005-10-10 08:43:26 -0700160 for_each_online_cpu(i) {
161 if (i != smp_processor_id())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 send_IPI_single(i, op);
163 }
164}
165
166/*
Simon Arlott72fdbdc2007-05-11 14:55:43 -0700167 * Called with preemption disabled.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 */
169static inline void
Rusty Russell40fe6972009-03-16 14:12:41 +1030170send_IPI_mask(const struct cpumask *mask, int op)
Xiantao Zhang31a6b112008-04-03 11:39:43 -0700171{
172 unsigned int cpu;
173
Rusty Russell40fe6972009-03-16 14:12:41 +1030174 for_each_cpu(cpu, mask) {
Xiantao Zhang31a6b112008-04-03 11:39:43 -0700175 send_IPI_single(cpu, op);
176 }
177}
178
179/*
180 * Called with preemption disabled.
181 */
182static inline void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183send_IPI_all (int op)
184{
185 int i;
186
hawkes@sgi.comdc565b52005-10-10 08:43:26 -0700187 for_each_online_cpu(i) {
188 send_IPI_single(i, op);
189 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190}
191
192/*
Simon Arlott72fdbdc2007-05-11 14:55:43 -0700193 * Called with preemption disabled.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 */
195static inline void
196send_IPI_self (int op)
197{
198 send_IPI_single(smp_processor_id(), op);
199}
200
Horms45a98fc2006-12-12 17:49:03 +0900201#ifdef CONFIG_KEXEC
Zou Nan haia79561132006-12-07 09:51:35 -0800202void
Al Viroccbebda2007-02-09 16:38:10 +0000203kdump_smp_send_stop(void)
Zou Nan haia79561132006-12-07 09:51:35 -0800204{
205 send_IPI_allbutself(IPI_KDUMP_CPU_STOP);
206}
207
208void
Al Viroccbebda2007-02-09 16:38:10 +0000209kdump_smp_send_init(void)
Zou Nan haia79561132006-12-07 09:51:35 -0800210{
211 unsigned int cpu, self_cpu;
212 self_cpu = smp_processor_id();
213 for_each_online_cpu(cpu) {
214 if (cpu != self_cpu) {
215 if(kdump_status[cpu] == 0)
216 platform_send_ipi(cpu, 0, IA64_IPI_DM_INIT, 0);
217 }
218 }
219}
220#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221/*
Simon Arlott72fdbdc2007-05-11 14:55:43 -0700222 * Called with preemption disabled.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 */
224void
225smp_send_reschedule (int cpu)
226{
227 platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0);
228}
Marcelo Tosattic4cb7682009-04-14 18:08:37 -0300229EXPORT_SYMBOL_GPL(smp_send_reschedule);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
Jack Steiner3be44b92007-05-08 14:50:43 -0700231/*
Simon Arlott72fdbdc2007-05-11 14:55:43 -0700232 * Called with preemption disabled.
Jack Steiner3be44b92007-05-08 14:50:43 -0700233 */
234static void
235smp_send_local_flush_tlb (int cpu)
236{
237 platform_send_ipi(cpu, IA64_IPI_LOCAL_TLB_FLUSH, IA64_IPI_DM_INT, 0);
238}
239
240void
241smp_local_flush_tlb(void)
242{
243 /*
244 * Use atomic ops. Otherwise, the load/increment/store sequence from
245 * a "++" operation can have the line stolen between the load & store.
246 * The overhead of the atomic op in negligible in this case & offers
247 * significant benefit for the brief periods where lots of cpus
248 * are simultaneously flushing TLBs.
249 */
250 ia64_fetchadd(1, &local_tlb_flush_counts[smp_processor_id()].count, acq);
251 local_flush_tlb_all();
252}
253
254#define FLUSH_DELAY 5 /* Usec backoff to eliminate excessive cacheline bouncing */
255
256void
257smp_flush_tlb_cpumask(cpumask_t xcpumask)
258{
Robin Holt97653f92008-08-15 09:21:24 -0500259 unsigned short *counts = __ia64_per_cpu_var(shadow_flush_counts);
Jack Steiner3be44b92007-05-08 14:50:43 -0700260 cpumask_t cpumask = xcpumask;
261 int mycpu, cpu, flush_mycpu = 0;
262
263 preempt_disable();
264 mycpu = smp_processor_id();
265
266 for_each_cpu_mask(cpu, cpumask)
Robin Holt97653f92008-08-15 09:21:24 -0500267 counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff;
Jack Steiner3be44b92007-05-08 14:50:43 -0700268
269 mb();
270 for_each_cpu_mask(cpu, cpumask) {
271 if (cpu == mycpu)
272 flush_mycpu = 1;
273 else
274 smp_send_local_flush_tlb(cpu);
275 }
276
277 if (flush_mycpu)
278 smp_local_flush_tlb();
279
280 for_each_cpu_mask(cpu, cpumask)
Robin Holt97653f92008-08-15 09:21:24 -0500281 while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff))
Jack Steiner3be44b92007-05-08 14:50:43 -0700282 udelay(FLUSH_DELAY);
283
284 preempt_enable();
285}
286
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287void
288smp_flush_tlb_all (void)
289{
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200290 on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291}
292
293void
294smp_flush_tlb_mm (struct mm_struct *mm)
295{
Dimitri Sivanich75c1c912010-12-28 13:34:42 -0600296 cpumask_var_t cpus;
Peter Chubba68db7632005-06-23 21:14:00 -0700297 preempt_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 /* this happens for the common case of a single-threaded fork(): */
299 if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
300 {
301 local_finish_flush_tlb_mm(mm);
Peter Chubba68db7632005-06-23 21:14:00 -0700302 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 return;
304 }
Dimitri Sivanich75c1c912010-12-28 13:34:42 -0600305 if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) {
306 smp_call_function((void (*)(void *))local_finish_flush_tlb_mm,
307 mm, 1);
308 } else {
309 cpumask_copy(cpus, mm_cpumask(mm));
310 smp_call_function_many(cpus,
311 (void (*)(void *))local_finish_flush_tlb_mm, mm, 1);
312 free_cpumask_var(cpus);
313 }
Dimitri Sivanichedb91dc2009-04-15 10:56:25 -0500314 local_irq_disable();
315 local_finish_flush_tlb_mm(mm);
316 local_irq_enable();
Peter Chubba68db7632005-06-23 21:14:00 -0700317 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318}
319
Jens Axboef27b4332008-06-26 11:22:30 +0200320void arch_send_call_function_single_ipi(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321{
Jens Axboef27b4332008-06-26 11:22:30 +0200322 send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324
Rusty Russell40fe6972009-03-16 14:12:41 +1030325void arch_send_call_function_ipi_mask(const struct cpumask *mask)
Xiantao Zhang31a6b112008-04-03 11:39:43 -0700326{
Jens Axboef27b4332008-06-26 11:22:30 +0200327 send_IPI_mask(mask, IPI_CALL_FUNC);
Xiantao Zhang31a6b112008-04-03 11:39:43 -0700328}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329
330/*
331 * this function calls the 'stop' function on all other CPUs in the system.
332 */
333void
334smp_send_stop (void)
335{
336 send_IPI_allbutself(IPI_CPU_STOP);
337}
338
Tony Luckcb2e0912007-07-20 16:14:28 -0700339int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340setup_profiling_timer (unsigned int multiplier)
341{
342 return -EINVAL;
343}