blob: deff2e693766997a259b4b3ac2fc4f789e554f1d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
Anton Vorontsovcb792952012-05-31 16:26:22 -070013#include <linux/oom.h>
14#include <linux/rcupdate.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040015#include <linux/export.h>
Anton Vorontsove4cc2f82012-05-31 16:26:26 -070016#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/kthread.h>
18#include <linux/stop_machine.h>
Ingo Molnar81615b622006-06-26 00:24:32 -070019#include <linux/mutex.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/gfp.h>
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +010021#include <linux/suspend.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
Thomas Gleixner38498a62012-04-20 13:05:44 +000023#include "smpboot.h"
24
Rusty Russell98a79d62008-12-13 21:19:41 +103025#ifdef CONFIG_SMP
Rusty Russellb3199c02008-12-30 09:05:14 +103026/* Serializes the updates to cpu_online_mask, cpu_present_mask */
Linus Torvaldsaa953872006-07-23 12:12:16 -070027static DEFINE_MUTEX(cpu_add_remove_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Lai Jiangshan79a6cde2010-05-26 14:43:36 -070029/*
30 * The following two API's must be used when attempting
31 * to serialize the updates to cpu_online_mask, cpu_present_mask.
32 */
33void cpu_maps_update_begin(void)
34{
35 mutex_lock(&cpu_add_remove_lock);
36}
37
38void cpu_maps_update_done(void)
39{
40 mutex_unlock(&cpu_add_remove_lock);
41}
42
Daniel J Blueman5c113fb2010-06-01 12:15:11 +010043static RAW_NOTIFIER_HEAD(cpu_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -070045/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
46 * Should always be manipulated under cpu_add_remove_lock
47 */
48static int cpu_hotplug_disabled;
49
Lai Jiangshan79a6cde2010-05-26 14:43:36 -070050#ifdef CONFIG_HOTPLUG_CPU
51
Gautham R Shenoyd2219382008-01-25 21:08:01 +010052static struct {
53 struct task_struct *active_writer;
54 struct mutex lock; /* Synchronizes accesses to refcount, */
55 /*
56 * Also blocks the new readers during
57 * an ongoing cpu hotplug operation.
58 */
59 int refcount;
Linus Torvalds31950eb2009-06-22 21:18:12 -070060} cpu_hotplug = {
61 .active_writer = NULL,
62 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
63 .refcount = 0,
64};
Gautham R Shenoyd2219382008-01-25 21:08:01 +010065
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +010066void get_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -080067{
Gautham R Shenoyd2219382008-01-25 21:08:01 +010068 might_sleep();
69 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -070070 return;
Gautham R Shenoyd2219382008-01-25 21:08:01 +010071 mutex_lock(&cpu_hotplug.lock);
72 cpu_hotplug.refcount++;
73 mutex_unlock(&cpu_hotplug.lock);
74
Ashok Raja9d9baa2005-11-28 13:43:46 -080075}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +010076EXPORT_SYMBOL_GPL(get_online_cpus);
Ashok Raj90d45d12005-11-08 21:34:24 -080077
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +010078void put_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -080079{
Gautham R Shenoyd2219382008-01-25 21:08:01 +010080 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -070081 return;
Gautham R Shenoyd2219382008-01-25 21:08:01 +010082 mutex_lock(&cpu_hotplug.lock);
Srivatsa S. Bhat075663d2012-10-08 16:28:20 -070083
84 if (WARN_ON(!cpu_hotplug.refcount))
85 cpu_hotplug.refcount++; /* try to fix things up */
86
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -070087 if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
88 wake_up_process(cpu_hotplug.active_writer);
Gautham R Shenoyd2219382008-01-25 21:08:01 +010089 mutex_unlock(&cpu_hotplug.lock);
90
Ashok Raja9d9baa2005-11-28 13:43:46 -080091}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +010092EXPORT_SYMBOL_GPL(put_online_cpus);
Ashok Raja9d9baa2005-11-28 13:43:46 -080093
Gautham R Shenoyd2219382008-01-25 21:08:01 +010094/*
95 * This ensures that the hotplug operation can begin only when the
96 * refcount goes to zero.
97 *
98 * Note that during a cpu-hotplug operation, the new readers, if any,
99 * will be blocked by the cpu_hotplug.lock
100 *
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700101 * Since cpu_hotplug_begin() is always called after invoking
102 * cpu_maps_update_begin(), we can be sure that only one writer is active.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100103 *
104 * Note that theoretically, there is a possibility of a livelock:
105 * - Refcount goes to zero, last reader wakes up the sleeping
106 * writer.
107 * - Last reader unlocks the cpu_hotplug.lock.
108 * - A new reader arrives at this moment, bumps up the refcount.
109 * - The writer acquires the cpu_hotplug.lock finds the refcount
110 * non zero and goes to sleep again.
111 *
112 * However, this is very difficult to achieve in practice since
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100113 * get_online_cpus() not an api which is called all that often.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100114 *
115 */
Toshi Kanib9d10be2013-08-12 09:45:53 -0600116void cpu_hotplug_begin(void)
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100117{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100118 cpu_hotplug.active_writer = current;
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700119
120 for (;;) {
121 mutex_lock(&cpu_hotplug.lock);
122 if (likely(!cpu_hotplug.refcount))
123 break;
124 __set_current_state(TASK_UNINTERRUPTIBLE);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100125 mutex_unlock(&cpu_hotplug.lock);
126 schedule();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100127 }
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100128}
129
Toshi Kanib9d10be2013-08-12 09:45:53 -0600130void cpu_hotplug_done(void)
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100131{
132 cpu_hotplug.active_writer = NULL;
133 mutex_unlock(&cpu_hotplug.lock);
134}
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700135
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700136/*
137 * Wait for currently running CPU hotplug operations to complete (if any) and
138 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
139 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
140 * hotplug path before performing hotplug operations. So acquiring that lock
141 * guarantees mutual exclusion from any currently running hotplug operations.
142 */
143void cpu_hotplug_disable(void)
144{
145 cpu_maps_update_begin();
146 cpu_hotplug_disabled = 1;
147 cpu_maps_update_done();
148}
149
150void cpu_hotplug_enable(void)
151{
152 cpu_maps_update_begin();
153 cpu_hotplug_disabled = 0;
154 cpu_maps_update_done();
155}
156
Toshi Kanib9d10be2013-08-12 09:45:53 -0600157#endif /* CONFIG_HOTPLUG_CPU */
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700158
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159/* Need to know about CPUs going up/down? */
Sam Ravnborgf7b16c12008-04-29 00:58:51 -0700160int __ref register_cpu_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161{
Neil Brownbd5349c2006-10-17 00:10:35 -0700162 int ret;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100163 cpu_maps_update_begin();
Neil Brownbd5349c2006-10-17 00:10:35 -0700164 ret = raw_notifier_chain_register(&cpu_chain, nb);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100165 cpu_maps_update_done();
Neil Brownbd5349c2006-10-17 00:10:35 -0700166 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167}
Chandra Seetharaman65edc682006-06-27 02:54:08 -0700168
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700169static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
170 int *nr_calls)
171{
Akinobu Mitae6bde732010-05-26 14:43:29 -0700172 int ret;
173
174 ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700175 nr_calls);
Akinobu Mitae6bde732010-05-26 14:43:29 -0700176
177 return notifier_to_errno(ret);
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700178}
179
180static int cpu_notify(unsigned long val, void *v)
181{
182 return __cpu_notify(val, v, -1, NULL);
183}
184
Linus Torvalds00b9b0a2010-05-27 10:32:08 -0700185#ifdef CONFIG_HOTPLUG_CPU
186
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700187static void cpu_notify_nofail(unsigned long val, void *v)
188{
Linus Torvalds00b9b0a2010-05-27 10:32:08 -0700189 BUG_ON(cpu_notify(val, v));
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700190}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191EXPORT_SYMBOL(register_cpu_notifier);
192
Sam Ravnborg96471552008-04-29 00:58:48 -0700193void __ref unregister_cpu_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100195 cpu_maps_update_begin();
Neil Brownbd5349c2006-10-17 00:10:35 -0700196 raw_notifier_chain_unregister(&cpu_chain, nb);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100197 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198}
199EXPORT_SYMBOL(unregister_cpu_notifier);
200
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700201/**
202 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
203 * @cpu: a CPU id
204 *
205 * This function walks all processes, finds a valid mm struct for each one and
206 * then clears a corresponding bit in mm's cpumask. While this all sounds
207 * trivial, there are various non-obvious corner cases, which this function
208 * tries to solve in a safe manner.
209 *
210 * Also note that the function uses a somewhat relaxed locking scheme, so it may
211 * be called only for an already offlined CPU.
212 */
Anton Vorontsovcb792952012-05-31 16:26:22 -0700213void clear_tasks_mm_cpumask(int cpu)
214{
215 struct task_struct *p;
216
217 /*
218 * This function is called after the cpu is taken down and marked
219 * offline, so its not like new tasks will ever get this cpu set in
220 * their mm mask. -- Peter Zijlstra
221 * Thus, we may use rcu_read_lock() here, instead of grabbing
222 * full-fledged tasklist_lock.
223 */
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700224 WARN_ON(cpu_online(cpu));
Anton Vorontsovcb792952012-05-31 16:26:22 -0700225 rcu_read_lock();
226 for_each_process(p) {
227 struct task_struct *t;
228
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700229 /*
230 * Main thread might exit, but other threads may still have
231 * a valid mm. Find one.
232 */
Anton Vorontsovcb792952012-05-31 16:26:22 -0700233 t = find_lock_task_mm(p);
234 if (!t)
235 continue;
236 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
237 task_unlock(t);
238 }
239 rcu_read_unlock();
240}
241
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242static inline void check_for_tasks(int cpu)
243{
244 struct task_struct *p;
Frederic Weisbecker6fac4822012-11-13 14:20:55 +0100245 cputime_t utime, stime;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246
247 write_lock_irq(&tasklist_lock);
248 for_each_process(p) {
Frederic Weisbecker6fac4822012-11-13 14:20:55 +0100249 task_cputime(p, &utime, &stime);
Peter Zijlstra11854242010-01-21 16:34:27 +0100250 if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
Frederic Weisbecker6fac4822012-11-13 14:20:55 +0100251 (utime || stime))
Frans Pop9d3cfc42010-01-25 14:56:34 +0100252 printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
253 "(state = %ld, flags = %x)\n",
254 p->comm, task_pid_nr(p), cpu,
255 p->state, p->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 }
257 write_unlock_irq(&tasklist_lock);
258}
259
Avi Kivitydb912f92007-05-24 12:23:10 +0300260struct take_cpu_down_param {
261 unsigned long mod;
262 void *hcpu;
263};
264
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265/* Take this CPU down. */
Sam Ravnborg514a20a2008-04-29 00:58:50 -0700266static int __ref take_cpu_down(void *_param)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267{
Avi Kivitydb912f92007-05-24 12:23:10 +0300268 struct take_cpu_down_param *param = _param;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 int err;
270
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 /* Ensure this CPU doesn't handle any more interrupts. */
272 err = __cpu_disable();
273 if (err < 0)
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700274 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700276 cpu_notify(CPU_DYING | param->mod, param->hcpu);
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000277 /* Park the stopper thread */
278 kthread_park(current);
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700279 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280}
281
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700282/* Requires cpu_add_remove_lock to be held */
Sam Ravnborg514a20a2008-04-29 00:58:50 -0700283static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284{
Heiko Carstense7407dc2007-05-09 02:34:04 -0700285 int err, nr_calls = 0;
Heiko Carstense7407dc2007-05-09 02:34:04 -0700286 void *hcpu = (void *)(long)cpu;
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700287 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
Avi Kivitydb912f92007-05-24 12:23:10 +0300288 struct take_cpu_down_param tcd_param = {
289 .mod = mod,
290 .hcpu = hcpu,
291 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700293 if (num_online_cpus() == 1)
294 return -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700296 if (!cpu_online(cpu))
297 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100299 cpu_hotplug_begin();
Michael Rodriguez4d519852011-03-22 16:34:07 -0700300
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700301 err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
Akinobu Mitae6bde732010-05-26 14:43:29 -0700302 if (err) {
Akinobu Mitaa0d8cdb2007-10-18 03:05:12 -0700303 nr_calls--;
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700304 __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 printk("%s: attempt to take down CPU %u failed\n",
Harvey Harrisonaf1f16d2008-04-30 00:55:08 -0700306 __func__, cpu);
Gautham R Shenoybaaca492007-05-09 02:34:03 -0700307 goto out_release;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 }
309
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200310 /*
311 * By now we've cleared cpu_active_mask, wait for all preempt-disabled
312 * and RCU users of this state to go away such that all new such users
313 * will observe it.
314 *
315 * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might
316 * not imply sync_sched(), so explicitly call both.
Michael wang106dd5a2013-11-13 11:10:56 +0800317 *
318 * Do sync before park smpboot threads to take care the rcu boost case.
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200319 */
320#ifdef CONFIG_PREEMPT
321 synchronize_sched();
322#endif
323 synchronize_rcu();
324
Michael wang106dd5a2013-11-13 11:10:56 +0800325 smpboot_park_threads(cpu);
326
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200327 /*
328 * So now all preempt/rcu users must observe !cpu_active().
329 */
330
Rusty Russelle0b582e2009-01-01 10:12:28 +1030331 err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
Rusty Russell04321582008-07-28 12:16:29 -0500332 if (err) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 /* CPU didn't die: tell everyone. Can't complain. */
Thomas Gleixnerf97f8f02012-07-16 10:42:36 +0000334 smpboot_unpark_threads(cpu);
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700335 cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
Oleg Nesterov6a1bdc12010-03-15 10:10:23 +0100336 goto out_release;
Satoru Takeuchi8fa1d7d2006-10-28 10:38:57 -0700337 }
Rusty Russell04321582008-07-28 12:16:29 -0500338 BUG_ON(cpu_online(cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100340 /*
341 * The migration_call() CPU_DYING callback will have removed all
342 * runnable tasks from the cpu, there's only the idle task left now
343 * that the migration thread is done doing the stop_machine thing.
Peter Zijlstra51a96c72010-11-19 20:37:53 +0100344 *
345 * Wait for the stop thread to go away.
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100346 */
Peter Zijlstra51a96c72010-11-19 20:37:53 +0100347 while (!idle_cpu(cpu))
348 cpu_relax();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349
350 /* This actually kills the CPU. */
351 __cpu_die(cpu);
352
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 /* CPU is completely dead: tell everyone. Too late to complain. */
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700354 cpu_notify_nofail(CPU_DEAD | mod, hcpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355
356 check_for_tasks(cpu);
357
Gautham R Shenoybaaca492007-05-09 02:34:03 -0700358out_release:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100359 cpu_hotplug_done();
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700360 if (!err)
361 cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700362 return err;
363}
364
Sam Ravnborg514a20a2008-04-29 00:58:50 -0700365int __ref cpu_down(unsigned int cpu)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700366{
Heiko Carstens9ea09af2008-12-22 12:36:30 +0100367 int err;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700368
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100369 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700370
Max Krasnyanskye761b772008-07-15 04:43:49 -0700371 if (cpu_hotplug_disabled) {
372 err = -EBUSY;
373 goto out;
374 }
375
Max Krasnyanskye761b772008-07-15 04:43:49 -0700376 err = _cpu_down(cpu, 0);
377
Max Krasnyanskye761b772008-07-15 04:43:49 -0700378out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100379 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 return err;
381}
Zhang Ruib62b8ef2008-04-29 02:35:56 -0400382EXPORT_SYMBOL(cpu_down);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383#endif /*CONFIG_HOTPLUG_CPU*/
384
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700385/* Requires cpu_add_remove_lock to be held */
Paul Gortmaker0db06282013-06-19 14:53:51 -0400386static int _cpu_up(unsigned int cpu, int tasks_frozen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387{
Gautham R Shenoybaaca492007-05-09 02:34:03 -0700388 int ret, nr_calls = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 void *hcpu = (void *)(long)cpu;
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700390 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700391 struct task_struct *idle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100393 cpu_hotplug_begin();
Thomas Gleixner38498a62012-04-20 13:05:44 +0000394
Yasuaki Ishimatsu5e5041f2012-10-23 01:30:54 +0200395 if (cpu_online(cpu) || !cpu_present(cpu)) {
396 ret = -EINVAL;
397 goto out;
398 }
399
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700400 idle = idle_thread_get(cpu);
401 if (IS_ERR(idle)) {
402 ret = PTR_ERR(idle);
Thomas Gleixner38498a62012-04-20 13:05:44 +0000403 goto out;
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700404 }
Thomas Gleixner38498a62012-04-20 13:05:44 +0000405
Thomas Gleixnerf97f8f02012-07-16 10:42:36 +0000406 ret = smpboot_create_threads(cpu);
407 if (ret)
408 goto out;
409
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700410 ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
Akinobu Mitae6bde732010-05-26 14:43:29 -0700411 if (ret) {
Akinobu Mitaa0d8cdb2007-10-18 03:05:12 -0700412 nr_calls--;
Michael Rodriguez4d519852011-03-22 16:34:07 -0700413 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
Harvey Harrisonaf1f16d2008-04-30 00:55:08 -0700414 __func__, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 goto out_notify;
416 }
417
418 /* Arch-specific enabling code. */
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700419 ret = __cpu_up(cpu, idle);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 if (ret != 0)
421 goto out_notify;
Eric Sesterhenn6978c702006-03-24 18:45:21 +0100422 BUG_ON(!cpu_online(cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423
Thomas Gleixnerf97f8f02012-07-16 10:42:36 +0000424 /* Wake the per cpu threads */
425 smpboot_unpark_threads(cpu);
426
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 /* Now call notifier in preparation. */
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700428 cpu_notify(CPU_ONLINE | mod, hcpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429
430out_notify:
431 if (ret != 0)
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700432 __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
Thomas Gleixner38498a62012-04-20 13:05:44 +0000433out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100434 cpu_hotplug_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700435
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 return ret;
437}
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700438
Paul Gortmaker0db06282013-06-19 14:53:51 -0400439int cpu_up(unsigned int cpu)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700440{
441 int err = 0;
minskey guocf234222010-05-24 14:32:41 -0700442
Rusty Russelle0b582e2009-01-01 10:12:28 +1030443 if (!cpu_possible(cpu)) {
KAMEZAWA Hiroyuki73e753a2007-10-18 23:40:47 -0700444 printk(KERN_ERR "can't online cpu %d because it is not "
445 "configured as may-hotadd at boot time\n", cpu);
Chen Gong87d5e0232010-03-05 13:42:38 -0800446#if defined(CONFIG_IA64)
KAMEZAWA Hiroyuki73e753a2007-10-18 23:40:47 -0700447 printk(KERN_ERR "please check additional_cpus= boot "
448 "parameter\n");
449#endif
450 return -EINVAL;
451 }
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700452
Toshi Kani01b0f192013-11-12 15:07:25 -0800453 err = try_online_node(cpu_to_node(cpu));
454 if (err)
455 return err;
minskey guocf234222010-05-24 14:32:41 -0700456
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100457 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700458
Max Krasnyanskye761b772008-07-15 04:43:49 -0700459 if (cpu_hotplug_disabled) {
460 err = -EBUSY;
461 goto out;
462 }
463
464 err = _cpu_up(cpu, 0);
465
Max Krasnyanskye761b772008-07-15 04:43:49 -0700466out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100467 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700468 return err;
469}
Paul E. McKenneya513f6b2011-12-11 21:54:45 -0800470EXPORT_SYMBOL_GPL(cpu_up);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700471
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -0700472#ifdef CONFIG_PM_SLEEP_SMP
Rusty Russelle0b582e2009-01-01 10:12:28 +1030473static cpumask_var_t frozen_cpus;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700474
475int disable_nonboot_cpus(void)
476{
Rafael J. Wysockie9a5f422010-05-27 22:16:22 +0200477 int cpu, first_cpu, error = 0;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700478
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100479 cpu_maps_update_begin();
Rusty Russelle0b582e2009-01-01 10:12:28 +1030480 first_cpu = cpumask_first(cpu_online_mask);
Xiaotian Feng9ee349a2009-12-16 18:04:32 +0100481 /*
482 * We take down all of the non-boot CPUs in one shot to avoid races
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700483 * with the userspace trying to use the CPU hotplug at the same time
484 */
Rusty Russelle0b582e2009-01-01 10:12:28 +1030485 cpumask_clear(frozen_cpus);
Peter Zijlstra6ad4c182009-11-25 13:31:39 +0100486
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700487 printk("Disabling non-boot CPUs ...\n");
488 for_each_online_cpu(cpu) {
489 if (cpu == first_cpu)
490 continue;
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700491 error = _cpu_down(cpu, 1);
Mike Travisfeae3202009-11-17 18:22:13 -0600492 if (!error)
Rusty Russelle0b582e2009-01-01 10:12:28 +1030493 cpumask_set_cpu(cpu, frozen_cpus);
Mike Travisfeae3202009-11-17 18:22:13 -0600494 else {
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700495 printk(KERN_ERR "Error taking CPU%d down: %d\n",
496 cpu, error);
497 break;
498 }
499 }
Joseph Cihula86886e52009-06-30 19:31:07 -0700500
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700501 if (!error) {
502 BUG_ON(num_online_cpus() > 1);
503 /* Make sure the CPUs won't be enabled by someone else */
504 cpu_hotplug_disabled = 1;
505 } else {
Ingo Molnare1d9fd22006-12-23 16:55:29 +0100506 printk(KERN_ERR "Non-boot CPUs are not disabled\n");
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700507 }
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100508 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700509 return error;
510}
511
Suresh Siddhad0af9ee2009-08-19 18:05:36 -0700512void __weak arch_enable_nonboot_cpus_begin(void)
513{
514}
515
516void __weak arch_enable_nonboot_cpus_end(void)
517{
518}
519
Sam Ravnborgfa7303e2008-02-08 04:21:55 -0800520void __ref enable_nonboot_cpus(void)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700521{
522 int cpu, error;
523
524 /* Allow everyone to use the CPU hotplug again */
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100525 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700526 cpu_hotplug_disabled = 0;
Rusty Russelle0b582e2009-01-01 10:12:28 +1030527 if (cpumask_empty(frozen_cpus))
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -0700528 goto out;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700529
Michael Rodriguez4d519852011-03-22 16:34:07 -0700530 printk(KERN_INFO "Enabling non-boot CPUs ...\n");
Suresh Siddhad0af9ee2009-08-19 18:05:36 -0700531
532 arch_enable_nonboot_cpus_begin();
533
Rusty Russelle0b582e2009-01-01 10:12:28 +1030534 for_each_cpu(cpu, frozen_cpus) {
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700535 error = _cpu_up(cpu, 1);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700536 if (!error) {
Michael Rodriguez4d519852011-03-22 16:34:07 -0700537 printk(KERN_INFO "CPU%d is up\n", cpu);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700538 continue;
539 }
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -0700540 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700541 }
Suresh Siddhad0af9ee2009-08-19 18:05:36 -0700542
543 arch_enable_nonboot_cpus_end();
544
Rusty Russelle0b582e2009-01-01 10:12:28 +1030545 cpumask_clear(frozen_cpus);
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -0700546out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100547 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700548}
Rusty Russelle0b582e2009-01-01 10:12:28 +1030549
Fenghua Yud7268a32011-11-15 21:59:31 +0100550static int __init alloc_frozen_cpus(void)
Rusty Russelle0b582e2009-01-01 10:12:28 +1030551{
552 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
553 return -ENOMEM;
554 return 0;
555}
556core_initcall(alloc_frozen_cpus);
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +0100557
558/*
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +0100559 * When callbacks for CPU hotplug notifications are being executed, we must
560 * ensure that the state of the system with respect to the tasks being frozen
561 * or not, as reported by the notification, remains unchanged *throughout the
562 * duration* of the execution of the callbacks.
563 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
564 *
565 * This synchronization is implemented by mutually excluding regular CPU
566 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
567 * Hibernate notifications.
568 */
569static int
570cpu_hotplug_pm_callback(struct notifier_block *nb,
571 unsigned long action, void *ptr)
572{
573 switch (action) {
574
575 case PM_SUSPEND_PREPARE:
576 case PM_HIBERNATION_PREPARE:
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700577 cpu_hotplug_disable();
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +0100578 break;
579
580 case PM_POST_SUSPEND:
581 case PM_POST_HIBERNATION:
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700582 cpu_hotplug_enable();
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +0100583 break;
584
585 default:
586 return NOTIFY_DONE;
587 }
588
589 return NOTIFY_OK;
590}
591
592
Fenghua Yud7268a32011-11-15 21:59:31 +0100593static int __init cpu_hotplug_pm_sync_init(void)
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +0100594{
Fenghua Yu6e32d472012-11-13 11:32:43 -0800595 /*
596 * cpu_hotplug_pm_callback has higher priority than x86
597 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
598 * to disable cpu hotplug to avoid cpu hotplug race.
599 */
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +0100600 pm_notifier(cpu_hotplug_pm_callback, 0);
601 return 0;
602}
603core_initcall(cpu_hotplug_pm_sync_init);
604
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -0700605#endif /* CONFIG_PM_SLEEP_SMP */
Max Krasnyansky68f4f1e2008-05-29 11:17:02 -0700606
Manfred Spraule545a612008-09-07 16:57:22 +0200607/**
608 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
609 * @cpu: cpu that just started
610 *
611 * This function calls the cpu_chain notifiers with CPU_STARTING.
612 * It must be called by the arch code on the new cpu, before the new cpu
613 * enables interrupts and before the "boot" cpu returns from __cpu_up().
614 */
Paul Gortmaker0db06282013-06-19 14:53:51 -0400615void notify_cpu_starting(unsigned int cpu)
Manfred Spraule545a612008-09-07 16:57:22 +0200616{
617 unsigned long val = CPU_STARTING;
618
619#ifdef CONFIG_PM_SLEEP_SMP
Rusty Russelle0b582e2009-01-01 10:12:28 +1030620 if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
Manfred Spraule545a612008-09-07 16:57:22 +0200621 val = CPU_STARTING_FROZEN;
622#endif /* CONFIG_PM_SLEEP_SMP */
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700623 cpu_notify(val, (void *)(long)cpu);
Manfred Spraule545a612008-09-07 16:57:22 +0200624}
625
Max Krasnyansky68f4f1e2008-05-29 11:17:02 -0700626#endif /* CONFIG_SMP */
Mike Travisb8d317d2008-07-24 18:21:29 -0700627
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700628/*
629 * cpu_bit_bitmap[] is a special, "compressed" data structure that
630 * represents all NR_CPUS bits binary values of 1<<nr.
631 *
Rusty Russelle0b582e2009-01-01 10:12:28 +1030632 * It is used by cpumask_of() to get a constant address to a CPU
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700633 * mask value that has a single bit set only.
634 */
Mike Travisb8d317d2008-07-24 18:21:29 -0700635
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700636/* cpu_bit_bitmap[0] is empty - so we can back into it */
Michael Rodriguez4d519852011-03-22 16:34:07 -0700637#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700638#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
639#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
640#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
Mike Travisb8d317d2008-07-24 18:21:29 -0700641
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700642const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
Mike Travisb8d317d2008-07-24 18:21:29 -0700643
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700644 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
645 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
646#if BITS_PER_LONG > 32
647 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
648 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
Mike Travisb8d317d2008-07-24 18:21:29 -0700649#endif
650};
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700651EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100652
653const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
654EXPORT_SYMBOL(cpu_all_bits);
Rusty Russellb3199c02008-12-30 09:05:14 +1030655
656#ifdef CONFIG_INIT_ALL_POSSIBLE
657static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
658 = CPU_BITS_ALL;
659#else
660static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
661#endif
662const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
663EXPORT_SYMBOL(cpu_possible_mask);
664
665static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
666const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
667EXPORT_SYMBOL(cpu_online_mask);
668
669static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
670const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
671EXPORT_SYMBOL(cpu_present_mask);
672
673static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
674const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
675EXPORT_SYMBOL(cpu_active_mask);
Rusty Russell3fa41522008-12-30 09:05:16 +1030676
677void set_cpu_possible(unsigned int cpu, bool possible)
678{
679 if (possible)
680 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
681 else
682 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
683}
684
685void set_cpu_present(unsigned int cpu, bool present)
686{
687 if (present)
688 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
689 else
690 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
691}
692
693void set_cpu_online(unsigned int cpu, bool online)
694{
695 if (online)
696 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
697 else
698 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
699}
700
701void set_cpu_active(unsigned int cpu, bool active)
702{
703 if (active)
704 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
705 else
706 cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
707}
708
709void init_cpu_present(const struct cpumask *src)
710{
711 cpumask_copy(to_cpumask(cpu_present_bits), src);
712}
713
714void init_cpu_possible(const struct cpumask *src)
715{
716 cpumask_copy(to_cpumask(cpu_possible_bits), src);
717}
718
719void init_cpu_online(const struct cpumask *src)
720{
721 cpumask_copy(to_cpumask(cpu_online_bits), src);
722}