Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* CPU control. |
| 2 | * (C) 2001, 2002, 2003, 2004 Rusty Russell |
| 3 | * |
| 4 | * This code is licenced under the GPL. |
| 5 | */ |
| 6 | #include <linux/proc_fs.h> |
| 7 | #include <linux/smp.h> |
| 8 | #include <linux/init.h> |
| 9 | #include <linux/notifier.h> |
Ingo Molnar | 3f07c01 | 2017-02-08 18:51:30 +0100 | [diff] [blame] | 10 | #include <linux/sched/signal.h> |
Ingo Molnar | ef8bd77 | 2017-02-08 18:51:36 +0100 | [diff] [blame] | 11 | #include <linux/sched/hotplug.h> |
Nicholas Piggin | 9ca12ac | 2019-04-11 13:34:46 +1000 | [diff] [blame] | 12 | #include <linux/sched/isolation.h> |
Ingo Molnar | 2993002 | 2017-02-08 18:51:36 +0100 | [diff] [blame] | 13 | #include <linux/sched/task.h> |
Thomas Gleixner | a74cfff | 2018-11-25 19:33:39 +0100 | [diff] [blame] | 14 | #include <linux/sched/smt.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <linux/unistd.h> |
| 16 | #include <linux/cpu.h> |
Anton Vorontsov | cb79295 | 2012-05-31 16:26:22 -0700 | [diff] [blame] | 17 | #include <linux/oom.h> |
| 18 | #include <linux/rcupdate.h> |
Paul Gortmaker | 9984de1 | 2011-05-23 14:51:41 -0400 | [diff] [blame] | 19 | #include <linux/export.h> |
Anton Vorontsov | e4cc2f8 | 2012-05-31 16:26:26 -0700 | [diff] [blame] | 20 | #include <linux/bug.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #include <linux/kthread.h> |
| 22 | #include <linux/stop_machine.h> |
Ingo Molnar | 81615b62 | 2006-06-26 00:24:32 -0700 | [diff] [blame] | 23 | #include <linux/mutex.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 24 | #include <linux/gfp.h> |
Srivatsa S. Bhat | 79cfbdf | 2011-11-03 00:59:25 +0100 | [diff] [blame] | 25 | #include <linux/suspend.h> |
Gautham R. Shenoy | a19423b | 2014-03-11 02:04:03 +0530 | [diff] [blame] | 26 | #include <linux/lockdep.h> |
Preeti U Murthy | 345527b | 2015-03-30 14:59:19 +0530 | [diff] [blame] | 27 | #include <linux/tick.h> |
Thomas Gleixner | a899418 | 2015-07-05 17:12:30 +0000 | [diff] [blame] | 28 | #include <linux/irq.h> |
Thomas Gleixner | 941154b | 2017-09-12 21:37:04 +0200 | [diff] [blame] | 29 | #include <linux/nmi.h> |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 30 | #include <linux/smpboot.h> |
Richard Weinberger | e6d4989 | 2016-08-18 14:57:17 +0200 | [diff] [blame] | 31 | #include <linux/relay.h> |
Sebastian Andrzej Siewior | 6731d4f | 2016-08-23 14:53:19 +0200 | [diff] [blame] | 32 | #include <linux/slab.h> |
Thomas Gleixner | fc8dffd | 2017-05-24 10:15:40 +0200 | [diff] [blame] | 33 | #include <linux/percpu-rwsem.h> |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 34 | |
Todd E Brandt | bb3632c | 2014-06-06 05:40:17 -0700 | [diff] [blame] | 35 | #include <trace/events/power.h> |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 36 | #define CREATE_TRACE_POINTS |
| 37 | #include <trace/events/cpuhp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | |
Thomas Gleixner | 38498a6 | 2012-04-20 13:05:44 +0000 | [diff] [blame] | 39 | #include "smpboot.h" |
| 40 | |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 41 | /** |
| 42 | * cpuhp_cpu_state - Per cpu hotplug state storage |
| 43 | * @state: The current cpu state |
| 44 | * @target: The target state |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 45 | * @thread: Pointer to the hotplug thread |
| 46 | * @should_run: Thread should execute |
Sebastian Andrzej Siewior | 3b9d6da | 2016-04-08 14:40:15 +0200 | [diff] [blame] | 47 | * @rollback: Perform a rollback |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 48 | * @single: Single callback invocation |
| 49 | * @bringup: Single callback bringup or teardown selector |
| 50 | * @cb_state: The state for a single callback (install/uninstall) |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 51 | * @result: Result of the operation |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 52 | * @done_up: Signal completion to the issuer of the task for cpu-up |
| 53 | * @done_down: Signal completion to the issuer of the task for cpu-down |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 54 | */ |
| 55 | struct cpuhp_cpu_state { |
| 56 | enum cpuhp_state state; |
| 57 | enum cpuhp_state target; |
Peter Zijlstra | 1db4948 | 2017-09-20 19:00:21 +0200 | [diff] [blame] | 58 | enum cpuhp_state fail; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 59 | #ifdef CONFIG_SMP |
| 60 | struct task_struct *thread; |
| 61 | bool should_run; |
Sebastian Andrzej Siewior | 3b9d6da | 2016-04-08 14:40:15 +0200 | [diff] [blame] | 62 | bool rollback; |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 63 | bool single; |
| 64 | bool bringup; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 65 | struct hlist_node *node; |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 66 | struct hlist_node *last; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 67 | enum cpuhp_state cb_state; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 68 | int result; |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 69 | struct completion done_up; |
| 70 | struct completion done_down; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 71 | #endif |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 72 | }; |
| 73 | |
Peter Zijlstra | 1db4948 | 2017-09-20 19:00:21 +0200 | [diff] [blame] | 74 | static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { |
| 75 | .fail = CPUHP_INVALID, |
| 76 | }; |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 77 | |
Thomas Gleixner | e797bda | 2019-07-22 20:47:16 +0200 | [diff] [blame] | 78 | #ifdef CONFIG_SMP |
| 79 | cpumask_t cpus_booted_once_mask; |
| 80 | #endif |
| 81 | |
Thomas Gleixner | 49dfe2a | 2017-05-24 10:15:43 +0200 | [diff] [blame] | 82 | #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP) |
Peter Zijlstra | 5f4b55e | 2017-09-20 19:00:20 +0200 | [diff] [blame] | 83 | static struct lockdep_map cpuhp_state_up_map = |
| 84 | STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map); |
| 85 | static struct lockdep_map cpuhp_state_down_map = |
| 86 | STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map); |
| 87 | |
| 88 | |
Mathieu Malaterre | 76dc6c0 | 2017-12-26 15:08:53 +0100 | [diff] [blame] | 89 | static inline void cpuhp_lock_acquire(bool bringup) |
Peter Zijlstra | 5f4b55e | 2017-09-20 19:00:20 +0200 | [diff] [blame] | 90 | { |
| 91 | lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map); |
| 92 | } |
| 93 | |
Mathieu Malaterre | 76dc6c0 | 2017-12-26 15:08:53 +0100 | [diff] [blame] | 94 | static inline void cpuhp_lock_release(bool bringup) |
Peter Zijlstra | 5f4b55e | 2017-09-20 19:00:20 +0200 | [diff] [blame] | 95 | { |
| 96 | lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map); |
| 97 | } |
| 98 | #else |
| 99 | |
Mathieu Malaterre | 76dc6c0 | 2017-12-26 15:08:53 +0100 | [diff] [blame] | 100 | static inline void cpuhp_lock_acquire(bool bringup) { } |
| 101 | static inline void cpuhp_lock_release(bool bringup) { } |
Peter Zijlstra | 5f4b55e | 2017-09-20 19:00:20 +0200 | [diff] [blame] | 102 | |
Thomas Gleixner | 49dfe2a | 2017-05-24 10:15:43 +0200 | [diff] [blame] | 103 | #endif |
| 104 | |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 105 | /** |
| 106 | * cpuhp_step - Hotplug state machine step |
| 107 | * @name: Name of the step |
| 108 | * @startup: Startup function of the step |
| 109 | * @teardown: Teardown function of the step |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 110 | * @cant_stop: Bringup/teardown can't be stopped at this step |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 111 | */ |
| 112 | struct cpuhp_step { |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 113 | const char *name; |
| 114 | union { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 115 | int (*single)(unsigned int cpu); |
| 116 | int (*multi)(unsigned int cpu, |
| 117 | struct hlist_node *node); |
| 118 | } startup; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 119 | union { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 120 | int (*single)(unsigned int cpu); |
| 121 | int (*multi)(unsigned int cpu, |
| 122 | struct hlist_node *node); |
| 123 | } teardown; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 124 | struct hlist_head list; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 125 | bool cant_stop; |
| 126 | bool multi_instance; |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 127 | }; |
| 128 | |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 129 | static DEFINE_MUTEX(cpuhp_state_mutex); |
Lai Jiangshan | 17a2f1c | 2017-12-01 21:50:05 +0800 | [diff] [blame] | 130 | static struct cpuhp_step cpuhp_hp_states[]; |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 131 | |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 132 | static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) |
| 133 | { |
Lai Jiangshan | 17a2f1c | 2017-12-01 21:50:05 +0800 | [diff] [blame] | 134 | return cpuhp_hp_states + state; |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 135 | } |
| 136 | |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 137 | /** |
| 138 | * cpuhp_invoke_callback _ Invoke the callbacks for a given state |
| 139 | * @cpu: The cpu for which the callback should be invoked |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 140 | * @state: The state to do callbacks for |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 141 | * @bringup: True if the bringup callback should be invoked |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 142 | * @node: For multi-instance, do a single entry callback for install/remove |
| 143 | * @lastp: For multi-instance rollback, remember how far we got |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 144 | * |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 145 | * Called from cpu hotplug and from the state register machinery. |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 146 | */ |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 147 | static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 148 | bool bringup, struct hlist_node *node, |
| 149 | struct hlist_node **lastp) |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 150 | { |
| 151 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 152 | struct cpuhp_step *step = cpuhp_get_step(state); |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 153 | int (*cbm)(unsigned int cpu, struct hlist_node *node); |
| 154 | int (*cb)(unsigned int cpu); |
| 155 | int ret, cnt; |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 156 | |
Peter Zijlstra | 1db4948 | 2017-09-20 19:00:21 +0200 | [diff] [blame] | 157 | if (st->fail == state) { |
| 158 | st->fail = CPUHP_INVALID; |
| 159 | |
| 160 | if (!(bringup ? step->startup.single : step->teardown.single)) |
| 161 | return 0; |
| 162 | |
| 163 | return -EAGAIN; |
| 164 | } |
| 165 | |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 166 | if (!step->multi_instance) { |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 167 | WARN_ON_ONCE(lastp && *lastp); |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 168 | cb = bringup ? step->startup.single : step->teardown.single; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 169 | if (!cb) |
| 170 | return 0; |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 171 | trace_cpuhp_enter(cpu, st->target, state, cb); |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 172 | ret = cb(cpu); |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 173 | trace_cpuhp_exit(cpu, st->state, state, ret); |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 174 | return ret; |
| 175 | } |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 176 | cbm = bringup ? step->startup.multi : step->teardown.multi; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 177 | if (!cbm) |
| 178 | return 0; |
| 179 | |
| 180 | /* Single invocation for instance add/remove */ |
| 181 | if (node) { |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 182 | WARN_ON_ONCE(lastp && *lastp); |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 183 | trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); |
| 184 | ret = cbm(cpu, node); |
| 185 | trace_cpuhp_exit(cpu, st->state, state, ret); |
| 186 | return ret; |
| 187 | } |
| 188 | |
| 189 | /* State transition. Invoke on all instances */ |
| 190 | cnt = 0; |
| 191 | hlist_for_each(node, &step->list) { |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 192 | if (lastp && node == *lastp) |
| 193 | break; |
| 194 | |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 195 | trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); |
| 196 | ret = cbm(cpu, node); |
| 197 | trace_cpuhp_exit(cpu, st->state, state, ret); |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 198 | if (ret) { |
| 199 | if (!lastp) |
| 200 | goto err; |
| 201 | |
| 202 | *lastp = node; |
| 203 | return ret; |
| 204 | } |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 205 | cnt++; |
| 206 | } |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 207 | if (lastp) |
| 208 | *lastp = NULL; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 209 | return 0; |
| 210 | err: |
| 211 | /* Rollback the instances if one failed */ |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 212 | cbm = !bringup ? step->startup.multi : step->teardown.multi; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 213 | if (!cbm) |
| 214 | return ret; |
| 215 | |
| 216 | hlist_for_each(node, &step->list) { |
| 217 | if (!cnt--) |
| 218 | break; |
Peter Zijlstra | 724a868 | 2017-09-20 19:00:18 +0200 | [diff] [blame] | 219 | |
| 220 | trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); |
| 221 | ret = cbm(cpu, node); |
| 222 | trace_cpuhp_exit(cpu, st->state, state, ret); |
| 223 | /* |
| 224 | * Rollback must not fail, |
| 225 | */ |
| 226 | WARN_ON_ONCE(ret); |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 227 | } |
| 228 | return ret; |
| 229 | } |
| 230 | |
Rusty Russell | 98a79d6 | 2008-12-13 21:19:41 +1030 | [diff] [blame] | 231 | #ifdef CONFIG_SMP |
Arnd Bergmann | fcb3029 | 2018-03-15 16:38:04 +0100 | [diff] [blame] | 232 | static bool cpuhp_is_ap_state(enum cpuhp_state state) |
| 233 | { |
| 234 | /* |
| 235 | * The extra check for CPUHP_TEARDOWN_CPU is only for documentation |
| 236 | * purposes as that state is handled explicitly in cpu_down. |
| 237 | */ |
| 238 | return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; |
| 239 | } |
| 240 | |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 241 | static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup) |
| 242 | { |
| 243 | struct completion *done = bringup ? &st->done_up : &st->done_down; |
| 244 | wait_for_completion(done); |
| 245 | } |
| 246 | |
| 247 | static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup) |
| 248 | { |
| 249 | struct completion *done = bringup ? &st->done_up : &st->done_down; |
| 250 | complete(done); |
| 251 | } |
| 252 | |
| 253 | /* |
| 254 | * The former STARTING/DYING states, ran with IRQs disabled and must not fail. |
| 255 | */ |
| 256 | static bool cpuhp_is_atomic_state(enum cpuhp_state state) |
| 257 | { |
| 258 | return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE; |
| 259 | } |
| 260 | |
Rusty Russell | b3199c0 | 2008-12-30 09:05:14 +1030 | [diff] [blame] | 261 | /* Serializes the updates to cpu_online_mask, cpu_present_mask */ |
Linus Torvalds | aa95387 | 2006-07-23 12:12:16 -0700 | [diff] [blame] | 262 | static DEFINE_MUTEX(cpu_add_remove_lock); |
Thomas Gleixner | 090e77c | 2016-02-26 18:43:23 +0000 | [diff] [blame] | 263 | bool cpuhp_tasks_frozen; |
| 264 | EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 265 | |
Lai Jiangshan | 79a6cde | 2010-05-26 14:43:36 -0700 | [diff] [blame] | 266 | /* |
Srivatsa S. Bhat | 93ae4f9 | 2014-03-11 02:04:14 +0530 | [diff] [blame] | 267 | * The following two APIs (cpu_maps_update_begin/done) must be used when |
| 268 | * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. |
Lai Jiangshan | 79a6cde | 2010-05-26 14:43:36 -0700 | [diff] [blame] | 269 | */ |
| 270 | void cpu_maps_update_begin(void) |
| 271 | { |
| 272 | mutex_lock(&cpu_add_remove_lock); |
| 273 | } |
| 274 | |
| 275 | void cpu_maps_update_done(void) |
| 276 | { |
| 277 | mutex_unlock(&cpu_add_remove_lock); |
| 278 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 279 | |
Thomas Gleixner | fc8dffd | 2017-05-24 10:15:40 +0200 | [diff] [blame] | 280 | /* |
| 281 | * If set, cpu_up and cpu_down will return -EBUSY and do nothing. |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 282 | * Should always be manipulated under cpu_add_remove_lock |
| 283 | */ |
| 284 | static int cpu_hotplug_disabled; |
| 285 | |
Lai Jiangshan | 79a6cde | 2010-05-26 14:43:36 -0700 | [diff] [blame] | 286 | #ifdef CONFIG_HOTPLUG_CPU |
| 287 | |
Thomas Gleixner | fc8dffd | 2017-05-24 10:15:40 +0200 | [diff] [blame] | 288 | DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); |
Gautham R. Shenoy | a19423b | 2014-03-11 02:04:03 +0530 | [diff] [blame] | 289 | |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 290 | void cpus_read_lock(void) |
Ashok Raj | a9d9baa | 2005-11-28 13:43:46 -0800 | [diff] [blame] | 291 | { |
Thomas Gleixner | fc8dffd | 2017-05-24 10:15:40 +0200 | [diff] [blame] | 292 | percpu_down_read(&cpu_hotplug_lock); |
Ashok Raj | a9d9baa | 2005-11-28 13:43:46 -0800 | [diff] [blame] | 293 | } |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 294 | EXPORT_SYMBOL_GPL(cpus_read_lock); |
Ashok Raj | 90d45d1 | 2005-11-08 21:34:24 -0800 | [diff] [blame] | 295 | |
Waiman Long | 6f4ceee | 2018-07-24 14:26:04 -0400 | [diff] [blame] | 296 | int cpus_read_trylock(void) |
| 297 | { |
| 298 | return percpu_down_read_trylock(&cpu_hotplug_lock); |
| 299 | } |
| 300 | EXPORT_SYMBOL_GPL(cpus_read_trylock); |
| 301 | |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 302 | void cpus_read_unlock(void) |
Ashok Raj | a9d9baa | 2005-11-28 13:43:46 -0800 | [diff] [blame] | 303 | { |
Thomas Gleixner | fc8dffd | 2017-05-24 10:15:40 +0200 | [diff] [blame] | 304 | percpu_up_read(&cpu_hotplug_lock); |
Ashok Raj | a9d9baa | 2005-11-28 13:43:46 -0800 | [diff] [blame] | 305 | } |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 306 | EXPORT_SYMBOL_GPL(cpus_read_unlock); |
Ashok Raj | a9d9baa | 2005-11-28 13:43:46 -0800 | [diff] [blame] | 307 | |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 308 | void cpus_write_lock(void) |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 309 | { |
Thomas Gleixner | fc8dffd | 2017-05-24 10:15:40 +0200 | [diff] [blame] | 310 | percpu_down_write(&cpu_hotplug_lock); |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 311 | } |
| 312 | |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 313 | void cpus_write_unlock(void) |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 314 | { |
Thomas Gleixner | fc8dffd | 2017-05-24 10:15:40 +0200 | [diff] [blame] | 315 | percpu_up_write(&cpu_hotplug_lock); |
| 316 | } |
| 317 | |
| 318 | void lockdep_assert_cpus_held(void) |
| 319 | { |
Valentin Schneider | ce48c457 | 2018-12-19 18:23:15 +0000 | [diff] [blame] | 320 | /* |
| 321 | * We can't have hotplug operations before userspace starts running, |
| 322 | * and some init codepaths will knowingly not take the hotplug lock. |
| 323 | * This is all valid, so mute lockdep until it makes sense to report |
| 324 | * unheld locks. |
| 325 | */ |
| 326 | if (system_state < SYSTEM_RUNNING) |
| 327 | return; |
| 328 | |
Thomas Gleixner | fc8dffd | 2017-05-24 10:15:40 +0200 | [diff] [blame] | 329 | percpu_rwsem_assert_held(&cpu_hotplug_lock); |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 330 | } |
Lai Jiangshan | 79a6cde | 2010-05-26 14:43:36 -0700 | [diff] [blame] | 331 | |
Peter Zijlstra | cb92173 | 2018-09-11 11:51:27 +0200 | [diff] [blame] | 332 | static void lockdep_acquire_cpus_lock(void) |
| 333 | { |
| 334 | rwsem_acquire(&cpu_hotplug_lock.rw_sem.dep_map, 0, 0, _THIS_IP_); |
| 335 | } |
| 336 | |
| 337 | static void lockdep_release_cpus_lock(void) |
| 338 | { |
| 339 | rwsem_release(&cpu_hotplug_lock.rw_sem.dep_map, 1, _THIS_IP_); |
| 340 | } |
| 341 | |
Srivatsa S. Bhat | 16e53db | 2013-06-12 14:04:36 -0700 | [diff] [blame] | 342 | /* |
| 343 | * Wait for currently running CPU hotplug operations to complete (if any) and |
| 344 | * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects |
| 345 | * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the |
| 346 | * hotplug path before performing hotplug operations. So acquiring that lock |
| 347 | * guarantees mutual exclusion from any currently running hotplug operations. |
| 348 | */ |
| 349 | void cpu_hotplug_disable(void) |
| 350 | { |
| 351 | cpu_maps_update_begin(); |
Vitaly Kuznetsov | 89af7ba | 2015-08-05 00:52:46 -0700 | [diff] [blame] | 352 | cpu_hotplug_disabled++; |
Srivatsa S. Bhat | 16e53db | 2013-06-12 14:04:36 -0700 | [diff] [blame] | 353 | cpu_maps_update_done(); |
| 354 | } |
Vitaly Kuznetsov | 32145c4 | 2015-08-05 00:52:47 -0700 | [diff] [blame] | 355 | EXPORT_SYMBOL_GPL(cpu_hotplug_disable); |
Srivatsa S. Bhat | 16e53db | 2013-06-12 14:04:36 -0700 | [diff] [blame] | 356 | |
Lianwei Wang | 01b4115 | 2016-06-09 23:43:28 -0700 | [diff] [blame] | 357 | static void __cpu_hotplug_enable(void) |
| 358 | { |
| 359 | if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n")) |
| 360 | return; |
| 361 | cpu_hotplug_disabled--; |
| 362 | } |
| 363 | |
Srivatsa S. Bhat | 16e53db | 2013-06-12 14:04:36 -0700 | [diff] [blame] | 364 | void cpu_hotplug_enable(void) |
| 365 | { |
| 366 | cpu_maps_update_begin(); |
Lianwei Wang | 01b4115 | 2016-06-09 23:43:28 -0700 | [diff] [blame] | 367 | __cpu_hotplug_enable(); |
Srivatsa S. Bhat | 16e53db | 2013-06-12 14:04:36 -0700 | [diff] [blame] | 368 | cpu_maps_update_done(); |
| 369 | } |
Vitaly Kuznetsov | 32145c4 | 2015-08-05 00:52:47 -0700 | [diff] [blame] | 370 | EXPORT_SYMBOL_GPL(cpu_hotplug_enable); |
Peter Zijlstra | cb92173 | 2018-09-11 11:51:27 +0200 | [diff] [blame] | 371 | |
| 372 | #else |
| 373 | |
| 374 | static void lockdep_acquire_cpus_lock(void) |
| 375 | { |
| 376 | } |
| 377 | |
| 378 | static void lockdep_release_cpus_lock(void) |
| 379 | { |
| 380 | } |
| 381 | |
Toshi Kani | b9d10be | 2013-08-12 09:45:53 -0600 | [diff] [blame] | 382 | #endif /* CONFIG_HOTPLUG_CPU */ |
Lai Jiangshan | 79a6cde | 2010-05-26 14:43:36 -0700 | [diff] [blame] | 383 | |
Thomas Gleixner | a74cfff | 2018-11-25 19:33:39 +0100 | [diff] [blame] | 384 | /* |
| 385 | * Architectures that need SMT-specific errata handling during SMT hotplug |
| 386 | * should override this. |
| 387 | */ |
| 388 | void __weak arch_smt_update(void) { } |
| 389 | |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 390 | #ifdef CONFIG_HOTPLUG_SMT |
| 391 | enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED; |
Thomas Gleixner | bc2d8d26 | 2018-08-07 08:19:57 +0200 | [diff] [blame] | 392 | |
Jiri Kosina | 8e1b706 | 2018-07-13 16:23:23 +0200 | [diff] [blame] | 393 | void __init cpu_smt_disable(bool force) |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 394 | { |
Vitaly Kuznetsov | e1572f1 | 2019-09-16 18:22:56 +0200 | [diff] [blame] | 395 | if (!cpu_smt_possible()) |
Jiri Kosina | 8e1b706 | 2018-07-13 16:23:23 +0200 | [diff] [blame] | 396 | return; |
| 397 | |
| 398 | if (force) { |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 399 | pr_info("SMT: Force disabled\n"); |
| 400 | cpu_smt_control = CPU_SMT_FORCE_DISABLED; |
Jiri Kosina | 8e1b706 | 2018-07-13 16:23:23 +0200 | [diff] [blame] | 401 | } else { |
Borislav Petkov | d0e7d14 | 2018-10-04 19:22:27 +0200 | [diff] [blame] | 402 | pr_info("SMT: disabled\n"); |
Jiri Kosina | 8e1b706 | 2018-07-13 16:23:23 +0200 | [diff] [blame] | 403 | cpu_smt_control = CPU_SMT_DISABLED; |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 404 | } |
Jiri Kosina | 8e1b706 | 2018-07-13 16:23:23 +0200 | [diff] [blame] | 405 | } |
| 406 | |
Thomas Gleixner | fee0aed | 2018-07-13 16:23:24 +0200 | [diff] [blame] | 407 | /* |
| 408 | * The decision whether SMT is supported can only be done after the full |
Josh Poimboeuf | b284909a | 2019-01-30 07:13:58 -0600 | [diff] [blame] | 409 | * CPU identification. Called from architecture code. |
Thomas Gleixner | fee0aed | 2018-07-13 16:23:24 +0200 | [diff] [blame] | 410 | */ |
| 411 | void __init cpu_smt_check_topology(void) |
| 412 | { |
Josh Poimboeuf | b284909a | 2019-01-30 07:13:58 -0600 | [diff] [blame] | 413 | if (!topology_smt_supported()) |
Thomas Gleixner | fee0aed | 2018-07-13 16:23:24 +0200 | [diff] [blame] | 414 | cpu_smt_control = CPU_SMT_NOT_SUPPORTED; |
| 415 | } |
| 416 | |
Jiri Kosina | 8e1b706 | 2018-07-13 16:23:23 +0200 | [diff] [blame] | 417 | static int __init smt_cmdline_disable(char *str) |
| 418 | { |
| 419 | cpu_smt_disable(str && !strcmp(str, "force")); |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 420 | return 0; |
| 421 | } |
| 422 | early_param("nosmt", smt_cmdline_disable); |
| 423 | |
| 424 | static inline bool cpu_smt_allowed(unsigned int cpu) |
| 425 | { |
Josh Poimboeuf | b284909a | 2019-01-30 07:13:58 -0600 | [diff] [blame] | 426 | if (cpu_smt_control == CPU_SMT_ENABLED) |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 427 | return true; |
| 428 | |
Josh Poimboeuf | b284909a | 2019-01-30 07:13:58 -0600 | [diff] [blame] | 429 | if (topology_is_primary_thread(cpu)) |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 430 | return true; |
| 431 | |
| 432 | /* |
| 433 | * On x86 it's required to boot all logical CPUs at least once so |
| 434 | * that the init code can get a chance to set CR4.MCE on each |
| 435 | * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any |
| 436 | * core will shutdown the machine. |
| 437 | */ |
Thomas Gleixner | e797bda | 2019-07-22 20:47:16 +0200 | [diff] [blame] | 438 | return !cpumask_test_cpu(cpu, &cpus_booted_once_mask); |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 439 | } |
Vitaly Kuznetsov | e1572f1 | 2019-09-16 18:22:56 +0200 | [diff] [blame] | 440 | |
| 441 | /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */ |
| 442 | bool cpu_smt_possible(void) |
| 443 | { |
| 444 | return cpu_smt_control != CPU_SMT_FORCE_DISABLED && |
| 445 | cpu_smt_control != CPU_SMT_NOT_SUPPORTED; |
| 446 | } |
| 447 | EXPORT_SYMBOL_GPL(cpu_smt_possible); |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 448 | #else |
| 449 | static inline bool cpu_smt_allowed(unsigned int cpu) { return true; } |
| 450 | #endif |
| 451 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 452 | static inline enum cpuhp_state |
| 453 | cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target) |
| 454 | { |
| 455 | enum cpuhp_state prev_state = st->state; |
| 456 | |
| 457 | st->rollback = false; |
| 458 | st->last = NULL; |
| 459 | |
| 460 | st->target = target; |
| 461 | st->single = false; |
| 462 | st->bringup = st->state < target; |
| 463 | |
| 464 | return prev_state; |
| 465 | } |
| 466 | |
| 467 | static inline void |
| 468 | cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state) |
| 469 | { |
| 470 | st->rollback = true; |
| 471 | |
| 472 | /* |
| 473 | * If we have st->last we need to undo partial multi_instance of this |
| 474 | * state first. Otherwise start undo at the previous state. |
| 475 | */ |
| 476 | if (!st->last) { |
| 477 | if (st->bringup) |
| 478 | st->state--; |
| 479 | else |
| 480 | st->state++; |
| 481 | } |
| 482 | |
| 483 | st->target = prev_state; |
| 484 | st->bringup = !st->bringup; |
| 485 | } |
| 486 | |
| 487 | /* Regular hotplug invocation of the AP hotplug thread */ |
| 488 | static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st) |
| 489 | { |
| 490 | if (!st->single && st->state == st->target) |
| 491 | return; |
| 492 | |
| 493 | st->result = 0; |
| 494 | /* |
| 495 | * Make sure the above stores are visible before should_run becomes |
| 496 | * true. Paired with the mb() above in cpuhp_thread_fun() |
| 497 | */ |
| 498 | smp_mb(); |
| 499 | st->should_run = true; |
| 500 | wake_up_process(st->thread); |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 501 | wait_for_ap_thread(st, st->bringup); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 502 | } |
| 503 | |
| 504 | static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target) |
| 505 | { |
| 506 | enum cpuhp_state prev_state; |
| 507 | int ret; |
| 508 | |
| 509 | prev_state = cpuhp_set_state(st, target); |
| 510 | __cpuhp_kick_ap(st); |
| 511 | if ((ret = st->result)) { |
| 512 | cpuhp_reset_state(st, prev_state); |
| 513 | __cpuhp_kick_ap(st); |
| 514 | } |
| 515 | |
| 516 | return ret; |
| 517 | } |
Thomas Gleixner | 9cd4f1a | 2017-07-04 22:20:23 +0200 | [diff] [blame] | 518 | |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 519 | static int bringup_wait_for_ap(unsigned int cpu) |
| 520 | { |
| 521 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 522 | |
Thomas Gleixner | 9cd4f1a | 2017-07-04 22:20:23 +0200 | [diff] [blame] | 523 | /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */ |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 524 | wait_for_ap_thread(st, true); |
Thomas Gleixner | dea1d0f | 2017-07-11 22:06:24 +0200 | [diff] [blame] | 525 | if (WARN_ON_ONCE((!cpu_online(cpu)))) |
| 526 | return -ECANCELED; |
Thomas Gleixner | 9cd4f1a | 2017-07-04 22:20:23 +0200 | [diff] [blame] | 527 | |
| 528 | /* Unpark the stopper thread and the hotplug thread of the target cpu */ |
| 529 | stop_machine_unpark(cpu); |
| 530 | kthread_unpark(st->thread); |
| 531 | |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 532 | /* |
| 533 | * SMT soft disabling on X86 requires to bring the CPU out of the |
| 534 | * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The |
Jiri Kosina | f560201 | 2019-05-28 21:31:49 +0200 | [diff] [blame] | 535 | * CPU marked itself as booted_once in notify_cpu_starting() so the |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 536 | * cpu_smt_allowed() check will now return false if this is not the |
| 537 | * primary sibling. |
| 538 | */ |
| 539 | if (!cpu_smt_allowed(cpu)) |
| 540 | return -ECANCELED; |
| 541 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 542 | if (st->target <= CPUHP_AP_ONLINE_IDLE) |
| 543 | return 0; |
| 544 | |
| 545 | return cpuhp_kick_ap(st, st->target); |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 546 | } |
| 547 | |
Thomas Gleixner | ba99746 | 2016-02-26 18:43:24 +0000 | [diff] [blame] | 548 | static int bringup_cpu(unsigned int cpu) |
| 549 | { |
| 550 | struct task_struct *idle = idle_thread_get(cpu); |
| 551 | int ret; |
| 552 | |
Boris Ostrovsky | aa87717 | 2016-08-03 13:22:28 -0400 | [diff] [blame] | 553 | /* |
| 554 | * Some architectures have to walk the irq descriptors to |
| 555 | * setup the vector space for the cpu which comes online. |
| 556 | * Prevent irq alloc/free across the bringup. |
| 557 | */ |
| 558 | irq_lock_sparse(); |
| 559 | |
Thomas Gleixner | ba99746 | 2016-02-26 18:43:24 +0000 | [diff] [blame] | 560 | /* Arch-specific enabling code. */ |
| 561 | ret = __cpu_up(cpu, idle); |
Boris Ostrovsky | aa87717 | 2016-08-03 13:22:28 -0400 | [diff] [blame] | 562 | irq_unlock_sparse(); |
Thomas Gleixner | 530e9b76 | 2016-12-21 20:19:53 +0100 | [diff] [blame] | 563 | if (ret) |
Thomas Gleixner | ba99746 | 2016-02-26 18:43:24 +0000 | [diff] [blame] | 564 | return ret; |
Thomas Gleixner | 9cd4f1a | 2017-07-04 22:20:23 +0200 | [diff] [blame] | 565 | return bringup_wait_for_ap(cpu); |
Thomas Gleixner | ba99746 | 2016-02-26 18:43:24 +0000 | [diff] [blame] | 566 | } |
| 567 | |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 568 | /* |
| 569 | * Hotplug state machine related functions |
| 570 | */ |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 571 | |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 572 | static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st) |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 573 | { |
Mukesh Ojha | 6fb86d9 | 2018-08-28 12:24:54 +0530 | [diff] [blame] | 574 | for (st->state--; st->state > st->target; st->state--) |
| 575 | cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 576 | } |
| 577 | |
Thomas Gleixner | 206b923 | 2019-03-26 17:36:05 +0100 | [diff] [blame] | 578 | static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st) |
| 579 | { |
| 580 | if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) |
| 581 | return true; |
| 582 | /* |
| 583 | * When CPU hotplug is disabled, then taking the CPU down is not |
| 584 | * possible because takedown_cpu() and the architecture and |
| 585 | * subsystem specific mechanisms are not available. So the CPU |
| 586 | * which would be completely unplugged again needs to stay around |
| 587 | * in the current state. |
| 588 | */ |
| 589 | return st->state <= CPUHP_BRINGUP_CPU; |
| 590 | } |
| 591 | |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 592 | static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 593 | enum cpuhp_state target) |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 594 | { |
| 595 | enum cpuhp_state prev_state = st->state; |
| 596 | int ret = 0; |
| 597 | |
| 598 | while (st->state < target) { |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 599 | st->state++; |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 600 | ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 601 | if (ret) { |
Thomas Gleixner | 206b923 | 2019-03-26 17:36:05 +0100 | [diff] [blame] | 602 | if (can_rollback_cpu(st)) { |
| 603 | st->target = prev_state; |
| 604 | undo_cpu_up(cpu, st); |
| 605 | } |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 606 | break; |
| 607 | } |
| 608 | } |
| 609 | return ret; |
| 610 | } |
| 611 | |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 612 | /* |
| 613 | * The cpu hotplug threads manage the bringup and teardown of the cpus |
| 614 | */ |
| 615 | static void cpuhp_create(unsigned int cpu) |
| 616 | { |
| 617 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 618 | |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 619 | init_completion(&st->done_up); |
| 620 | init_completion(&st->done_down); |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 621 | } |
| 622 | |
| 623 | static int cpuhp_should_run(unsigned int cpu) |
| 624 | { |
| 625 | struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); |
| 626 | |
| 627 | return st->should_run; |
| 628 | } |
| 629 | |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 630 | /* |
| 631 | * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke |
| 632 | * callbacks when a state gets [un]installed at runtime. |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 633 | * |
| 634 | * Each invocation of this function by the smpboot thread does a single AP |
| 635 | * state callback. |
| 636 | * |
| 637 | * It has 3 modes of operation: |
| 638 | * - single: runs st->cb_state |
| 639 | * - up: runs ++st->state, while st->state < st->target |
| 640 | * - down: runs st->state--, while st->state > st->target |
| 641 | * |
| 642 | * When complete or on error, should_run is cleared and the completion is fired. |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 643 | */ |
| 644 | static void cpuhp_thread_fun(unsigned int cpu) |
| 645 | { |
| 646 | struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 647 | bool bringup = st->bringup; |
| 648 | enum cpuhp_state state; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 649 | |
Neeraj Upadhyay | f8b7530 | 2018-09-05 11:22:07 +0530 | [diff] [blame] | 650 | if (WARN_ON_ONCE(!st->should_run)) |
| 651 | return; |
| 652 | |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 653 | /* |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 654 | * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures |
| 655 | * that if we see ->should_run we also see the rest of the state. |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 656 | */ |
| 657 | smp_mb(); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 658 | |
Peter Zijlstra | cb92173 | 2018-09-11 11:51:27 +0200 | [diff] [blame] | 659 | /* |
| 660 | * The BP holds the hotplug lock, but we're now running on the AP, |
| 661 | * ensure that anybody asserting the lock is held, will actually find |
| 662 | * it so. |
| 663 | */ |
| 664 | lockdep_acquire_cpus_lock(); |
Peter Zijlstra | 5f4b55e | 2017-09-20 19:00:20 +0200 | [diff] [blame] | 665 | cpuhp_lock_acquire(bringup); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 666 | |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 667 | if (st->single) { |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 668 | state = st->cb_state; |
| 669 | st->should_run = false; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 670 | } else { |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 671 | if (bringup) { |
| 672 | st->state++; |
| 673 | state = st->state; |
| 674 | st->should_run = (st->state < st->target); |
| 675 | WARN_ON_ONCE(st->state > st->target); |
| 676 | } else { |
| 677 | state = st->state; |
| 678 | st->state--; |
| 679 | st->should_run = (st->state > st->target); |
| 680 | WARN_ON_ONCE(st->state < st->target); |
| 681 | } |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 682 | } |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 683 | |
| 684 | WARN_ON_ONCE(!cpuhp_is_ap_state(state)); |
| 685 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 686 | if (cpuhp_is_atomic_state(state)) { |
| 687 | local_irq_disable(); |
| 688 | st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); |
| 689 | local_irq_enable(); |
| 690 | |
| 691 | /* |
| 692 | * STARTING/DYING must not fail! |
| 693 | */ |
| 694 | WARN_ON_ONCE(st->result); |
| 695 | } else { |
| 696 | st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); |
| 697 | } |
| 698 | |
| 699 | if (st->result) { |
| 700 | /* |
| 701 | * If we fail on a rollback, we're up a creek without no |
| 702 | * paddle, no way forward, no way back. We loose, thanks for |
| 703 | * playing. |
| 704 | */ |
| 705 | WARN_ON_ONCE(st->rollback); |
| 706 | st->should_run = false; |
| 707 | } |
| 708 | |
Peter Zijlstra | 5f4b55e | 2017-09-20 19:00:20 +0200 | [diff] [blame] | 709 | cpuhp_lock_release(bringup); |
Peter Zijlstra | cb92173 | 2018-09-11 11:51:27 +0200 | [diff] [blame] | 710 | lockdep_release_cpus_lock(); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 711 | |
| 712 | if (!st->should_run) |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 713 | complete_ap_thread(st, bringup); |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 714 | } |
| 715 | |
| 716 | /* Invoke a single callback on a remote cpu */ |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 717 | static int |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 718 | cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup, |
| 719 | struct hlist_node *node) |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 720 | { |
| 721 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 722 | int ret; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 723 | |
| 724 | if (!cpu_online(cpu)) |
| 725 | return 0; |
| 726 | |
Peter Zijlstra | 5f4b55e | 2017-09-20 19:00:20 +0200 | [diff] [blame] | 727 | cpuhp_lock_acquire(false); |
| 728 | cpuhp_lock_release(false); |
| 729 | |
| 730 | cpuhp_lock_acquire(true); |
| 731 | cpuhp_lock_release(true); |
Thomas Gleixner | 49dfe2a | 2017-05-24 10:15:43 +0200 | [diff] [blame] | 732 | |
Thomas Gleixner | 6a4e245 | 2016-07-13 17:16:03 +0000 | [diff] [blame] | 733 | /* |
| 734 | * If we are up and running, use the hotplug thread. For early calls |
| 735 | * we invoke the thread function directly. |
| 736 | */ |
| 737 | if (!st->thread) |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 738 | return cpuhp_invoke_callback(cpu, state, bringup, node, NULL); |
Thomas Gleixner | 6a4e245 | 2016-07-13 17:16:03 +0000 | [diff] [blame] | 739 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 740 | st->rollback = false; |
| 741 | st->last = NULL; |
| 742 | |
| 743 | st->node = node; |
| 744 | st->bringup = bringup; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 745 | st->cb_state = state; |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 746 | st->single = true; |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 747 | |
| 748 | __cpuhp_kick_ap(st); |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 749 | |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 750 | /* |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 751 | * If we failed and did a partial, do a rollback. |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 752 | */ |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 753 | if ((ret = st->result) && st->last) { |
| 754 | st->rollback = true; |
| 755 | st->bringup = !bringup; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 756 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 757 | __cpuhp_kick_ap(st); |
| 758 | } |
| 759 | |
Thomas Gleixner | 1f7c70d | 2017-10-21 16:06:52 +0200 | [diff] [blame] | 760 | /* |
| 761 | * Clean up the leftovers so the next hotplug operation wont use stale |
| 762 | * data. |
| 763 | */ |
| 764 | st->node = st->last = NULL; |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 765 | return ret; |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 766 | } |
| 767 | |
| 768 | static int cpuhp_kick_ap_work(unsigned int cpu) |
| 769 | { |
| 770 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 771 | enum cpuhp_state prev_state = st->state; |
| 772 | int ret; |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 773 | |
Peter Zijlstra | 5f4b55e | 2017-09-20 19:00:20 +0200 | [diff] [blame] | 774 | cpuhp_lock_acquire(false); |
| 775 | cpuhp_lock_release(false); |
| 776 | |
| 777 | cpuhp_lock_acquire(true); |
| 778 | cpuhp_lock_release(true); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 779 | |
| 780 | trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); |
| 781 | ret = cpuhp_kick_ap(st, st->target); |
| 782 | trace_cpuhp_exit(cpu, st->state, prev_state, ret); |
| 783 | |
| 784 | return ret; |
Thomas Gleixner | 4cb28ce | 2016-02-26 18:43:38 +0000 | [diff] [blame] | 785 | } |
| 786 | |
| 787 | static struct smp_hotplug_thread cpuhp_threads = { |
| 788 | .store = &cpuhp_state.thread, |
| 789 | .create = &cpuhp_create, |
| 790 | .thread_should_run = cpuhp_should_run, |
| 791 | .thread_fn = cpuhp_thread_fun, |
| 792 | .thread_comm = "cpuhp/%u", |
| 793 | .selfparking = true, |
| 794 | }; |
| 795 | |
| 796 | void __init cpuhp_threads_init(void) |
| 797 | { |
| 798 | BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads)); |
| 799 | kthread_unpark(this_cpu_read(cpuhp_state.thread)); |
| 800 | } |
| 801 | |
Michal Hocko | 777c6e0 | 2016-12-07 14:54:38 +0100 | [diff] [blame] | 802 | #ifdef CONFIG_HOTPLUG_CPU |
Anton Vorontsov | e4cc2f8 | 2012-05-31 16:26:26 -0700 | [diff] [blame] | 803 | /** |
| 804 | * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU |
| 805 | * @cpu: a CPU id |
| 806 | * |
| 807 | * This function walks all processes, finds a valid mm struct for each one and |
| 808 | * then clears a corresponding bit in mm's cpumask. While this all sounds |
| 809 | * trivial, there are various non-obvious corner cases, which this function |
| 810 | * tries to solve in a safe manner. |
| 811 | * |
| 812 | * Also note that the function uses a somewhat relaxed locking scheme, so it may |
| 813 | * be called only for an already offlined CPU. |
| 814 | */ |
Anton Vorontsov | cb79295 | 2012-05-31 16:26:22 -0700 | [diff] [blame] | 815 | void clear_tasks_mm_cpumask(int cpu) |
| 816 | { |
| 817 | struct task_struct *p; |
| 818 | |
| 819 | /* |
| 820 | * This function is called after the cpu is taken down and marked |
| 821 | * offline, so its not like new tasks will ever get this cpu set in |
| 822 | * their mm mask. -- Peter Zijlstra |
| 823 | * Thus, we may use rcu_read_lock() here, instead of grabbing |
| 824 | * full-fledged tasklist_lock. |
| 825 | */ |
Anton Vorontsov | e4cc2f8 | 2012-05-31 16:26:26 -0700 | [diff] [blame] | 826 | WARN_ON(cpu_online(cpu)); |
Anton Vorontsov | cb79295 | 2012-05-31 16:26:22 -0700 | [diff] [blame] | 827 | rcu_read_lock(); |
| 828 | for_each_process(p) { |
| 829 | struct task_struct *t; |
| 830 | |
Anton Vorontsov | e4cc2f8 | 2012-05-31 16:26:26 -0700 | [diff] [blame] | 831 | /* |
| 832 | * Main thread might exit, but other threads may still have |
| 833 | * a valid mm. Find one. |
| 834 | */ |
Anton Vorontsov | cb79295 | 2012-05-31 16:26:22 -0700 | [diff] [blame] | 835 | t = find_lock_task_mm(p); |
| 836 | if (!t) |
| 837 | continue; |
| 838 | cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); |
| 839 | task_unlock(t); |
| 840 | } |
| 841 | rcu_read_unlock(); |
| 842 | } |
| 843 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 844 | /* Take this CPU down. */ |
Mathias Krause | 71cf5ae | 2015-07-19 20:06:22 +0200 | [diff] [blame] | 845 | static int take_cpu_down(void *_param) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 846 | { |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 847 | struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); |
| 848 | enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); |
Thomas Gleixner | 090e77c | 2016-02-26 18:43:23 +0000 | [diff] [blame] | 849 | int err, cpu = smp_processor_id(); |
Peter Zijlstra | 724a868 | 2017-09-20 19:00:18 +0200 | [diff] [blame] | 850 | int ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 851 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 852 | /* Ensure this CPU doesn't handle any more interrupts. */ |
| 853 | err = __cpu_disable(); |
| 854 | if (err < 0) |
Zwane Mwaikambo | f370513 | 2005-06-25 14:54:50 -0700 | [diff] [blame] | 855 | return err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 856 | |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 857 | /* |
| 858 | * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not |
| 859 | * do this step again. |
| 860 | */ |
| 861 | WARN_ON(st->state != CPUHP_TEARDOWN_CPU); |
| 862 | st->state--; |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 863 | /* Invoke the former CPU_DYING callbacks */ |
Peter Zijlstra | 724a868 | 2017-09-20 19:00:18 +0200 | [diff] [blame] | 864 | for (; st->state > target; st->state--) { |
| 865 | ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); |
| 866 | /* |
| 867 | * DYING must not fail! |
| 868 | */ |
| 869 | WARN_ON_ONCE(ret); |
| 870 | } |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 871 | |
Thomas Gleixner | 52c063d | 2015-04-03 02:37:24 +0200 | [diff] [blame] | 872 | /* Give up timekeeping duties */ |
| 873 | tick_handover_do_timer(); |
Thomas Gleixner | 1b72d43 | 2019-03-21 16:39:20 +0100 | [diff] [blame] | 874 | /* Remove CPU from timer broadcasting */ |
| 875 | tick_offline_cpu(cpu); |
Thomas Gleixner | 14e568e | 2013-01-31 12:11:14 +0000 | [diff] [blame] | 876 | /* Park the stopper thread */ |
Thomas Gleixner | 090e77c | 2016-02-26 18:43:23 +0000 | [diff] [blame] | 877 | stop_machine_park(cpu); |
Zwane Mwaikambo | f370513 | 2005-06-25 14:54:50 -0700 | [diff] [blame] | 878 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 879 | } |
| 880 | |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 881 | static int takedown_cpu(unsigned int cpu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 882 | { |
Thomas Gleixner | e69aab1 | 2016-02-26 18:43:43 +0000 | [diff] [blame] | 883 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 884 | int err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 885 | |
Thomas Gleixner | 2a58c52 | 2016-03-10 20:42:08 +0100 | [diff] [blame] | 886 | /* Park the smpboot threads */ |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 887 | kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); |
| 888 | |
Peter Zijlstra | 6acce3e | 2013-10-11 14:38:20 +0200 | [diff] [blame] | 889 | /* |
Thomas Gleixner | a899418 | 2015-07-05 17:12:30 +0000 | [diff] [blame] | 890 | * Prevent irq alloc/free while the dying cpu reorganizes the |
| 891 | * interrupt affinities. |
| 892 | */ |
| 893 | irq_lock_sparse(); |
| 894 | |
| 895 | /* |
Peter Zijlstra | 6acce3e | 2013-10-11 14:38:20 +0200 | [diff] [blame] | 896 | * So now all preempt/rcu users must observe !cpu_active(). |
| 897 | */ |
Sebastian Andrzej Siewior | 210e213 | 2017-05-24 10:15:28 +0200 | [diff] [blame] | 898 | err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu)); |
Rusty Russell | 0432158 | 2008-07-28 12:16:29 -0500 | [diff] [blame] | 899 | if (err) { |
Sebastian Andrzej Siewior | 3b9d6da | 2016-04-08 14:40:15 +0200 | [diff] [blame] | 900 | /* CPU refused to die */ |
Thomas Gleixner | a899418 | 2015-07-05 17:12:30 +0000 | [diff] [blame] | 901 | irq_unlock_sparse(); |
Sebastian Andrzej Siewior | 3b9d6da | 2016-04-08 14:40:15 +0200 | [diff] [blame] | 902 | /* Unpark the hotplug thread so we can rollback there */ |
| 903 | kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread); |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 904 | return err; |
Satoru Takeuchi | 8fa1d7d | 2006-10-28 10:38:57 -0700 | [diff] [blame] | 905 | } |
Rusty Russell | 0432158 | 2008-07-28 12:16:29 -0500 | [diff] [blame] | 906 | BUG_ON(cpu_online(cpu)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 907 | |
Peter Zijlstra | 48c5ccae | 2010-11-13 19:32:29 +0100 | [diff] [blame] | 908 | /* |
Brendan Jackman | 5b1ead6 | 2017-12-06 10:59:11 +0000 | [diff] [blame] | 909 | * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed |
| 910 | * all runnable tasks from the CPU, there's only the idle task left now |
Peter Zijlstra | 48c5ccae | 2010-11-13 19:32:29 +0100 | [diff] [blame] | 911 | * that the migration thread is done doing the stop_machine thing. |
Peter Zijlstra | 51a96c7 | 2010-11-19 20:37:53 +0100 | [diff] [blame] | 912 | * |
| 913 | * Wait for the stop thread to go away. |
Peter Zijlstra | 48c5ccae | 2010-11-13 19:32:29 +0100 | [diff] [blame] | 914 | */ |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 915 | wait_for_ap_thread(st, false); |
Thomas Gleixner | e69aab1 | 2016-02-26 18:43:43 +0000 | [diff] [blame] | 916 | BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 917 | |
Thomas Gleixner | a899418 | 2015-07-05 17:12:30 +0000 | [diff] [blame] | 918 | /* Interrupts are moved away from the dying cpu, reenable alloc/free */ |
| 919 | irq_unlock_sparse(); |
| 920 | |
Preeti U Murthy | 345527b | 2015-03-30 14:59:19 +0530 | [diff] [blame] | 921 | hotplug_cpu__broadcast_tick_pull(cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 922 | /* This actually kills the CPU. */ |
| 923 | __cpu_die(cpu); |
| 924 | |
Thomas Gleixner | a49b116 | 2015-04-03 02:38:05 +0200 | [diff] [blame] | 925 | tick_cleanup_dead_cpu(cpu); |
Paul E. McKenney | a58163d | 2017-06-20 12:11:34 -0700 | [diff] [blame] | 926 | rcutree_migrate_callbacks(cpu); |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 927 | return 0; |
| 928 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 929 | |
Thomas Gleixner | 71f87b2 | 2016-03-03 10:52:10 +0100 | [diff] [blame] | 930 | static void cpuhp_complete_idle_dead(void *arg) |
| 931 | { |
| 932 | struct cpuhp_cpu_state *st = arg; |
| 933 | |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 934 | complete_ap_thread(st, false); |
Thomas Gleixner | 71f87b2 | 2016-03-03 10:52:10 +0100 | [diff] [blame] | 935 | } |
| 936 | |
Thomas Gleixner | e69aab1 | 2016-02-26 18:43:43 +0000 | [diff] [blame] | 937 | void cpuhp_report_idle_dead(void) |
| 938 | { |
| 939 | struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); |
| 940 | |
| 941 | BUG_ON(st->state != CPUHP_AP_OFFLINE); |
Thomas Gleixner | 27d50c7 | 2016-02-26 18:43:44 +0000 | [diff] [blame] | 942 | rcu_report_dead(smp_processor_id()); |
Thomas Gleixner | 71f87b2 | 2016-03-03 10:52:10 +0100 | [diff] [blame] | 943 | st->state = CPUHP_AP_IDLE_DEAD; |
| 944 | /* |
| 945 | * We cannot call complete after rcu_report_dead() so we delegate it |
| 946 | * to an online cpu. |
| 947 | */ |
| 948 | smp_call_function_single(cpumask_first(cpu_online_mask), |
| 949 | cpuhp_complete_idle_dead, st, 0); |
Thomas Gleixner | e69aab1 | 2016-02-26 18:43:43 +0000 | [diff] [blame] | 950 | } |
| 951 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 952 | static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st) |
| 953 | { |
Mukesh Ojha | 6fb86d9 | 2018-08-28 12:24:54 +0530 | [diff] [blame] | 954 | for (st->state++; st->state < st->target; st->state++) |
| 955 | cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 956 | } |
| 957 | |
| 958 | static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, |
| 959 | enum cpuhp_state target) |
| 960 | { |
| 961 | enum cpuhp_state prev_state = st->state; |
| 962 | int ret = 0; |
| 963 | |
| 964 | for (; st->state > target; st->state--) { |
| 965 | ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); |
| 966 | if (ret) { |
| 967 | st->target = prev_state; |
Thomas Gleixner | 69fa6eb | 2018-09-06 15:21:38 +0200 | [diff] [blame] | 968 | if (st->state < prev_state) |
| 969 | undo_cpu_down(cpu, st); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 970 | break; |
| 971 | } |
| 972 | } |
| 973 | return ret; |
| 974 | } |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 975 | |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 976 | /* Requires cpu_add_remove_lock to be held */ |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 977 | static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, |
| 978 | enum cpuhp_state target) |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 979 | { |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 980 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 981 | int prev_state, ret = 0; |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 982 | |
| 983 | if (num_online_cpus() == 1) |
| 984 | return -EBUSY; |
| 985 | |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 986 | if (!cpu_present(cpu)) |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 987 | return -EINVAL; |
| 988 | |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 989 | cpus_write_lock(); |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 990 | |
| 991 | cpuhp_tasks_frozen = tasks_frozen; |
| 992 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 993 | prev_state = cpuhp_set_state(st, target); |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 994 | /* |
| 995 | * If the current CPU state is in the range of the AP hotplug thread, |
| 996 | * then we need to kick the thread. |
| 997 | */ |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 998 | if (st->state > CPUHP_TEARDOWN_CPU) { |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 999 | st->target = max((int)target, CPUHP_TEARDOWN_CPU); |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1000 | ret = cpuhp_kick_ap_work(cpu); |
| 1001 | /* |
| 1002 | * The AP side has done the error rollback already. Just |
| 1003 | * return the error code.. |
| 1004 | */ |
| 1005 | if (ret) |
| 1006 | goto out; |
| 1007 | |
| 1008 | /* |
| 1009 | * We might have stopped still in the range of the AP hotplug |
| 1010 | * thread. Nothing to do anymore. |
| 1011 | */ |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 1012 | if (st->state > CPUHP_TEARDOWN_CPU) |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1013 | goto out; |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 1014 | |
| 1015 | st->target = target; |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1016 | } |
| 1017 | /* |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 1018 | * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1019 | * to do the further cleanups. |
| 1020 | */ |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 1021 | ret = cpuhp_down_callbacks(cpu, st, target); |
Thomas Gleixner | 69fa6eb | 2018-09-06 15:21:38 +0200 | [diff] [blame] | 1022 | if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) { |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 1023 | cpuhp_reset_state(st, prev_state); |
| 1024 | __cpuhp_kick_ap(st); |
Sebastian Andrzej Siewior | 3b9d6da | 2016-04-08 14:40:15 +0200 | [diff] [blame] | 1025 | } |
Thomas Gleixner | 9845817 | 2016-02-26 18:43:25 +0000 | [diff] [blame] | 1026 | |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1027 | out: |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 1028 | cpus_write_unlock(); |
Thomas Gleixner | 941154b | 2017-09-12 21:37:04 +0200 | [diff] [blame] | 1029 | /* |
| 1030 | * Do post unplug cleanup. This is still protected against |
| 1031 | * concurrent CPU hotplug via cpu_add_remove_lock. |
| 1032 | */ |
| 1033 | lockup_detector_cleanup(); |
Thomas Gleixner | a74cfff | 2018-11-25 19:33:39 +0100 | [diff] [blame] | 1034 | arch_smt_update(); |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1035 | return ret; |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1036 | } |
| 1037 | |
Thomas Gleixner | cc1fe21 | 2018-05-29 17:49:05 +0200 | [diff] [blame] | 1038 | static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target) |
| 1039 | { |
| 1040 | if (cpu_hotplug_disabled) |
| 1041 | return -EBUSY; |
| 1042 | return _cpu_down(cpu, 0, target); |
| 1043 | } |
| 1044 | |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 1045 | static int do_cpu_down(unsigned int cpu, enum cpuhp_state target) |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1046 | { |
Heiko Carstens | 9ea09af | 2008-12-22 12:36:30 +0100 | [diff] [blame] | 1047 | int err; |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1048 | |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 1049 | cpu_maps_update_begin(); |
Thomas Gleixner | cc1fe21 | 2018-05-29 17:49:05 +0200 | [diff] [blame] | 1050 | err = cpu_down_maps_locked(cpu, target); |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 1051 | cpu_maps_update_done(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1052 | return err; |
| 1053 | } |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 1054 | |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 1055 | int cpu_down(unsigned int cpu) |
| 1056 | { |
| 1057 | return do_cpu_down(cpu, CPUHP_OFFLINE); |
| 1058 | } |
Zhang Rui | b62b8ef | 2008-04-29 02:35:56 -0400 | [diff] [blame] | 1059 | EXPORT_SYMBOL(cpu_down); |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 1060 | |
| 1061 | #else |
| 1062 | #define takedown_cpu NULL |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1063 | #endif /*CONFIG_HOTPLUG_CPU*/ |
| 1064 | |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1065 | /** |
Thomas Gleixner | ee1e714 | 2016-08-18 14:57:16 +0200 | [diff] [blame] | 1066 | * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1067 | * @cpu: cpu that just started |
| 1068 | * |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1069 | * It must be called by the arch code on the new cpu, before the new cpu |
| 1070 | * enables interrupts and before the "boot" cpu returns from __cpu_up(). |
| 1071 | */ |
| 1072 | void notify_cpu_starting(unsigned int cpu) |
| 1073 | { |
| 1074 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 1075 | enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); |
Peter Zijlstra | 724a868 | 2017-09-20 19:00:18 +0200 | [diff] [blame] | 1076 | int ret; |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1077 | |
Sebastian Andrzej Siewior | 0c6d457 | 2016-08-17 14:21:04 +0200 | [diff] [blame] | 1078 | rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */ |
Thomas Gleixner | e797bda | 2019-07-22 20:47:16 +0200 | [diff] [blame] | 1079 | cpumask_set_cpu(cpu, &cpus_booted_once_mask); |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1080 | while (st->state < target) { |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1081 | st->state++; |
Peter Zijlstra | 724a868 | 2017-09-20 19:00:18 +0200 | [diff] [blame] | 1082 | ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); |
| 1083 | /* |
| 1084 | * STARTING must not fail! |
| 1085 | */ |
| 1086 | WARN_ON_ONCE(ret); |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1087 | } |
| 1088 | } |
| 1089 | |
Thomas Gleixner | 949338e | 2016-02-26 18:43:35 +0000 | [diff] [blame] | 1090 | /* |
Thomas Gleixner | 9cd4f1a | 2017-07-04 22:20:23 +0200 | [diff] [blame] | 1091 | * Called from the idle task. Wake up the controlling task which brings the |
| 1092 | * stopper and the hotplug thread of the upcoming CPU up and then delegates |
| 1093 | * the rest of the online bringup to the hotplug thread. |
Thomas Gleixner | 949338e | 2016-02-26 18:43:35 +0000 | [diff] [blame] | 1094 | */ |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 1095 | void cpuhp_online_idle(enum cpuhp_state state) |
Thomas Gleixner | 949338e | 2016-02-26 18:43:35 +0000 | [diff] [blame] | 1096 | { |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 1097 | struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 1098 | |
| 1099 | /* Happens for the boot cpu */ |
| 1100 | if (state != CPUHP_AP_ONLINE_IDLE) |
| 1101 | return; |
| 1102 | |
| 1103 | st->state = CPUHP_AP_ONLINE_IDLE; |
Peter Zijlstra | 5ebe774 | 2017-09-20 19:00:19 +0200 | [diff] [blame] | 1104 | complete_ap_thread(st, true); |
Thomas Gleixner | 949338e | 2016-02-26 18:43:35 +0000 | [diff] [blame] | 1105 | } |
| 1106 | |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1107 | /* Requires cpu_add_remove_lock to be held */ |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 1108 | static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1109 | { |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1110 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
Suresh Siddha | 3bb5d2e | 2012-04-20 17:08:50 -0700 | [diff] [blame] | 1111 | struct task_struct *idle; |
Thomas Gleixner | 2e1a348 | 2016-02-26 18:43:37 +0000 | [diff] [blame] | 1112 | int ret = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1113 | |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 1114 | cpus_write_lock(); |
Thomas Gleixner | 38498a6 | 2012-04-20 13:05:44 +0000 | [diff] [blame] | 1115 | |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1116 | if (!cpu_present(cpu)) { |
Yasuaki Ishimatsu | 5e5041f | 2012-10-23 01:30:54 +0200 | [diff] [blame] | 1117 | ret = -EINVAL; |
| 1118 | goto out; |
| 1119 | } |
| 1120 | |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1121 | /* |
| 1122 | * The caller of do_cpu_up might have raced with another |
| 1123 | * caller. Ignore it for now. |
| 1124 | */ |
| 1125 | if (st->state >= target) |
Thomas Gleixner | 38498a6 | 2012-04-20 13:05:44 +0000 | [diff] [blame] | 1126 | goto out; |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1127 | |
| 1128 | if (st->state == CPUHP_OFFLINE) { |
| 1129 | /* Let it fail before we try to bring the cpu up */ |
| 1130 | idle = idle_thread_get(cpu); |
| 1131 | if (IS_ERR(idle)) { |
| 1132 | ret = PTR_ERR(idle); |
| 1133 | goto out; |
| 1134 | } |
Suresh Siddha | 3bb5d2e | 2012-04-20 17:08:50 -0700 | [diff] [blame] | 1135 | } |
Thomas Gleixner | 38498a6 | 2012-04-20 13:05:44 +0000 | [diff] [blame] | 1136 | |
Thomas Gleixner | ba99746 | 2016-02-26 18:43:24 +0000 | [diff] [blame] | 1137 | cpuhp_tasks_frozen = tasks_frozen; |
| 1138 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 1139 | cpuhp_set_state(st, target); |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1140 | /* |
| 1141 | * If the current CPU state is in the range of the AP hotplug thread, |
| 1142 | * then we need to kick the thread once more. |
| 1143 | */ |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 1144 | if (st->state > CPUHP_BRINGUP_CPU) { |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1145 | ret = cpuhp_kick_ap_work(cpu); |
| 1146 | /* |
| 1147 | * The AP side has done the error rollback already. Just |
| 1148 | * return the error code.. |
| 1149 | */ |
| 1150 | if (ret) |
| 1151 | goto out; |
| 1152 | } |
| 1153 | |
| 1154 | /* |
| 1155 | * Try to reach the target state. We max out on the BP at |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 1156 | * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1157 | * responsible for bringing it up to the target state. |
| 1158 | */ |
Thomas Gleixner | 8df3e07 | 2016-02-26 18:43:41 +0000 | [diff] [blame] | 1159 | target = min((int)target, CPUHP_BRINGUP_CPU); |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 1160 | ret = cpuhp_up_callbacks(cpu, st, target); |
Thomas Gleixner | 38498a6 | 2012-04-20 13:05:44 +0000 | [diff] [blame] | 1161 | out: |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 1162 | cpus_write_unlock(); |
Thomas Gleixner | a74cfff | 2018-11-25 19:33:39 +0100 | [diff] [blame] | 1163 | arch_smt_update(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1164 | return ret; |
| 1165 | } |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1166 | |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 1167 | static int do_cpu_up(unsigned int cpu, enum cpuhp_state target) |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1168 | { |
| 1169 | int err = 0; |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 1170 | |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1171 | if (!cpu_possible(cpu)) { |
Fabian Frederick | 84117da | 2014-06-04 16:11:17 -0700 | [diff] [blame] | 1172 | pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", |
| 1173 | cpu); |
Chen Gong | 87d5e023 | 2010-03-05 13:42:38 -0800 | [diff] [blame] | 1174 | #if defined(CONFIG_IA64) |
Fabian Frederick | 84117da | 2014-06-04 16:11:17 -0700 | [diff] [blame] | 1175 | pr_err("please check additional_cpus= boot parameter\n"); |
KAMEZAWA Hiroyuki | 73e753a | 2007-10-18 23:40:47 -0700 | [diff] [blame] | 1176 | #endif |
| 1177 | return -EINVAL; |
| 1178 | } |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1179 | |
Toshi Kani | 01b0f19 | 2013-11-12 15:07:25 -0800 | [diff] [blame] | 1180 | err = try_online_node(cpu_to_node(cpu)); |
| 1181 | if (err) |
| 1182 | return err; |
minskey guo | cf23422 | 2010-05-24 14:32:41 -0700 | [diff] [blame] | 1183 | |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 1184 | cpu_maps_update_begin(); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1185 | |
Max Krasnyansky | e761b77 | 2008-07-15 04:43:49 -0700 | [diff] [blame] | 1186 | if (cpu_hotplug_disabled) { |
| 1187 | err = -EBUSY; |
| 1188 | goto out; |
| 1189 | } |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 1190 | if (!cpu_smt_allowed(cpu)) { |
| 1191 | err = -EPERM; |
| 1192 | goto out; |
| 1193 | } |
Max Krasnyansky | e761b77 | 2008-07-15 04:43:49 -0700 | [diff] [blame] | 1194 | |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 1195 | err = _cpu_up(cpu, 0, target); |
Max Krasnyansky | e761b77 | 2008-07-15 04:43:49 -0700 | [diff] [blame] | 1196 | out: |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 1197 | cpu_maps_update_done(); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1198 | return err; |
| 1199 | } |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 1200 | |
| 1201 | int cpu_up(unsigned int cpu) |
| 1202 | { |
| 1203 | return do_cpu_up(cpu, CPUHP_ONLINE); |
| 1204 | } |
Paul E. McKenney | a513f6b | 2011-12-11 21:54:45 -0800 | [diff] [blame] | 1205 | EXPORT_SYMBOL_GPL(cpu_up); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1206 | |
Rafael J. Wysocki | f3de4be | 2007-08-30 23:56:29 -0700 | [diff] [blame] | 1207 | #ifdef CONFIG_PM_SLEEP_SMP |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1208 | static cpumask_var_t frozen_cpus; |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1209 | |
James Morse | d391e55 | 2016-08-17 13:50:25 +0100 | [diff] [blame] | 1210 | int freeze_secondary_cpus(int primary) |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1211 | { |
James Morse | d391e55 | 2016-08-17 13:50:25 +0100 | [diff] [blame] | 1212 | int cpu, error = 0; |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1213 | |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 1214 | cpu_maps_update_begin(); |
Nicholas Piggin | 9ca12ac | 2019-04-11 13:34:46 +1000 | [diff] [blame] | 1215 | if (primary == -1) { |
James Morse | d391e55 | 2016-08-17 13:50:25 +0100 | [diff] [blame] | 1216 | primary = cpumask_first(cpu_online_mask); |
Nicholas Piggin | 9ca12ac | 2019-04-11 13:34:46 +1000 | [diff] [blame] | 1217 | if (!housekeeping_cpu(primary, HK_FLAG_TIMER)) |
| 1218 | primary = housekeeping_any_cpu(HK_FLAG_TIMER); |
| 1219 | } else { |
| 1220 | if (!cpu_online(primary)) |
| 1221 | primary = cpumask_first(cpu_online_mask); |
| 1222 | } |
| 1223 | |
Xiaotian Feng | 9ee349a | 2009-12-16 18:04:32 +0100 | [diff] [blame] | 1224 | /* |
| 1225 | * We take down all of the non-boot CPUs in one shot to avoid races |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1226 | * with the userspace trying to use the CPU hotplug at the same time |
| 1227 | */ |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1228 | cpumask_clear(frozen_cpus); |
Peter Zijlstra | 6ad4c18 | 2009-11-25 13:31:39 +0100 | [diff] [blame] | 1229 | |
Fabian Frederick | 84117da | 2014-06-04 16:11:17 -0700 | [diff] [blame] | 1230 | pr_info("Disabling non-boot CPUs ...\n"); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1231 | for_each_online_cpu(cpu) { |
James Morse | d391e55 | 2016-08-17 13:50:25 +0100 | [diff] [blame] | 1232 | if (cpu == primary) |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1233 | continue; |
Pavankumar Kondeti | a66d955 | 2019-06-03 10:01:03 +0530 | [diff] [blame] | 1234 | |
| 1235 | if (pm_wakeup_pending()) { |
| 1236 | pr_info("Wakeup pending. Abort CPU freeze\n"); |
| 1237 | error = -EBUSY; |
| 1238 | break; |
| 1239 | } |
| 1240 | |
Todd E Brandt | bb3632c | 2014-06-06 05:40:17 -0700 | [diff] [blame] | 1241 | trace_suspend_resume(TPS("CPU_OFF"), cpu, true); |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 1242 | error = _cpu_down(cpu, 1, CPUHP_OFFLINE); |
Todd E Brandt | bb3632c | 2014-06-06 05:40:17 -0700 | [diff] [blame] | 1243 | trace_suspend_resume(TPS("CPU_OFF"), cpu, false); |
Mike Travis | feae320 | 2009-11-17 18:22:13 -0600 | [diff] [blame] | 1244 | if (!error) |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1245 | cpumask_set_cpu(cpu, frozen_cpus); |
Mike Travis | feae320 | 2009-11-17 18:22:13 -0600 | [diff] [blame] | 1246 | else { |
Fabian Frederick | 84117da | 2014-06-04 16:11:17 -0700 | [diff] [blame] | 1247 | pr_err("Error taking CPU%d down: %d\n", cpu, error); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1248 | break; |
| 1249 | } |
| 1250 | } |
Joseph Cihula | 86886e5 | 2009-06-30 19:31:07 -0700 | [diff] [blame] | 1251 | |
Vitaly Kuznetsov | 89af7ba | 2015-08-05 00:52:46 -0700 | [diff] [blame] | 1252 | if (!error) |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1253 | BUG_ON(num_online_cpus() > 1); |
Vitaly Kuznetsov | 89af7ba | 2015-08-05 00:52:46 -0700 | [diff] [blame] | 1254 | else |
Fabian Frederick | 84117da | 2014-06-04 16:11:17 -0700 | [diff] [blame] | 1255 | pr_err("Non-boot CPUs are not disabled\n"); |
Vitaly Kuznetsov | 89af7ba | 2015-08-05 00:52:46 -0700 | [diff] [blame] | 1256 | |
| 1257 | /* |
| 1258 | * Make sure the CPUs won't be enabled by someone else. We need to do |
| 1259 | * this even in case of failure as all disable_nonboot_cpus() users are |
| 1260 | * supposed to do enable_nonboot_cpus() on the failure path. |
| 1261 | */ |
| 1262 | cpu_hotplug_disabled++; |
| 1263 | |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 1264 | cpu_maps_update_done(); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1265 | return error; |
| 1266 | } |
| 1267 | |
Suresh Siddha | d0af9ee | 2009-08-19 18:05:36 -0700 | [diff] [blame] | 1268 | void __weak arch_enable_nonboot_cpus_begin(void) |
| 1269 | { |
| 1270 | } |
| 1271 | |
| 1272 | void __weak arch_enable_nonboot_cpus_end(void) |
| 1273 | { |
| 1274 | } |
| 1275 | |
Mathias Krause | 71cf5ae | 2015-07-19 20:06:22 +0200 | [diff] [blame] | 1276 | void enable_nonboot_cpus(void) |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1277 | { |
| 1278 | int cpu, error; |
| 1279 | |
| 1280 | /* Allow everyone to use the CPU hotplug again */ |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 1281 | cpu_maps_update_begin(); |
Lianwei Wang | 01b4115 | 2016-06-09 23:43:28 -0700 | [diff] [blame] | 1282 | __cpu_hotplug_enable(); |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1283 | if (cpumask_empty(frozen_cpus)) |
Rafael J. Wysocki | 1d64b9c | 2007-04-01 23:49:49 -0700 | [diff] [blame] | 1284 | goto out; |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1285 | |
Fabian Frederick | 84117da | 2014-06-04 16:11:17 -0700 | [diff] [blame] | 1286 | pr_info("Enabling non-boot CPUs ...\n"); |
Suresh Siddha | d0af9ee | 2009-08-19 18:05:36 -0700 | [diff] [blame] | 1287 | |
| 1288 | arch_enable_nonboot_cpus_begin(); |
| 1289 | |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1290 | for_each_cpu(cpu, frozen_cpus) { |
Todd E Brandt | bb3632c | 2014-06-06 05:40:17 -0700 | [diff] [blame] | 1291 | trace_suspend_resume(TPS("CPU_ON"), cpu, true); |
Thomas Gleixner | af1f404 | 2016-02-26 18:43:30 +0000 | [diff] [blame] | 1292 | error = _cpu_up(cpu, 1, CPUHP_ONLINE); |
Todd E Brandt | bb3632c | 2014-06-06 05:40:17 -0700 | [diff] [blame] | 1293 | trace_suspend_resume(TPS("CPU_ON"), cpu, false); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1294 | if (!error) { |
Fabian Frederick | 84117da | 2014-06-04 16:11:17 -0700 | [diff] [blame] | 1295 | pr_info("CPU%d is up\n", cpu); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1296 | continue; |
| 1297 | } |
Fabian Frederick | 84117da | 2014-06-04 16:11:17 -0700 | [diff] [blame] | 1298 | pr_warn("Error taking CPU%d up: %d\n", cpu, error); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1299 | } |
Suresh Siddha | d0af9ee | 2009-08-19 18:05:36 -0700 | [diff] [blame] | 1300 | |
| 1301 | arch_enable_nonboot_cpus_end(); |
| 1302 | |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1303 | cpumask_clear(frozen_cpus); |
Rafael J. Wysocki | 1d64b9c | 2007-04-01 23:49:49 -0700 | [diff] [blame] | 1304 | out: |
Gautham R Shenoy | d221938 | 2008-01-25 21:08:01 +0100 | [diff] [blame] | 1305 | cpu_maps_update_done(); |
Rafael J. Wysocki | e3920fb | 2006-09-25 23:32:48 -0700 | [diff] [blame] | 1306 | } |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1307 | |
Fenghua Yu | d7268a3 | 2011-11-15 21:59:31 +0100 | [diff] [blame] | 1308 | static int __init alloc_frozen_cpus(void) |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 1309 | { |
| 1310 | if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) |
| 1311 | return -ENOMEM; |
| 1312 | return 0; |
| 1313 | } |
| 1314 | core_initcall(alloc_frozen_cpus); |
Srivatsa S. Bhat | 79cfbdf | 2011-11-03 00:59:25 +0100 | [diff] [blame] | 1315 | |
| 1316 | /* |
Srivatsa S. Bhat | 79cfbdf | 2011-11-03 00:59:25 +0100 | [diff] [blame] | 1317 | * When callbacks for CPU hotplug notifications are being executed, we must |
| 1318 | * ensure that the state of the system with respect to the tasks being frozen |
| 1319 | * or not, as reported by the notification, remains unchanged *throughout the |
| 1320 | * duration* of the execution of the callbacks. |
| 1321 | * Hence we need to prevent the freezer from racing with regular CPU hotplug. |
| 1322 | * |
| 1323 | * This synchronization is implemented by mutually excluding regular CPU |
| 1324 | * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ |
| 1325 | * Hibernate notifications. |
| 1326 | */ |
| 1327 | static int |
| 1328 | cpu_hotplug_pm_callback(struct notifier_block *nb, |
| 1329 | unsigned long action, void *ptr) |
| 1330 | { |
| 1331 | switch (action) { |
| 1332 | |
| 1333 | case PM_SUSPEND_PREPARE: |
| 1334 | case PM_HIBERNATION_PREPARE: |
Srivatsa S. Bhat | 16e53db | 2013-06-12 14:04:36 -0700 | [diff] [blame] | 1335 | cpu_hotplug_disable(); |
Srivatsa S. Bhat | 79cfbdf | 2011-11-03 00:59:25 +0100 | [diff] [blame] | 1336 | break; |
| 1337 | |
| 1338 | case PM_POST_SUSPEND: |
| 1339 | case PM_POST_HIBERNATION: |
Srivatsa S. Bhat | 16e53db | 2013-06-12 14:04:36 -0700 | [diff] [blame] | 1340 | cpu_hotplug_enable(); |
Srivatsa S. Bhat | 79cfbdf | 2011-11-03 00:59:25 +0100 | [diff] [blame] | 1341 | break; |
| 1342 | |
| 1343 | default: |
| 1344 | return NOTIFY_DONE; |
| 1345 | } |
| 1346 | |
| 1347 | return NOTIFY_OK; |
| 1348 | } |
| 1349 | |
| 1350 | |
Fenghua Yu | d7268a3 | 2011-11-15 21:59:31 +0100 | [diff] [blame] | 1351 | static int __init cpu_hotplug_pm_sync_init(void) |
Srivatsa S. Bhat | 79cfbdf | 2011-11-03 00:59:25 +0100 | [diff] [blame] | 1352 | { |
Fenghua Yu | 6e32d47 | 2012-11-13 11:32:43 -0800 | [diff] [blame] | 1353 | /* |
| 1354 | * cpu_hotplug_pm_callback has higher priority than x86 |
| 1355 | * bsp_pm_callback which depends on cpu_hotplug_pm_callback |
| 1356 | * to disable cpu hotplug to avoid cpu hotplug race. |
| 1357 | */ |
Srivatsa S. Bhat | 79cfbdf | 2011-11-03 00:59:25 +0100 | [diff] [blame] | 1358 | pm_notifier(cpu_hotplug_pm_callback, 0); |
| 1359 | return 0; |
| 1360 | } |
| 1361 | core_initcall(cpu_hotplug_pm_sync_init); |
| 1362 | |
Rafael J. Wysocki | f3de4be | 2007-08-30 23:56:29 -0700 | [diff] [blame] | 1363 | #endif /* CONFIG_PM_SLEEP_SMP */ |
Max Krasnyansky | 68f4f1e | 2008-05-29 11:17:02 -0700 | [diff] [blame] | 1364 | |
Peter Zijlstra | 8ce371f | 2017-03-20 12:26:55 +0100 | [diff] [blame] | 1365 | int __boot_cpu_id; |
| 1366 | |
Max Krasnyansky | 68f4f1e | 2008-05-29 11:17:02 -0700 | [diff] [blame] | 1367 | #endif /* CONFIG_SMP */ |
Mike Travis | b8d317d | 2008-07-24 18:21:29 -0700 | [diff] [blame] | 1368 | |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1369 | /* Boot processor state steps */ |
Lai Jiangshan | 17a2f1c | 2017-12-01 21:50:05 +0800 | [diff] [blame] | 1370 | static struct cpuhp_step cpuhp_hp_states[] = { |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1371 | [CPUHP_OFFLINE] = { |
| 1372 | .name = "offline", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1373 | .startup.single = NULL, |
| 1374 | .teardown.single = NULL, |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1375 | }, |
| 1376 | #ifdef CONFIG_SMP |
| 1377 | [CPUHP_CREATE_THREADS]= { |
Thomas Gleixner | 677f664 | 2016-09-06 16:13:48 +0200 | [diff] [blame] | 1378 | .name = "threads:prepare", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1379 | .startup.single = smpboot_create_threads, |
| 1380 | .teardown.single = NULL, |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1381 | .cant_stop = true, |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1382 | }, |
Thomas Gleixner | 00e16c3 | 2016-07-13 17:16:09 +0000 | [diff] [blame] | 1383 | [CPUHP_PERF_PREPARE] = { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1384 | .name = "perf:prepare", |
| 1385 | .startup.single = perf_event_init_cpu, |
| 1386 | .teardown.single = perf_event_exit_cpu, |
Thomas Gleixner | 00e16c3 | 2016-07-13 17:16:09 +0000 | [diff] [blame] | 1387 | }, |
Thomas Gleixner | 7ee681b | 2016-07-13 17:16:29 +0000 | [diff] [blame] | 1388 | [CPUHP_WORKQUEUE_PREP] = { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1389 | .name = "workqueue:prepare", |
| 1390 | .startup.single = workqueue_prepare_cpu, |
| 1391 | .teardown.single = NULL, |
Thomas Gleixner | 7ee681b | 2016-07-13 17:16:29 +0000 | [diff] [blame] | 1392 | }, |
Thomas Gleixner | 27590dc | 2016-07-15 10:41:04 +0200 | [diff] [blame] | 1393 | [CPUHP_HRTIMERS_PREPARE] = { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1394 | .name = "hrtimers:prepare", |
| 1395 | .startup.single = hrtimers_prepare_cpu, |
| 1396 | .teardown.single = hrtimers_dead_cpu, |
Thomas Gleixner | 27590dc | 2016-07-15 10:41:04 +0200 | [diff] [blame] | 1397 | }, |
Richard Weinberger | 31487f8 | 2016-07-13 17:17:01 +0000 | [diff] [blame] | 1398 | [CPUHP_SMPCFD_PREPARE] = { |
Thomas Gleixner | 677f664 | 2016-09-06 16:13:48 +0200 | [diff] [blame] | 1399 | .name = "smpcfd:prepare", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1400 | .startup.single = smpcfd_prepare_cpu, |
| 1401 | .teardown.single = smpcfd_dead_cpu, |
Richard Weinberger | 31487f8 | 2016-07-13 17:17:01 +0000 | [diff] [blame] | 1402 | }, |
Richard Weinberger | e6d4989 | 2016-08-18 14:57:17 +0200 | [diff] [blame] | 1403 | [CPUHP_RELAY_PREPARE] = { |
| 1404 | .name = "relay:prepare", |
| 1405 | .startup.single = relay_prepare_cpu, |
| 1406 | .teardown.single = NULL, |
| 1407 | }, |
Sebastian Andrzej Siewior | 6731d4f | 2016-08-23 14:53:19 +0200 | [diff] [blame] | 1408 | [CPUHP_SLAB_PREPARE] = { |
| 1409 | .name = "slab:prepare", |
| 1410 | .startup.single = slab_prepare_cpu, |
| 1411 | .teardown.single = slab_dead_cpu, |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1412 | }, |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 1413 | [CPUHP_RCUTREE_PREP] = { |
Thomas Gleixner | 677f664 | 2016-09-06 16:13:48 +0200 | [diff] [blame] | 1414 | .name = "RCU/tree:prepare", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1415 | .startup.single = rcutree_prepare_cpu, |
| 1416 | .teardown.single = rcutree_dead_cpu, |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 1417 | }, |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1418 | /* |
Richard Cochran | 4fae16d | 2016-07-27 11:08:18 +0200 | [diff] [blame] | 1419 | * On the tear-down path, timers_dead_cpu() must be invoked |
| 1420 | * before blk_mq_queue_reinit_notify() from notify_dead(), |
| 1421 | * otherwise a RCU stall occurs. |
| 1422 | */ |
Thomas Gleixner | 26456f8 | 2017-12-27 21:37:25 +0100 | [diff] [blame] | 1423 | [CPUHP_TIMERS_PREPARE] = { |
Mukesh Ojha | d018031 | 2018-07-24 20:17:48 +0530 | [diff] [blame] | 1424 | .name = "timers:prepare", |
Thomas Gleixner | 26456f8 | 2017-12-27 21:37:25 +0100 | [diff] [blame] | 1425 | .startup.single = timers_prepare_cpu, |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1426 | .teardown.single = timers_dead_cpu, |
Richard Cochran | 4fae16d | 2016-07-27 11:08:18 +0200 | [diff] [blame] | 1427 | }, |
Thomas Gleixner | d10ef6f | 2016-03-08 10:36:13 +0100 | [diff] [blame] | 1428 | /* Kicks the plugged cpu into life */ |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 1429 | [CPUHP_BRINGUP_CPU] = { |
| 1430 | .name = "cpu:bringup", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1431 | .startup.single = bringup_cpu, |
| 1432 | .teardown.single = NULL, |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1433 | .cant_stop = true, |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1434 | }, |
Thomas Gleixner | d10ef6f | 2016-03-08 10:36:13 +0100 | [diff] [blame] | 1435 | /* Final state before CPU kills itself */ |
| 1436 | [CPUHP_AP_IDLE_DEAD] = { |
| 1437 | .name = "idle:dead", |
| 1438 | }, |
| 1439 | /* |
| 1440 | * Last state before CPU enters the idle loop to die. Transient state |
| 1441 | * for synchronization. |
| 1442 | */ |
| 1443 | [CPUHP_AP_OFFLINE] = { |
| 1444 | .name = "ap:offline", |
| 1445 | .cant_stop = true, |
| 1446 | }, |
Thomas Gleixner | 9cf7243 | 2016-03-10 12:54:09 +0100 | [diff] [blame] | 1447 | /* First state is scheduler control. Interrupts are disabled */ |
| 1448 | [CPUHP_AP_SCHED_STARTING] = { |
| 1449 | .name = "sched:starting", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1450 | .startup.single = sched_cpu_starting, |
| 1451 | .teardown.single = sched_cpu_dying, |
Thomas Gleixner | 9cf7243 | 2016-03-10 12:54:09 +0100 | [diff] [blame] | 1452 | }, |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 1453 | [CPUHP_AP_RCUTREE_DYING] = { |
Thomas Gleixner | 677f664 | 2016-09-06 16:13:48 +0200 | [diff] [blame] | 1454 | .name = "RCU/tree:dying", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1455 | .startup.single = NULL, |
| 1456 | .teardown.single = rcutree_dying_cpu, |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1457 | }, |
Lai Jiangshan | 46febd3 | 2017-11-28 21:19:53 +0800 | [diff] [blame] | 1458 | [CPUHP_AP_SMPCFD_DYING] = { |
| 1459 | .name = "smpcfd:dying", |
| 1460 | .startup.single = NULL, |
| 1461 | .teardown.single = smpcfd_dying_cpu, |
| 1462 | }, |
Thomas Gleixner | d10ef6f | 2016-03-08 10:36:13 +0100 | [diff] [blame] | 1463 | /* Entry state on starting. Interrupts enabled from here on. Transient |
| 1464 | * state for synchronsization */ |
| 1465 | [CPUHP_AP_ONLINE] = { |
| 1466 | .name = "ap:online", |
| 1467 | }, |
Lai Jiangshan | 17a2f1c | 2017-12-01 21:50:05 +0800 | [diff] [blame] | 1468 | /* |
| 1469 | * Handled on controll processor until the plugged processor manages |
| 1470 | * this itself. |
| 1471 | */ |
| 1472 | [CPUHP_TEARDOWN_CPU] = { |
| 1473 | .name = "cpu:teardown", |
| 1474 | .startup.single = NULL, |
| 1475 | .teardown.single = takedown_cpu, |
| 1476 | .cant_stop = true, |
| 1477 | }, |
Thomas Gleixner | d10ef6f | 2016-03-08 10:36:13 +0100 | [diff] [blame] | 1478 | /* Handle smpboot threads park/unpark */ |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1479 | [CPUHP_AP_SMPBOOT_THREADS] = { |
Thomas Gleixner | 677f664 | 2016-09-06 16:13:48 +0200 | [diff] [blame] | 1480 | .name = "smpboot/threads:online", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1481 | .startup.single = smpboot_unpark_threads, |
Thomas Gleixner | c4de656 | 2018-05-29 19:05:25 +0200 | [diff] [blame] | 1482 | .teardown.single = smpboot_park_threads, |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1483 | }, |
Thomas Gleixner | c5cb83b | 2017-06-20 01:37:51 +0200 | [diff] [blame] | 1484 | [CPUHP_AP_IRQ_AFFINITY_ONLINE] = { |
| 1485 | .name = "irq/affinity:online", |
| 1486 | .startup.single = irq_affinity_online_cpu, |
| 1487 | .teardown.single = NULL, |
| 1488 | }, |
Thomas Gleixner | 00e16c3 | 2016-07-13 17:16:09 +0000 | [diff] [blame] | 1489 | [CPUHP_AP_PERF_ONLINE] = { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1490 | .name = "perf:online", |
| 1491 | .startup.single = perf_event_init_cpu, |
| 1492 | .teardown.single = perf_event_exit_cpu, |
Thomas Gleixner | 00e16c3 | 2016-07-13 17:16:09 +0000 | [diff] [blame] | 1493 | }, |
Peter Zijlstra | 9cf5773 | 2018-06-07 10:52:03 +0200 | [diff] [blame] | 1494 | [CPUHP_AP_WATCHDOG_ONLINE] = { |
| 1495 | .name = "lockup_detector:online", |
| 1496 | .startup.single = lockup_detector_online_cpu, |
| 1497 | .teardown.single = lockup_detector_offline_cpu, |
| 1498 | }, |
Thomas Gleixner | 7ee681b | 2016-07-13 17:16:29 +0000 | [diff] [blame] | 1499 | [CPUHP_AP_WORKQUEUE_ONLINE] = { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1500 | .name = "workqueue:online", |
| 1501 | .startup.single = workqueue_online_cpu, |
| 1502 | .teardown.single = workqueue_offline_cpu, |
Thomas Gleixner | 7ee681b | 2016-07-13 17:16:29 +0000 | [diff] [blame] | 1503 | }, |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 1504 | [CPUHP_AP_RCUTREE_ONLINE] = { |
Thomas Gleixner | 677f664 | 2016-09-06 16:13:48 +0200 | [diff] [blame] | 1505 | .name = "RCU/tree:online", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1506 | .startup.single = rcutree_online_cpu, |
| 1507 | .teardown.single = rcutree_offline_cpu, |
Thomas Gleixner | 4df8374 | 2016-07-13 17:17:03 +0000 | [diff] [blame] | 1508 | }, |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1509 | #endif |
Thomas Gleixner | d10ef6f | 2016-03-08 10:36:13 +0100 | [diff] [blame] | 1510 | /* |
| 1511 | * The dynamically registered state space is here |
| 1512 | */ |
| 1513 | |
Thomas Gleixner | aaddd7d | 2016-03-10 12:54:19 +0100 | [diff] [blame] | 1514 | #ifdef CONFIG_SMP |
| 1515 | /* Last state is scheduler control setting the cpu active */ |
| 1516 | [CPUHP_AP_ACTIVE] = { |
| 1517 | .name = "sched:active", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1518 | .startup.single = sched_cpu_activate, |
| 1519 | .teardown.single = sched_cpu_deactivate, |
Thomas Gleixner | aaddd7d | 2016-03-10 12:54:19 +0100 | [diff] [blame] | 1520 | }, |
| 1521 | #endif |
| 1522 | |
Thomas Gleixner | d10ef6f | 2016-03-08 10:36:13 +0100 | [diff] [blame] | 1523 | /* CPU is fully up and running. */ |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1524 | [CPUHP_ONLINE] = { |
| 1525 | .name = "online", |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1526 | .startup.single = NULL, |
| 1527 | .teardown.single = NULL, |
Thomas Gleixner | 4baa0af | 2016-02-26 18:43:29 +0000 | [diff] [blame] | 1528 | }, |
| 1529 | }; |
| 1530 | |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1531 | /* Sanity check for callbacks */ |
| 1532 | static int cpuhp_cb_check(enum cpuhp_state state) |
| 1533 | { |
| 1534 | if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE) |
| 1535 | return -EINVAL; |
| 1536 | return 0; |
| 1537 | } |
| 1538 | |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1539 | /* |
| 1540 | * Returns a free for dynamic slot assignment of the Online state. The states |
| 1541 | * are protected by the cpuhp_slot_states mutex and an empty slot is identified |
| 1542 | * by having no name assigned. |
| 1543 | */ |
| 1544 | static int cpuhp_reserve_state(enum cpuhp_state state) |
| 1545 | { |
Thomas Gleixner | 4205e47 | 2017-01-10 14:01:05 +0100 | [diff] [blame] | 1546 | enum cpuhp_state i, end; |
| 1547 | struct cpuhp_step *step; |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1548 | |
Thomas Gleixner | 4205e47 | 2017-01-10 14:01:05 +0100 | [diff] [blame] | 1549 | switch (state) { |
| 1550 | case CPUHP_AP_ONLINE_DYN: |
Lai Jiangshan | 17a2f1c | 2017-12-01 21:50:05 +0800 | [diff] [blame] | 1551 | step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN; |
Thomas Gleixner | 4205e47 | 2017-01-10 14:01:05 +0100 | [diff] [blame] | 1552 | end = CPUHP_AP_ONLINE_DYN_END; |
| 1553 | break; |
| 1554 | case CPUHP_BP_PREPARE_DYN: |
Lai Jiangshan | 17a2f1c | 2017-12-01 21:50:05 +0800 | [diff] [blame] | 1555 | step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN; |
Thomas Gleixner | 4205e47 | 2017-01-10 14:01:05 +0100 | [diff] [blame] | 1556 | end = CPUHP_BP_PREPARE_DYN_END; |
| 1557 | break; |
| 1558 | default: |
| 1559 | return -EINVAL; |
| 1560 | } |
| 1561 | |
| 1562 | for (i = state; i <= end; i++, step++) { |
| 1563 | if (!step->name) |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1564 | return i; |
| 1565 | } |
| 1566 | WARN(1, "No more dynamic states available for CPU hotplug\n"); |
| 1567 | return -ENOSPC; |
| 1568 | } |
| 1569 | |
| 1570 | static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name, |
| 1571 | int (*startup)(unsigned int cpu), |
| 1572 | int (*teardown)(unsigned int cpu), |
| 1573 | bool multi_instance) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1574 | { |
| 1575 | /* (Un)Install the callbacks for further cpu hotplug operations */ |
| 1576 | struct cpuhp_step *sp; |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1577 | int ret = 0; |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1578 | |
Ethan Barnes | 0c96b27 | 2017-07-19 22:36:00 +0000 | [diff] [blame] | 1579 | /* |
| 1580 | * If name is NULL, then the state gets removed. |
| 1581 | * |
| 1582 | * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on |
| 1583 | * the first allocation from these dynamic ranges, so the removal |
| 1584 | * would trigger a new allocation and clear the wrong (already |
| 1585 | * empty) state, leaving the callbacks of the to be cleared state |
| 1586 | * dangling, which causes wreckage on the next hotplug operation. |
| 1587 | */ |
| 1588 | if (name && (state == CPUHP_AP_ONLINE_DYN || |
| 1589 | state == CPUHP_BP_PREPARE_DYN)) { |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1590 | ret = cpuhp_reserve_state(state); |
| 1591 | if (ret < 0) |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1592 | return ret; |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1593 | state = ret; |
| 1594 | } |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1595 | sp = cpuhp_get_step(state); |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1596 | if (name && sp->name) |
| 1597 | return -EBUSY; |
| 1598 | |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1599 | sp->startup.single = startup; |
| 1600 | sp->teardown.single = teardown; |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1601 | sp->name = name; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1602 | sp->multi_instance = multi_instance; |
| 1603 | INIT_HLIST_HEAD(&sp->list); |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1604 | return ret; |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1605 | } |
| 1606 | |
| 1607 | static void *cpuhp_get_teardown_cb(enum cpuhp_state state) |
| 1608 | { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1609 | return cpuhp_get_step(state)->teardown.single; |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1610 | } |
| 1611 | |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1612 | /* |
| 1613 | * Call the startup/teardown function for a step either on the AP or |
| 1614 | * on the current CPU. |
| 1615 | */ |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1616 | static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup, |
| 1617 | struct hlist_node *node) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1618 | { |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 1619 | struct cpuhp_step *sp = cpuhp_get_step(state); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1620 | int ret; |
| 1621 | |
Peter Zijlstra | 4dddfb5 | 2017-09-20 19:00:17 +0200 | [diff] [blame] | 1622 | /* |
| 1623 | * If there's nothing to do, we done. |
| 1624 | * Relies on the union for multi_instance. |
| 1625 | */ |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1626 | if ((bringup && !sp->startup.single) || |
| 1627 | (!bringup && !sp->teardown.single)) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1628 | return 0; |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1629 | /* |
| 1630 | * The non AP bound callbacks can fail on bringup. On teardown |
| 1631 | * e.g. module removal we crash for now. |
| 1632 | */ |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1633 | #ifdef CONFIG_SMP |
| 1634 | if (cpuhp_is_ap_state(state)) |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1635 | ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node); |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1636 | else |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 1637 | ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1638 | #else |
Peter Zijlstra | 96abb96 | 2017-09-20 19:00:16 +0200 | [diff] [blame] | 1639 | ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); |
Thomas Gleixner | 1cf4f62 | 2016-02-26 18:43:39 +0000 | [diff] [blame] | 1640 | #endif |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1641 | BUG_ON(ret && !bringup); |
| 1642 | return ret; |
| 1643 | } |
| 1644 | |
| 1645 | /* |
| 1646 | * Called from __cpuhp_setup_state on a recoverable failure. |
| 1647 | * |
| 1648 | * Note: The teardown callbacks for rollback are not allowed to fail! |
| 1649 | */ |
| 1650 | static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state, |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1651 | struct hlist_node *node) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1652 | { |
| 1653 | int cpu; |
| 1654 | |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1655 | /* Roll back the already executed steps on the other cpus */ |
| 1656 | for_each_present_cpu(cpu) { |
| 1657 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 1658 | int cpustate = st->state; |
| 1659 | |
| 1660 | if (cpu >= failedcpu) |
| 1661 | break; |
| 1662 | |
| 1663 | /* Did we invoke the startup call on that cpu ? */ |
| 1664 | if (cpustate >= state) |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1665 | cpuhp_issue_call(cpu, state, false, node); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1666 | } |
| 1667 | } |
| 1668 | |
Thomas Gleixner | 9805c67 | 2017-05-24 10:15:15 +0200 | [diff] [blame] | 1669 | int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state, |
| 1670 | struct hlist_node *node, |
| 1671 | bool invoke) |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1672 | { |
| 1673 | struct cpuhp_step *sp; |
| 1674 | int cpu; |
| 1675 | int ret; |
| 1676 | |
Thomas Gleixner | 9805c67 | 2017-05-24 10:15:15 +0200 | [diff] [blame] | 1677 | lockdep_assert_cpus_held(); |
| 1678 | |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1679 | sp = cpuhp_get_step(state); |
| 1680 | if (sp->multi_instance == false) |
| 1681 | return -EINVAL; |
| 1682 | |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1683 | mutex_lock(&cpuhp_state_mutex); |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1684 | |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1685 | if (!invoke || !sp->startup.multi) |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1686 | goto add_node; |
| 1687 | |
| 1688 | /* |
| 1689 | * Try to call the startup callback for each present cpu |
| 1690 | * depending on the hotplug state of the cpu. |
| 1691 | */ |
| 1692 | for_each_present_cpu(cpu) { |
| 1693 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 1694 | int cpustate = st->state; |
| 1695 | |
| 1696 | if (cpustate < state) |
| 1697 | continue; |
| 1698 | |
| 1699 | ret = cpuhp_issue_call(cpu, state, true, node); |
| 1700 | if (ret) { |
Thomas Gleixner | 3c1627e | 2016-09-05 15:28:36 +0200 | [diff] [blame] | 1701 | if (sp->teardown.multi) |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1702 | cpuhp_rollback_install(cpu, state, node); |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1703 | goto unlock; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1704 | } |
| 1705 | } |
| 1706 | add_node: |
| 1707 | ret = 0; |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1708 | hlist_add_head(node, &sp->list); |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1709 | unlock: |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1710 | mutex_unlock(&cpuhp_state_mutex); |
Thomas Gleixner | 9805c67 | 2017-05-24 10:15:15 +0200 | [diff] [blame] | 1711 | return ret; |
| 1712 | } |
| 1713 | |
| 1714 | int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node, |
| 1715 | bool invoke) |
| 1716 | { |
| 1717 | int ret; |
| 1718 | |
| 1719 | cpus_read_lock(); |
| 1720 | ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke); |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 1721 | cpus_read_unlock(); |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1722 | return ret; |
| 1723 | } |
| 1724 | EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance); |
| 1725 | |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1726 | /** |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1727 | * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1728 | * @state: The state to setup |
| 1729 | * @invoke: If true, the startup function is invoked for cpus where |
| 1730 | * cpu state >= @state |
| 1731 | * @startup: startup callback function |
| 1732 | * @teardown: teardown callback function |
| 1733 | * @multi_instance: State is set up for multiple instances which get |
| 1734 | * added afterwards. |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1735 | * |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1736 | * The caller needs to hold cpus read locked while calling this function. |
Boris Ostrovsky | 512f098 | 2016-12-15 10:00:57 -0500 | [diff] [blame] | 1737 | * Returns: |
| 1738 | * On success: |
| 1739 | * Positive state number if @state is CPUHP_AP_ONLINE_DYN |
| 1740 | * 0 for all other states |
| 1741 | * On failure: proper (negative) error code |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1742 | */ |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1743 | int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state, |
| 1744 | const char *name, bool invoke, |
| 1745 | int (*startup)(unsigned int cpu), |
| 1746 | int (*teardown)(unsigned int cpu), |
| 1747 | bool multi_instance) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1748 | { |
| 1749 | int cpu, ret = 0; |
Thomas Gleixner | b9d9d69 | 2016-12-26 22:58:19 +0100 | [diff] [blame] | 1750 | bool dynstate; |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1751 | |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1752 | lockdep_assert_cpus_held(); |
| 1753 | |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1754 | if (cpuhp_cb_check(state) || !name) |
| 1755 | return -EINVAL; |
| 1756 | |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1757 | mutex_lock(&cpuhp_state_mutex); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1758 | |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1759 | ret = cpuhp_store_callbacks(state, name, startup, teardown, |
| 1760 | multi_instance); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1761 | |
Thomas Gleixner | b9d9d69 | 2016-12-26 22:58:19 +0100 | [diff] [blame] | 1762 | dynstate = state == CPUHP_AP_ONLINE_DYN; |
| 1763 | if (ret > 0 && dynstate) { |
| 1764 | state = ret; |
| 1765 | ret = 0; |
| 1766 | } |
| 1767 | |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1768 | if (ret || !invoke || !startup) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1769 | goto out; |
| 1770 | |
| 1771 | /* |
| 1772 | * Try to call the startup callback for each present cpu |
| 1773 | * depending on the hotplug state of the cpu. |
| 1774 | */ |
| 1775 | for_each_present_cpu(cpu) { |
| 1776 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 1777 | int cpustate = st->state; |
| 1778 | |
| 1779 | if (cpustate < state) |
| 1780 | continue; |
| 1781 | |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1782 | ret = cpuhp_issue_call(cpu, state, true, NULL); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1783 | if (ret) { |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 1784 | if (teardown) |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1785 | cpuhp_rollback_install(cpu, state, NULL); |
| 1786 | cpuhp_store_callbacks(state, NULL, NULL, NULL, false); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1787 | goto out; |
| 1788 | } |
| 1789 | } |
| 1790 | out: |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1791 | mutex_unlock(&cpuhp_state_mutex); |
Thomas Gleixner | dc280d93 | 2016-12-21 20:19:49 +0100 | [diff] [blame] | 1792 | /* |
| 1793 | * If the requested state is CPUHP_AP_ONLINE_DYN, return the |
| 1794 | * dynamically allocated state in case of success. |
| 1795 | */ |
Thomas Gleixner | b9d9d69 | 2016-12-26 22:58:19 +0100 | [diff] [blame] | 1796 | if (!ret && dynstate) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1797 | return state; |
| 1798 | return ret; |
| 1799 | } |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1800 | EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked); |
| 1801 | |
| 1802 | int __cpuhp_setup_state(enum cpuhp_state state, |
| 1803 | const char *name, bool invoke, |
| 1804 | int (*startup)(unsigned int cpu), |
| 1805 | int (*teardown)(unsigned int cpu), |
| 1806 | bool multi_instance) |
| 1807 | { |
| 1808 | int ret; |
| 1809 | |
| 1810 | cpus_read_lock(); |
| 1811 | ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup, |
| 1812 | teardown, multi_instance); |
| 1813 | cpus_read_unlock(); |
| 1814 | return ret; |
| 1815 | } |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1816 | EXPORT_SYMBOL(__cpuhp_setup_state); |
| 1817 | |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1818 | int __cpuhp_state_remove_instance(enum cpuhp_state state, |
| 1819 | struct hlist_node *node, bool invoke) |
| 1820 | { |
| 1821 | struct cpuhp_step *sp = cpuhp_get_step(state); |
| 1822 | int cpu; |
| 1823 | |
| 1824 | BUG_ON(cpuhp_cb_check(state)); |
| 1825 | |
| 1826 | if (!sp->multi_instance) |
| 1827 | return -EINVAL; |
| 1828 | |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 1829 | cpus_read_lock(); |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1830 | mutex_lock(&cpuhp_state_mutex); |
| 1831 | |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1832 | if (!invoke || !cpuhp_get_teardown_cb(state)) |
| 1833 | goto remove; |
| 1834 | /* |
| 1835 | * Call the teardown callback for each present cpu depending |
| 1836 | * on the hotplug state of the cpu. This function is not |
| 1837 | * allowed to fail currently! |
| 1838 | */ |
| 1839 | for_each_present_cpu(cpu) { |
| 1840 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 1841 | int cpustate = st->state; |
| 1842 | |
| 1843 | if (cpustate >= state) |
| 1844 | cpuhp_issue_call(cpu, state, false, node); |
| 1845 | } |
| 1846 | |
| 1847 | remove: |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1848 | hlist_del(node); |
| 1849 | mutex_unlock(&cpuhp_state_mutex); |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 1850 | cpus_read_unlock(); |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1851 | |
| 1852 | return 0; |
| 1853 | } |
| 1854 | EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance); |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1855 | |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1856 | /** |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1857 | * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1858 | * @state: The state to remove |
| 1859 | * @invoke: If true, the teardown function is invoked for cpus where |
| 1860 | * cpu state >= @state |
| 1861 | * |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1862 | * The caller needs to hold cpus read locked while calling this function. |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1863 | * The teardown callback is currently not allowed to fail. Think |
| 1864 | * about module removal! |
| 1865 | */ |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1866 | void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1867 | { |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1868 | struct cpuhp_step *sp = cpuhp_get_step(state); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1869 | int cpu; |
| 1870 | |
| 1871 | BUG_ON(cpuhp_cb_check(state)); |
| 1872 | |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1873 | lockdep_assert_cpus_held(); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1874 | |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1875 | mutex_lock(&cpuhp_state_mutex); |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1876 | if (sp->multi_instance) { |
| 1877 | WARN(!hlist_empty(&sp->list), |
| 1878 | "Error: Removing state %d which has instances left.\n", |
| 1879 | state); |
| 1880 | goto remove; |
| 1881 | } |
| 1882 | |
Thomas Gleixner | a724632 | 2016-08-12 19:49:38 +0200 | [diff] [blame] | 1883 | if (!invoke || !cpuhp_get_teardown_cb(state)) |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1884 | goto remove; |
| 1885 | |
| 1886 | /* |
| 1887 | * Call the teardown callback for each present cpu depending |
| 1888 | * on the hotplug state of the cpu. This function is not |
| 1889 | * allowed to fail currently! |
| 1890 | */ |
| 1891 | for_each_present_cpu(cpu) { |
| 1892 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| 1893 | int cpustate = st->state; |
| 1894 | |
| 1895 | if (cpustate >= state) |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1896 | cpuhp_issue_call(cpu, state, false, NULL); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1897 | } |
| 1898 | remove: |
Thomas Gleixner | cf392d1 | 2016-08-12 19:49:39 +0200 | [diff] [blame] | 1899 | cpuhp_store_callbacks(state, NULL, NULL, NULL, false); |
Sebastian Andrzej Siewior | dc434e05 | 2017-03-14 16:06:45 +0100 | [diff] [blame] | 1900 | mutex_unlock(&cpuhp_state_mutex); |
Sebastian Andrzej Siewior | 71def42 | 2017-05-24 10:15:14 +0200 | [diff] [blame] | 1901 | } |
| 1902 | EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked); |
| 1903 | |
| 1904 | void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) |
| 1905 | { |
| 1906 | cpus_read_lock(); |
| 1907 | __cpuhp_remove_state_cpuslocked(state, invoke); |
Thomas Gleixner | 8f553c4 | 2017-05-24 10:15:12 +0200 | [diff] [blame] | 1908 | cpus_read_unlock(); |
Thomas Gleixner | 5b7aa87 | 2016-02-26 18:43:33 +0000 | [diff] [blame] | 1909 | } |
| 1910 | EXPORT_SYMBOL(__cpuhp_remove_state); |
| 1911 | |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 1912 | #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU) |
| 1913 | static ssize_t show_cpuhp_state(struct device *dev, |
| 1914 | struct device_attribute *attr, char *buf) |
| 1915 | { |
| 1916 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); |
| 1917 | |
| 1918 | return sprintf(buf, "%d\n", st->state); |
| 1919 | } |
| 1920 | static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL); |
| 1921 | |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1922 | static ssize_t write_cpuhp_target(struct device *dev, |
| 1923 | struct device_attribute *attr, |
| 1924 | const char *buf, size_t count) |
| 1925 | { |
| 1926 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); |
| 1927 | struct cpuhp_step *sp; |
| 1928 | int target, ret; |
| 1929 | |
| 1930 | ret = kstrtoint(buf, 10, &target); |
| 1931 | if (ret) |
| 1932 | return ret; |
| 1933 | |
| 1934 | #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL |
| 1935 | if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE) |
| 1936 | return -EINVAL; |
| 1937 | #else |
| 1938 | if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE) |
| 1939 | return -EINVAL; |
| 1940 | #endif |
| 1941 | |
| 1942 | ret = lock_device_hotplug_sysfs(); |
| 1943 | if (ret) |
| 1944 | return ret; |
| 1945 | |
| 1946 | mutex_lock(&cpuhp_state_mutex); |
| 1947 | sp = cpuhp_get_step(target); |
| 1948 | ret = !sp->name || sp->cant_stop ? -EINVAL : 0; |
| 1949 | mutex_unlock(&cpuhp_state_mutex); |
| 1950 | if (ret) |
Sebastian Andrzej Siewior | 40da1b1 | 2017-06-02 16:27:14 +0200 | [diff] [blame] | 1951 | goto out; |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1952 | |
| 1953 | if (st->state < target) |
| 1954 | ret = do_cpu_up(dev->id, target); |
| 1955 | else |
| 1956 | ret = do_cpu_down(dev->id, target); |
Sebastian Andrzej Siewior | 40da1b1 | 2017-06-02 16:27:14 +0200 | [diff] [blame] | 1957 | out: |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1958 | unlock_device_hotplug(); |
| 1959 | return ret ? ret : count; |
| 1960 | } |
| 1961 | |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 1962 | static ssize_t show_cpuhp_target(struct device *dev, |
| 1963 | struct device_attribute *attr, char *buf) |
| 1964 | { |
| 1965 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); |
| 1966 | |
| 1967 | return sprintf(buf, "%d\n", st->target); |
| 1968 | } |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 1969 | static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target); |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 1970 | |
Peter Zijlstra | 1db4948 | 2017-09-20 19:00:21 +0200 | [diff] [blame] | 1971 | |
| 1972 | static ssize_t write_cpuhp_fail(struct device *dev, |
| 1973 | struct device_attribute *attr, |
| 1974 | const char *buf, size_t count) |
| 1975 | { |
| 1976 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); |
| 1977 | struct cpuhp_step *sp; |
| 1978 | int fail, ret; |
| 1979 | |
| 1980 | ret = kstrtoint(buf, 10, &fail); |
| 1981 | if (ret) |
| 1982 | return ret; |
| 1983 | |
Eiichi Tsukata | 33d4a5a | 2019-06-27 11:47:32 +0900 | [diff] [blame] | 1984 | if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE) |
| 1985 | return -EINVAL; |
| 1986 | |
Peter Zijlstra | 1db4948 | 2017-09-20 19:00:21 +0200 | [diff] [blame] | 1987 | /* |
| 1988 | * Cannot fail STARTING/DYING callbacks. |
| 1989 | */ |
| 1990 | if (cpuhp_is_atomic_state(fail)) |
| 1991 | return -EINVAL; |
| 1992 | |
| 1993 | /* |
| 1994 | * Cannot fail anything that doesn't have callbacks. |
| 1995 | */ |
| 1996 | mutex_lock(&cpuhp_state_mutex); |
| 1997 | sp = cpuhp_get_step(fail); |
| 1998 | if (!sp->startup.single && !sp->teardown.single) |
| 1999 | ret = -EINVAL; |
| 2000 | mutex_unlock(&cpuhp_state_mutex); |
| 2001 | if (ret) |
| 2002 | return ret; |
| 2003 | |
| 2004 | st->fail = fail; |
| 2005 | |
| 2006 | return count; |
| 2007 | } |
| 2008 | |
| 2009 | static ssize_t show_cpuhp_fail(struct device *dev, |
| 2010 | struct device_attribute *attr, char *buf) |
| 2011 | { |
| 2012 | struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); |
| 2013 | |
| 2014 | return sprintf(buf, "%d\n", st->fail); |
| 2015 | } |
| 2016 | |
| 2017 | static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail); |
| 2018 | |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 2019 | static struct attribute *cpuhp_cpu_attrs[] = { |
| 2020 | &dev_attr_state.attr, |
| 2021 | &dev_attr_target.attr, |
Peter Zijlstra | 1db4948 | 2017-09-20 19:00:21 +0200 | [diff] [blame] | 2022 | &dev_attr_fail.attr, |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 2023 | NULL |
| 2024 | }; |
| 2025 | |
Arvind Yadav | 993647a | 2017-06-29 17:40:47 +0530 | [diff] [blame] | 2026 | static const struct attribute_group cpuhp_cpu_attr_group = { |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 2027 | .attrs = cpuhp_cpu_attrs, |
| 2028 | .name = "hotplug", |
| 2029 | NULL |
| 2030 | }; |
| 2031 | |
| 2032 | static ssize_t show_cpuhp_states(struct device *dev, |
| 2033 | struct device_attribute *attr, char *buf) |
| 2034 | { |
| 2035 | ssize_t cur, res = 0; |
| 2036 | int i; |
| 2037 | |
| 2038 | mutex_lock(&cpuhp_state_mutex); |
Thomas Gleixner | 757c989 | 2016-02-26 18:43:32 +0000 | [diff] [blame] | 2039 | for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 2040 | struct cpuhp_step *sp = cpuhp_get_step(i); |
| 2041 | |
| 2042 | if (sp->name) { |
| 2043 | cur = sprintf(buf, "%3d: %s\n", i, sp->name); |
| 2044 | buf += cur; |
| 2045 | res += cur; |
| 2046 | } |
| 2047 | } |
| 2048 | mutex_unlock(&cpuhp_state_mutex); |
| 2049 | return res; |
| 2050 | } |
| 2051 | static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL); |
| 2052 | |
| 2053 | static struct attribute *cpuhp_cpu_root_attrs[] = { |
| 2054 | &dev_attr_states.attr, |
| 2055 | NULL |
| 2056 | }; |
| 2057 | |
Arvind Yadav | 993647a | 2017-06-29 17:40:47 +0530 | [diff] [blame] | 2058 | static const struct attribute_group cpuhp_cpu_root_attr_group = { |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 2059 | .attrs = cpuhp_cpu_root_attrs, |
| 2060 | .name = "hotplug", |
| 2061 | NULL |
| 2062 | }; |
| 2063 | |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2064 | #ifdef CONFIG_HOTPLUG_SMT |
| 2065 | |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2066 | static void cpuhp_offline_cpu_device(unsigned int cpu) |
| 2067 | { |
| 2068 | struct device *dev = get_cpu_device(cpu); |
| 2069 | |
| 2070 | dev->offline = true; |
| 2071 | /* Tell user space about the state change */ |
| 2072 | kobject_uevent(&dev->kobj, KOBJ_OFFLINE); |
| 2073 | } |
| 2074 | |
Thomas Gleixner | 215af54 | 2018-07-07 11:40:18 +0200 | [diff] [blame] | 2075 | static void cpuhp_online_cpu_device(unsigned int cpu) |
| 2076 | { |
| 2077 | struct device *dev = get_cpu_device(cpu); |
| 2078 | |
| 2079 | dev->offline = false; |
| 2080 | /* Tell user space about the state change */ |
| 2081 | kobject_uevent(&dev->kobj, KOBJ_ONLINE); |
| 2082 | } |
| 2083 | |
Jiri Kosina | ec527c3 | 2019-05-30 00:09:39 +0200 | [diff] [blame] | 2084 | int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2085 | { |
| 2086 | int cpu, ret = 0; |
| 2087 | |
| 2088 | cpu_maps_update_begin(); |
| 2089 | for_each_online_cpu(cpu) { |
| 2090 | if (topology_is_primary_thread(cpu)) |
| 2091 | continue; |
| 2092 | ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE); |
| 2093 | if (ret) |
| 2094 | break; |
| 2095 | /* |
| 2096 | * As this needs to hold the cpu maps lock it's impossible |
| 2097 | * to call device_offline() because that ends up calling |
| 2098 | * cpu_down() which takes cpu maps lock. cpu maps lock |
| 2099 | * needs to be held as this might race against in kernel |
| 2100 | * abusers of the hotplug machinery (thermal management). |
| 2101 | * |
| 2102 | * So nothing would update device:offline state. That would |
| 2103 | * leave the sysfs entry stale and prevent onlining after |
| 2104 | * smt control has been changed to 'off' again. This is |
| 2105 | * called under the sysfs hotplug lock, so it is properly |
| 2106 | * serialized against the regular offline usage. |
| 2107 | */ |
| 2108 | cpuhp_offline_cpu_device(cpu); |
| 2109 | } |
Zhenzhong Duan | 34d66ca | 2019-01-17 02:10:59 -0800 | [diff] [blame] | 2110 | if (!ret) |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2111 | cpu_smt_control = ctrlval; |
| 2112 | cpu_maps_update_done(); |
| 2113 | return ret; |
| 2114 | } |
| 2115 | |
Jiri Kosina | ec527c3 | 2019-05-30 00:09:39 +0200 | [diff] [blame] | 2116 | int cpuhp_smt_enable(void) |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2117 | { |
Thomas Gleixner | 215af54 | 2018-07-07 11:40:18 +0200 | [diff] [blame] | 2118 | int cpu, ret = 0; |
| 2119 | |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2120 | cpu_maps_update_begin(); |
| 2121 | cpu_smt_control = CPU_SMT_ENABLED; |
Thomas Gleixner | 215af54 | 2018-07-07 11:40:18 +0200 | [diff] [blame] | 2122 | for_each_present_cpu(cpu) { |
| 2123 | /* Skip online CPUs and CPUs on offline nodes */ |
| 2124 | if (cpu_online(cpu) || !node_online(cpu_to_node(cpu))) |
| 2125 | continue; |
| 2126 | ret = _cpu_up(cpu, 0, CPUHP_ONLINE); |
| 2127 | if (ret) |
| 2128 | break; |
| 2129 | /* See comment in cpuhp_smt_disable() */ |
| 2130 | cpuhp_online_cpu_device(cpu); |
| 2131 | } |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2132 | cpu_maps_update_done(); |
Thomas Gleixner | 215af54 | 2018-07-07 11:40:18 +0200 | [diff] [blame] | 2133 | return ret; |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2134 | } |
| 2135 | |
Josh Poimboeuf | de7b77e | 2019-03-27 07:00:29 -0500 | [diff] [blame] | 2136 | |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2137 | static ssize_t |
Josh Poimboeuf | de7b77e | 2019-03-27 07:00:29 -0500 | [diff] [blame] | 2138 | __store_smt_control(struct device *dev, struct device_attribute *attr, |
| 2139 | const char *buf, size_t count) |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2140 | { |
| 2141 | int ctrlval, ret; |
| 2142 | |
| 2143 | if (sysfs_streq(buf, "on")) |
| 2144 | ctrlval = CPU_SMT_ENABLED; |
| 2145 | else if (sysfs_streq(buf, "off")) |
| 2146 | ctrlval = CPU_SMT_DISABLED; |
| 2147 | else if (sysfs_streq(buf, "forceoff")) |
| 2148 | ctrlval = CPU_SMT_FORCE_DISABLED; |
| 2149 | else |
| 2150 | return -EINVAL; |
| 2151 | |
| 2152 | if (cpu_smt_control == CPU_SMT_FORCE_DISABLED) |
| 2153 | return -EPERM; |
| 2154 | |
| 2155 | if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED) |
| 2156 | return -ENODEV; |
| 2157 | |
| 2158 | ret = lock_device_hotplug_sysfs(); |
| 2159 | if (ret) |
| 2160 | return ret; |
| 2161 | |
| 2162 | if (ctrlval != cpu_smt_control) { |
| 2163 | switch (ctrlval) { |
| 2164 | case CPU_SMT_ENABLED: |
Thomas Gleixner | 215af54 | 2018-07-07 11:40:18 +0200 | [diff] [blame] | 2165 | ret = cpuhp_smt_enable(); |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2166 | break; |
| 2167 | case CPU_SMT_DISABLED: |
| 2168 | case CPU_SMT_FORCE_DISABLED: |
| 2169 | ret = cpuhp_smt_disable(ctrlval); |
| 2170 | break; |
| 2171 | } |
| 2172 | } |
| 2173 | |
| 2174 | unlock_device_hotplug(); |
| 2175 | return ret ? ret : count; |
| 2176 | } |
Josh Poimboeuf | de7b77e | 2019-03-27 07:00:29 -0500 | [diff] [blame] | 2177 | |
| 2178 | #else /* !CONFIG_HOTPLUG_SMT */ |
| 2179 | static ssize_t |
| 2180 | __store_smt_control(struct device *dev, struct device_attribute *attr, |
| 2181 | const char *buf, size_t count) |
| 2182 | { |
| 2183 | return -ENODEV; |
| 2184 | } |
| 2185 | #endif /* CONFIG_HOTPLUG_SMT */ |
| 2186 | |
| 2187 | static const char *smt_states[] = { |
| 2188 | [CPU_SMT_ENABLED] = "on", |
| 2189 | [CPU_SMT_DISABLED] = "off", |
| 2190 | [CPU_SMT_FORCE_DISABLED] = "forceoff", |
| 2191 | [CPU_SMT_NOT_SUPPORTED] = "notsupported", |
| 2192 | [CPU_SMT_NOT_IMPLEMENTED] = "notimplemented", |
| 2193 | }; |
| 2194 | |
| 2195 | static ssize_t |
| 2196 | show_smt_control(struct device *dev, struct device_attribute *attr, char *buf) |
| 2197 | { |
| 2198 | const char *state = smt_states[cpu_smt_control]; |
| 2199 | |
| 2200 | return snprintf(buf, PAGE_SIZE - 2, "%s\n", state); |
| 2201 | } |
| 2202 | |
| 2203 | static ssize_t |
| 2204 | store_smt_control(struct device *dev, struct device_attribute *attr, |
| 2205 | const char *buf, size_t count) |
| 2206 | { |
| 2207 | return __store_smt_control(dev, attr, buf, count); |
| 2208 | } |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2209 | static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control); |
| 2210 | |
| 2211 | static ssize_t |
| 2212 | show_smt_active(struct device *dev, struct device_attribute *attr, char *buf) |
| 2213 | { |
Josh Poimboeuf | de7b77e | 2019-03-27 07:00:29 -0500 | [diff] [blame] | 2214 | return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active()); |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2215 | } |
| 2216 | static DEVICE_ATTR(active, 0444, show_smt_active, NULL); |
| 2217 | |
| 2218 | static struct attribute *cpuhp_smt_attrs[] = { |
| 2219 | &dev_attr_control.attr, |
| 2220 | &dev_attr_active.attr, |
| 2221 | NULL |
| 2222 | }; |
| 2223 | |
| 2224 | static const struct attribute_group cpuhp_smt_attr_group = { |
| 2225 | .attrs = cpuhp_smt_attrs, |
| 2226 | .name = "smt", |
| 2227 | NULL |
| 2228 | }; |
| 2229 | |
Josh Poimboeuf | de7b77e | 2019-03-27 07:00:29 -0500 | [diff] [blame] | 2230 | static int __init cpu_smt_sysfs_init(void) |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2231 | { |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2232 | return sysfs_create_group(&cpu_subsys.dev_root->kobj, |
| 2233 | &cpuhp_smt_attr_group); |
| 2234 | } |
| 2235 | |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 2236 | static int __init cpuhp_sysfs_init(void) |
| 2237 | { |
| 2238 | int cpu, ret; |
| 2239 | |
Josh Poimboeuf | de7b77e | 2019-03-27 07:00:29 -0500 | [diff] [blame] | 2240 | ret = cpu_smt_sysfs_init(); |
Thomas Gleixner | 05736e4 | 2018-05-29 17:48:27 +0200 | [diff] [blame] | 2241 | if (ret) |
| 2242 | return ret; |
| 2243 | |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 2244 | ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, |
| 2245 | &cpuhp_cpu_root_attr_group); |
| 2246 | if (ret) |
| 2247 | return ret; |
| 2248 | |
| 2249 | for_each_possible_cpu(cpu) { |
| 2250 | struct device *dev = get_cpu_device(cpu); |
| 2251 | |
| 2252 | if (!dev) |
| 2253 | continue; |
| 2254 | ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); |
| 2255 | if (ret) |
| 2256 | return ret; |
| 2257 | } |
| 2258 | return 0; |
| 2259 | } |
| 2260 | device_initcall(cpuhp_sysfs_init); |
Josh Poimboeuf | de7b77e | 2019-03-27 07:00:29 -0500 | [diff] [blame] | 2261 | #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */ |
Thomas Gleixner | 98f8cdc | 2016-02-26 18:43:31 +0000 | [diff] [blame] | 2262 | |
Linus Torvalds | e56b3bc | 2008-07-28 11:32:33 -0700 | [diff] [blame] | 2263 | /* |
| 2264 | * cpu_bit_bitmap[] is a special, "compressed" data structure that |
| 2265 | * represents all NR_CPUS bits binary values of 1<<nr. |
| 2266 | * |
Rusty Russell | e0b582e | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 2267 | * It is used by cpumask_of() to get a constant address to a CPU |
Linus Torvalds | e56b3bc | 2008-07-28 11:32:33 -0700 | [diff] [blame] | 2268 | * mask value that has a single bit set only. |
| 2269 | */ |
Mike Travis | b8d317d | 2008-07-24 18:21:29 -0700 | [diff] [blame] | 2270 | |
Linus Torvalds | e56b3bc | 2008-07-28 11:32:33 -0700 | [diff] [blame] | 2271 | /* cpu_bit_bitmap[0] is empty - so we can back into it */ |
Michael Rodriguez | 4d51985 | 2011-03-22 16:34:07 -0700 | [diff] [blame] | 2272 | #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x)) |
Linus Torvalds | e56b3bc | 2008-07-28 11:32:33 -0700 | [diff] [blame] | 2273 | #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) |
| 2274 | #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) |
| 2275 | #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) |
Mike Travis | b8d317d | 2008-07-24 18:21:29 -0700 | [diff] [blame] | 2276 | |
Linus Torvalds | e56b3bc | 2008-07-28 11:32:33 -0700 | [diff] [blame] | 2277 | const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { |
Mike Travis | b8d317d | 2008-07-24 18:21:29 -0700 | [diff] [blame] | 2278 | |
Linus Torvalds | e56b3bc | 2008-07-28 11:32:33 -0700 | [diff] [blame] | 2279 | MASK_DECLARE_8(0), MASK_DECLARE_8(8), |
| 2280 | MASK_DECLARE_8(16), MASK_DECLARE_8(24), |
| 2281 | #if BITS_PER_LONG > 32 |
| 2282 | MASK_DECLARE_8(32), MASK_DECLARE_8(40), |
| 2283 | MASK_DECLARE_8(48), MASK_DECLARE_8(56), |
Mike Travis | b8d317d | 2008-07-24 18:21:29 -0700 | [diff] [blame] | 2284 | #endif |
| 2285 | }; |
Linus Torvalds | e56b3bc | 2008-07-28 11:32:33 -0700 | [diff] [blame] | 2286 | EXPORT_SYMBOL_GPL(cpu_bit_bitmap); |
Rusty Russell | 2d3854a | 2008-11-05 13:39:10 +1100 | [diff] [blame] | 2287 | |
| 2288 | const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; |
| 2289 | EXPORT_SYMBOL(cpu_all_bits); |
Rusty Russell | b3199c0 | 2008-12-30 09:05:14 +1030 | [diff] [blame] | 2290 | |
| 2291 | #ifdef CONFIG_INIT_ALL_POSSIBLE |
Rasmus Villemoes | 4b804c8 | 2016-01-20 15:00:19 -0800 | [diff] [blame] | 2292 | struct cpumask __cpu_possible_mask __read_mostly |
Rasmus Villemoes | c4c54dd | 2016-01-20 15:00:16 -0800 | [diff] [blame] | 2293 | = {CPU_BITS_ALL}; |
Rusty Russell | b3199c0 | 2008-12-30 09:05:14 +1030 | [diff] [blame] | 2294 | #else |
Rasmus Villemoes | 4b804c8 | 2016-01-20 15:00:19 -0800 | [diff] [blame] | 2295 | struct cpumask __cpu_possible_mask __read_mostly; |
Rusty Russell | b3199c0 | 2008-12-30 09:05:14 +1030 | [diff] [blame] | 2296 | #endif |
Rasmus Villemoes | 4b804c8 | 2016-01-20 15:00:19 -0800 | [diff] [blame] | 2297 | EXPORT_SYMBOL(__cpu_possible_mask); |
Rusty Russell | b3199c0 | 2008-12-30 09:05:14 +1030 | [diff] [blame] | 2298 | |
Rasmus Villemoes | 4b804c8 | 2016-01-20 15:00:19 -0800 | [diff] [blame] | 2299 | struct cpumask __cpu_online_mask __read_mostly; |
| 2300 | EXPORT_SYMBOL(__cpu_online_mask); |
Rusty Russell | b3199c0 | 2008-12-30 09:05:14 +1030 | [diff] [blame] | 2301 | |
Rasmus Villemoes | 4b804c8 | 2016-01-20 15:00:19 -0800 | [diff] [blame] | 2302 | struct cpumask __cpu_present_mask __read_mostly; |
| 2303 | EXPORT_SYMBOL(__cpu_present_mask); |
Rusty Russell | b3199c0 | 2008-12-30 09:05:14 +1030 | [diff] [blame] | 2304 | |
Rasmus Villemoes | 4b804c8 | 2016-01-20 15:00:19 -0800 | [diff] [blame] | 2305 | struct cpumask __cpu_active_mask __read_mostly; |
| 2306 | EXPORT_SYMBOL(__cpu_active_mask); |
Rusty Russell | 3fa4152 | 2008-12-30 09:05:16 +1030 | [diff] [blame] | 2307 | |
Thomas Gleixner | 0c09ab9 | 2019-07-09 16:23:40 +0200 | [diff] [blame] | 2308 | atomic_t __num_online_cpus __read_mostly; |
| 2309 | EXPORT_SYMBOL(__num_online_cpus); |
| 2310 | |
Rusty Russell | 3fa4152 | 2008-12-30 09:05:16 +1030 | [diff] [blame] | 2311 | void init_cpu_present(const struct cpumask *src) |
| 2312 | { |
Rasmus Villemoes | c4c54dd | 2016-01-20 15:00:16 -0800 | [diff] [blame] | 2313 | cpumask_copy(&__cpu_present_mask, src); |
Rusty Russell | 3fa4152 | 2008-12-30 09:05:16 +1030 | [diff] [blame] | 2314 | } |
| 2315 | |
| 2316 | void init_cpu_possible(const struct cpumask *src) |
| 2317 | { |
Rasmus Villemoes | c4c54dd | 2016-01-20 15:00:16 -0800 | [diff] [blame] | 2318 | cpumask_copy(&__cpu_possible_mask, src); |
Rusty Russell | 3fa4152 | 2008-12-30 09:05:16 +1030 | [diff] [blame] | 2319 | } |
| 2320 | |
| 2321 | void init_cpu_online(const struct cpumask *src) |
| 2322 | { |
Rasmus Villemoes | c4c54dd | 2016-01-20 15:00:16 -0800 | [diff] [blame] | 2323 | cpumask_copy(&__cpu_online_mask, src); |
Rusty Russell | 3fa4152 | 2008-12-30 09:05:16 +1030 | [diff] [blame] | 2324 | } |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 2325 | |
Thomas Gleixner | 0c09ab9 | 2019-07-09 16:23:40 +0200 | [diff] [blame] | 2326 | void set_cpu_online(unsigned int cpu, bool online) |
| 2327 | { |
| 2328 | /* |
| 2329 | * atomic_inc/dec() is required to handle the horrid abuse of this |
| 2330 | * function by the reboot and kexec code which invoke it from |
| 2331 | * IPI/NMI broadcasts when shutting down CPUs. Invocation from |
| 2332 | * regular CPU hotplug is properly serialized. |
| 2333 | * |
| 2334 | * Note, that the fact that __num_online_cpus is of type atomic_t |
| 2335 | * does not protect readers which are not serialized against |
| 2336 | * concurrent hotplug operations. |
| 2337 | */ |
| 2338 | if (online) { |
| 2339 | if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask)) |
| 2340 | atomic_inc(&__num_online_cpus); |
| 2341 | } else { |
| 2342 | if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask)) |
| 2343 | atomic_dec(&__num_online_cpus); |
| 2344 | } |
| 2345 | } |
| 2346 | |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 2347 | /* |
| 2348 | * Activate the first processor. |
| 2349 | */ |
| 2350 | void __init boot_cpu_init(void) |
| 2351 | { |
| 2352 | int cpu = smp_processor_id(); |
| 2353 | |
| 2354 | /* Mark the boot cpu "present", "online" etc for SMP and UP case */ |
| 2355 | set_cpu_online(cpu, true); |
| 2356 | set_cpu_active(cpu, true); |
| 2357 | set_cpu_present(cpu, true); |
| 2358 | set_cpu_possible(cpu, true); |
Peter Zijlstra | 8ce371f | 2017-03-20 12:26:55 +0100 | [diff] [blame] | 2359 | |
| 2360 | #ifdef CONFIG_SMP |
| 2361 | __boot_cpu_id = cpu; |
| 2362 | #endif |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 2363 | } |
| 2364 | |
| 2365 | /* |
| 2366 | * Must be called _AFTER_ setting up the per_cpu areas |
| 2367 | */ |
Linus Torvalds | b5b1404 | 2018-08-12 12:19:42 -0700 | [diff] [blame] | 2368 | void __init boot_cpu_hotplug_init(void) |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 2369 | { |
Abel Vesa | 269777a | 2018-08-15 00:26:00 +0300 | [diff] [blame] | 2370 | #ifdef CONFIG_SMP |
Thomas Gleixner | e797bda | 2019-07-22 20:47:16 +0200 | [diff] [blame] | 2371 | cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask); |
Abel Vesa | 269777a | 2018-08-15 00:26:00 +0300 | [diff] [blame] | 2372 | #endif |
Thomas Gleixner | 0cc3cd2 | 2018-06-29 16:05:48 +0200 | [diff] [blame] | 2373 | this_cpu_write(cpuhp_state.state, CPUHP_ONLINE); |
Thomas Gleixner | cff7d37 | 2016-02-26 18:43:28 +0000 | [diff] [blame] | 2374 | } |
Josh Poimboeuf | 98af845 | 2019-04-12 15:39:28 -0500 | [diff] [blame] | 2375 | |
| 2376 | enum cpu_mitigations cpu_mitigations __ro_after_init = CPU_MITIGATIONS_AUTO; |
| 2377 | |
| 2378 | static int __init mitigations_parse_cmdline(char *arg) |
| 2379 | { |
| 2380 | if (!strcmp(arg, "off")) |
| 2381 | cpu_mitigations = CPU_MITIGATIONS_OFF; |
| 2382 | else if (!strcmp(arg, "auto")) |
| 2383 | cpu_mitigations = CPU_MITIGATIONS_AUTO; |
| 2384 | else if (!strcmp(arg, "auto,nosmt")) |
| 2385 | cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT; |
Geert Uytterhoeven | 1bf7272 | 2019-05-16 09:09:35 +0200 | [diff] [blame] | 2386 | else |
| 2387 | pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n", |
| 2388 | arg); |
Josh Poimboeuf | 98af845 | 2019-04-12 15:39:28 -0500 | [diff] [blame] | 2389 | |
| 2390 | return 0; |
| 2391 | } |
| 2392 | early_param("mitigations", mitigations_parse_cmdline); |