Thomas Gleixner | 457c899 | 2019-05-19 13:08:55 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | /* |
Peter Zijlstra | 391e43d | 2011-11-15 17:14:39 +0100 | [diff] [blame] | 3 | * kernel/sched/core.c |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 5 | * Core kernel scheduler code and related syscalls |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6 | * |
| 7 | * Copyright (C) 1991-2002 Linus Torvalds |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | */ |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 9 | #include <linux/highmem.h> |
| 10 | #include <linux/hrtimer_api.h> |
| 11 | #include <linux/ktime_api.h> |
| 12 | #include <linux/sched/signal.h> |
| 13 | #include <linux/syscalls_api.h> |
| 14 | #include <linux/debug_locks.h> |
| 15 | #include <linux/prefetch.h> |
| 16 | #include <linux/capability.h> |
| 17 | #include <linux/pgtable_api.h> |
| 18 | #include <linux/wait_bit.h> |
| 19 | #include <linux/jiffies.h> |
| 20 | #include <linux/spinlock_api.h> |
| 21 | #include <linux/cpumask_api.h> |
| 22 | #include <linux/lockdep_api.h> |
| 23 | #include <linux/hardirq.h> |
| 24 | #include <linux/softirq.h> |
| 25 | #include <linux/refcount_api.h> |
| 26 | #include <linux/topology.h> |
| 27 | #include <linux/sched/clock.h> |
| 28 | #include <linux/sched/cond_resched.h> |
Thomas Gleixner | d664e39 | 2022-04-13 15:31:02 +0200 | [diff] [blame] | 29 | #include <linux/sched/cputime.h> |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 30 | #include <linux/sched/debug.h> |
Thomas Gleixner | d664e39 | 2022-04-13 15:31:02 +0200 | [diff] [blame] | 31 | #include <linux/sched/hotplug.h> |
| 32 | #include <linux/sched/init.h> |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 33 | #include <linux/sched/isolation.h> |
| 34 | #include <linux/sched/loadavg.h> |
| 35 | #include <linux/sched/mm.h> |
| 36 | #include <linux/sched/nohz.h> |
| 37 | #include <linux/sched/rseq_api.h> |
| 38 | #include <linux/sched/rt.h> |
Phil Auld | 9d24605 | 2020-06-29 15:23:03 -0400 | [diff] [blame] | 39 | |
Christoph Hellwig | 6a5850d | 2021-09-20 14:33:18 +0200 | [diff] [blame] | 40 | #include <linux/blkdev.h> |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 41 | #include <linux/context_tracking.h> |
| 42 | #include <linux/cpuset.h> |
| 43 | #include <linux/delayacct.h> |
| 44 | #include <linux/init_task.h> |
| 45 | #include <linux/interrupt.h> |
| 46 | #include <linux/ioprio.h> |
| 47 | #include <linux/kallsyms.h> |
Mark Rutland | 0ed557a | 2018-06-14 15:27:41 -0700 | [diff] [blame] | 48 | #include <linux/kcov.h> |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 49 | #include <linux/kprobes.h> |
| 50 | #include <linux/llist_api.h> |
| 51 | #include <linux/mmu_context.h> |
| 52 | #include <linux/mmzone.h> |
| 53 | #include <linux/mutex_api.h> |
| 54 | #include <linux/nmi.h> |
| 55 | #include <linux/nospec.h> |
| 56 | #include <linux/perf_event_api.h> |
| 57 | #include <linux/profile.h> |
| 58 | #include <linux/psi.h> |
| 59 | #include <linux/rcuwait_api.h> |
| 60 | #include <linux/sched/wake_q.h> |
Sami Tolvanen | d08b9f0 | 2020-04-27 09:00:07 -0700 | [diff] [blame] | 61 | #include <linux/scs.h> |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 62 | #include <linux/slab.h> |
| 63 | #include <linux/syscalls.h> |
| 64 | #include <linux/vtime.h> |
| 65 | #include <linux/wait_api.h> |
| 66 | #include <linux/workqueue_api.h> |
| 67 | |
| 68 | #ifdef CONFIG_PREEMPT_DYNAMIC |
Ingo Molnar | a7b2553 | 2022-03-15 10:33:53 +0100 | [diff] [blame] | 69 | # ifdef CONFIG_GENERIC_ENTRY |
| 70 | # include <linux/entry-common.h> |
| 71 | # endif |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 72 | #endif |
| 73 | |
| 74 | #include <uapi/linux/sched/types.h> |
Mark Rutland | 0ed557a | 2018-06-14 15:27:41 -0700 | [diff] [blame] | 75 | |
Zhen Lei | bc1cca9 | 2022-08-04 10:34:20 +0800 | [diff] [blame] | 76 | #include <asm/irq_regs.h> |
David Howells | 96f951e | 2012-03-28 18:30:03 +0100 | [diff] [blame] | 77 | #include <asm/switch_to.h> |
Eric Dumazet | 5517d86 | 2007-05-08 00:32:57 -0700 | [diff] [blame] | 78 | #include <asm/tlb.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 79 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 80 | #define CREATE_TRACE_POINTS |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 81 | #include <linux/sched/rseq_api.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | #include <trace/events/sched.h> |
| 83 | #undef CREATE_TRACE_POINTS |
| 84 | |
| 85 | #include "sched.h" |
Ingo Molnar | b9e9c6ca | 2022-02-13 08:19:43 +0100 | [diff] [blame] | 86 | #include "stats.h" |
| 87 | #include "autogroup.h" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 88 | |
Ingo Molnar | e66f648 | 2022-02-23 08:17:15 +0100 | [diff] [blame] | 89 | #include "autogroup.h" |
| 90 | #include "pelt.h" |
| 91 | #include "smp.h" |
| 92 | #include "stats.h" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | |
Tejun Heo | ea13844 | 2013-01-18 14:05:55 -0800 | [diff] [blame] | 94 | #include "../workqueue_internal.h" |
Jens Axboe | ed29b0b | 2022-05-23 17:05:03 -0600 | [diff] [blame] | 95 | #include "../../io_uring/io-wq.h" |
Thomas Gleixner | 29d5e04 | 2012-04-20 13:05:45 +0000 | [diff] [blame] | 96 | #include "../smpboot.h" |
Gregory Haskins | 6e0534f | 2008-05-12 21:21:01 +0200 | [diff] [blame] | 97 | |
Qais Yousef | a056a5b | 2019-06-04 12:14:59 +0100 | [diff] [blame] | 98 | /* |
| 99 | * Export tracepoints that act as a bare tracehook (ie: have no trace event |
| 100 | * associated with them) to allow external modules to probe them. |
| 101 | */ |
| 102 | EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); |
| 103 | EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); |
| 104 | EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); |
| 105 | EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); |
| 106 | EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); |
Qais Yousef | 77cf151 | 2021-10-28 12:50:05 +0100 | [diff] [blame] | 107 | EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_thermal_tp); |
Vincent Donnefort | 51cf18c | 2020-08-28 10:00:49 +0100 | [diff] [blame] | 108 | EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp); |
Qais Yousef | a056a5b | 2019-06-04 12:14:59 +0100 | [diff] [blame] | 109 | EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); |
Vincent Donnefort | 4581bea | 2020-05-27 17:39:14 +0100 | [diff] [blame] | 110 | EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); |
| 111 | EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); |
Phil Auld | 9d24605 | 2020-06-29 15:23:03 -0400 | [diff] [blame] | 112 | EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); |
Qais Yousef | a056a5b | 2019-06-04 12:14:59 +0100 | [diff] [blame] | 113 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 114 | DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); |
Peter Zijlstra | dc61b1d | 2010-06-08 11:40:42 +0200 | [diff] [blame] | 115 | |
Juri Lelli | a73f863 | 2020-10-13 07:31:14 +0200 | [diff] [blame] | 116 | #ifdef CONFIG_SCHED_DEBUG |
Ingo Molnar | e436d80 | 2007-07-19 21:28:35 +0200 | [diff] [blame] | 117 | /* |
Ingo Molnar | bf5c91b | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 118 | * Debugging: various feature bits |
Patrick Bellasi | 765cc3a | 2017-11-08 18:41:01 +0000 | [diff] [blame] | 119 | * |
| 120 | * If SCHED_DEBUG is disabled, each compilation unit has its own copy of |
| 121 | * sysctl_sched_features, defined in sched.h, to allow constants propagation |
| 122 | * at compile time and compiler optimization based on features default. |
Ingo Molnar | bf5c91b | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 123 | */ |
Peter Zijlstra | f00b45c | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 124 | #define SCHED_FEAT(name, enabled) \ |
Peter Zijlstra | f00b45c | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 125 | (1UL << __SCHED_FEAT_##name) * enabled | |
Peter Zijlstra | f00b45c | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 126 | const_debug unsigned int sysctl_sched_features = |
Peter Zijlstra | 391e43d | 2011-11-15 17:14:39 +0100 | [diff] [blame] | 127 | #include "features.h" |
Peter Zijlstra | f00b45c | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 128 | 0; |
Peter Zijlstra | f00b45c | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 129 | #undef SCHED_FEAT |
Paul Turner | c006fac | 2021-04-16 14:29:36 -0700 | [diff] [blame] | 130 | |
| 131 | /* |
| 132 | * Print a warning if need_resched is set for the given duration (if |
| 133 | * LATENCY_WARN is enabled). |
| 134 | * |
| 135 | * If sysctl_resched_latency_warn_once is set, only one warning will be shown |
| 136 | * per boot. |
| 137 | */ |
| 138 | __read_mostly int sysctl_resched_latency_warn_ms = 100; |
| 139 | __read_mostly int sysctl_resched_latency_warn_once = 1; |
| 140 | #endif /* CONFIG_SCHED_DEBUG */ |
Peter Zijlstra | f00b45c | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 141 | |
Ingo Molnar | bf5c91b | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 142 | /* |
Peter Zijlstra | b82d9fd | 2007-11-09 22:39:39 +0100 | [diff] [blame] | 143 | * Number of tasks to iterate in a single balance run. |
| 144 | * Limited because this is done with IRQs disabled. |
| 145 | */ |
Vincent Guittot | c59862f | 2022-08-25 14:27:24 +0200 | [diff] [blame] | 146 | const_debug unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK; |
Peter Zijlstra | b82d9fd | 2007-11-09 22:39:39 +0100 | [diff] [blame] | 147 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 148 | __read_mostly int scheduler_running; |
Ingo Molnar | 6892b75 | 2008-02-13 14:02:36 +0100 | [diff] [blame] | 149 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 150 | #ifdef CONFIG_SCHED_CORE |
| 151 | |
| 152 | DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); |
| 153 | |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 154 | /* kernel prio, less is more */ |
| 155 | static inline int __task_prio(struct task_struct *p) |
| 156 | { |
| 157 | if (p->sched_class == &stop_sched_class) /* trumps deadline */ |
| 158 | return -2; |
| 159 | |
| 160 | if (rt_prio(p->prio)) /* includes deadline */ |
| 161 | return p->prio; /* [-1, 99] */ |
| 162 | |
| 163 | if (p->sched_class == &idle_sched_class) |
| 164 | return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ |
| 165 | |
| 166 | return MAX_RT_PRIO + MAX_NICE; /* 120, squash fair */ |
| 167 | } |
| 168 | |
| 169 | /* |
| 170 | * l(a,b) |
| 171 | * le(a,b) := !l(b,a) |
| 172 | * g(a,b) := l(b,a) |
| 173 | * ge(a,b) := !l(a,b) |
| 174 | */ |
| 175 | |
| 176 | /* real prio, less is less */ |
Joel Fernandes (Google) | c6047c2 | 2020-11-17 18:19:39 -0500 | [diff] [blame] | 177 | static inline bool prio_less(struct task_struct *a, struct task_struct *b, bool in_fi) |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 178 | { |
| 179 | |
| 180 | int pa = __task_prio(a), pb = __task_prio(b); |
| 181 | |
| 182 | if (-pa < -pb) |
| 183 | return true; |
| 184 | |
| 185 | if (-pb < -pa) |
| 186 | return false; |
| 187 | |
| 188 | if (pa == -1) /* dl_prio() doesn't work because of stop_class above */ |
| 189 | return !dl_time_before(a->dl.deadline, b->dl.deadline); |
| 190 | |
Joel Fernandes (Google) | c6047c2 | 2020-11-17 18:19:39 -0500 | [diff] [blame] | 191 | if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ |
| 192 | return cfs_prio_less(a, b, in_fi); |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 193 | |
| 194 | return false; |
| 195 | } |
| 196 | |
| 197 | static inline bool __sched_core_less(struct task_struct *a, struct task_struct *b) |
| 198 | { |
| 199 | if (a->core_cookie < b->core_cookie) |
| 200 | return true; |
| 201 | |
| 202 | if (a->core_cookie > b->core_cookie) |
| 203 | return false; |
| 204 | |
| 205 | /* flip prio, so high prio is leftmost */ |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 206 | if (prio_less(b, a, !!task_rq(a)->core->core_forceidle_count)) |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 207 | return true; |
| 208 | |
| 209 | return false; |
| 210 | } |
| 211 | |
| 212 | #define __node_2_sc(node) rb_entry((node), struct task_struct, core_node) |
| 213 | |
| 214 | static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b) |
| 215 | { |
| 216 | return __sched_core_less(__node_2_sc(a), __node_2_sc(b)); |
| 217 | } |
| 218 | |
| 219 | static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) |
| 220 | { |
| 221 | const struct task_struct *p = __node_2_sc(node); |
| 222 | unsigned long cookie = (unsigned long)key; |
| 223 | |
| 224 | if (cookie < p->core_cookie) |
| 225 | return -1; |
| 226 | |
| 227 | if (cookie > p->core_cookie) |
| 228 | return 1; |
| 229 | |
| 230 | return 0; |
| 231 | } |
| 232 | |
Peter Zijlstra | 6e33cad | 2021-03-26 18:55:06 +0100 | [diff] [blame] | 233 | void sched_core_enqueue(struct rq *rq, struct task_struct *p) |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 234 | { |
| 235 | rq->core->core_task_seq++; |
| 236 | |
| 237 | if (!p->core_cookie) |
| 238 | return; |
| 239 | |
| 240 | rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); |
| 241 | } |
| 242 | |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 243 | void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 244 | { |
| 245 | rq->core->core_task_seq++; |
| 246 | |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 247 | if (sched_core_enqueued(p)) { |
| 248 | rb_erase(&p->core_node, &rq->core_tree); |
| 249 | RB_CLEAR_NODE(&p->core_node); |
| 250 | } |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 251 | |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 252 | /* |
| 253 | * Migrating the last task off the cpu, with the cpu in forced idle |
| 254 | * state. Reschedule to create an accounting edge for forced idle, |
| 255 | * and re-examine whether the core is still in forced idle state. |
| 256 | */ |
| 257 | if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && |
| 258 | rq->core->core_forceidle_count && rq->curr == rq->idle) |
| 259 | resched_curr(rq); |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 260 | } |
| 261 | |
| 262 | /* |
| 263 | * Find left-most (aka, highest priority) task matching @cookie. |
| 264 | */ |
| 265 | static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) |
| 266 | { |
| 267 | struct rb_node *node; |
| 268 | |
| 269 | node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); |
| 270 | /* |
| 271 | * The idle task always matches any cookie! |
| 272 | */ |
| 273 | if (!node) |
| 274 | return idle_sched_class.pick_task(rq); |
| 275 | |
| 276 | return __node_2_sc(node); |
| 277 | } |
| 278 | |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 279 | static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie) |
| 280 | { |
| 281 | struct rb_node *node = &p->core_node; |
| 282 | |
| 283 | node = rb_next(node); |
| 284 | if (!node) |
| 285 | return NULL; |
| 286 | |
| 287 | p = container_of(node, struct task_struct, core_node); |
| 288 | if (p->core_cookie != cookie) |
| 289 | return NULL; |
| 290 | |
| 291 | return p; |
| 292 | } |
| 293 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 294 | /* |
| 295 | * Magic required such that: |
| 296 | * |
| 297 | * raw_spin_rq_lock(rq); |
| 298 | * ... |
| 299 | * raw_spin_rq_unlock(rq); |
| 300 | * |
| 301 | * ends up locking and unlocking the _same_ lock, and all CPUs |
| 302 | * always agree on what rq has what lock. |
| 303 | * |
| 304 | * XXX entirely possible to selectively enable cores, don't bother for now. |
| 305 | */ |
| 306 | |
| 307 | static DEFINE_MUTEX(sched_core_mutex); |
Peter Zijlstra | 875feb4 | 2021-03-29 10:08:58 +0200 | [diff] [blame] | 308 | static atomic_t sched_core_count; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 309 | static struct cpumask sched_core_mask; |
| 310 | |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 311 | static void sched_core_lock(int cpu, unsigned long *flags) |
| 312 | { |
| 313 | const struct cpumask *smt_mask = cpu_smt_mask(cpu); |
| 314 | int t, i = 0; |
| 315 | |
| 316 | local_irq_save(*flags); |
| 317 | for_each_cpu(t, smt_mask) |
| 318 | raw_spin_lock_nested(&cpu_rq(t)->__lock, i++); |
| 319 | } |
| 320 | |
| 321 | static void sched_core_unlock(int cpu, unsigned long *flags) |
| 322 | { |
| 323 | const struct cpumask *smt_mask = cpu_smt_mask(cpu); |
| 324 | int t; |
| 325 | |
| 326 | for_each_cpu(t, smt_mask) |
| 327 | raw_spin_unlock(&cpu_rq(t)->__lock); |
| 328 | local_irq_restore(*flags); |
| 329 | } |
| 330 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 331 | static void __sched_core_flip(bool enabled) |
| 332 | { |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 333 | unsigned long flags; |
| 334 | int cpu, t; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 335 | |
| 336 | cpus_read_lock(); |
| 337 | |
| 338 | /* |
| 339 | * Toggle the online cores, one by one. |
| 340 | */ |
| 341 | cpumask_copy(&sched_core_mask, cpu_online_mask); |
| 342 | for_each_cpu(cpu, &sched_core_mask) { |
| 343 | const struct cpumask *smt_mask = cpu_smt_mask(cpu); |
| 344 | |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 345 | sched_core_lock(cpu, &flags); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 346 | |
| 347 | for_each_cpu(t, smt_mask) |
| 348 | cpu_rq(t)->core_enabled = enabled; |
| 349 | |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 350 | cpu_rq(cpu)->core->core_forceidle_start = 0; |
| 351 | |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 352 | sched_core_unlock(cpu, &flags); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 353 | |
| 354 | cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask); |
| 355 | } |
| 356 | |
| 357 | /* |
| 358 | * Toggle the offline CPUs. |
| 359 | */ |
Valentin Schneider | 585463f0 | 2022-10-03 16:34:20 +0100 | [diff] [blame] | 360 | for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask) |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 361 | cpu_rq(cpu)->core_enabled = enabled; |
| 362 | |
| 363 | cpus_read_unlock(); |
| 364 | } |
| 365 | |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 366 | static void sched_core_assert_empty(void) |
| 367 | { |
| 368 | int cpu; |
| 369 | |
| 370 | for_each_possible_cpu(cpu) |
| 371 | WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree)); |
| 372 | } |
| 373 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 374 | static void __sched_core_enable(void) |
| 375 | { |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 376 | static_branch_enable(&__sched_core_enabled); |
| 377 | /* |
| 378 | * Ensure all previous instances of raw_spin_rq_*lock() have finished |
| 379 | * and future ones will observe !sched_core_disabled(). |
| 380 | */ |
| 381 | synchronize_rcu(); |
| 382 | __sched_core_flip(true); |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 383 | sched_core_assert_empty(); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 384 | } |
| 385 | |
| 386 | static void __sched_core_disable(void) |
| 387 | { |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 388 | sched_core_assert_empty(); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 389 | __sched_core_flip(false); |
| 390 | static_branch_disable(&__sched_core_enabled); |
| 391 | } |
| 392 | |
| 393 | void sched_core_get(void) |
| 394 | { |
Peter Zijlstra | 875feb4 | 2021-03-29 10:08:58 +0200 | [diff] [blame] | 395 | if (atomic_inc_not_zero(&sched_core_count)) |
| 396 | return; |
| 397 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 398 | mutex_lock(&sched_core_mutex); |
Peter Zijlstra | 875feb4 | 2021-03-29 10:08:58 +0200 | [diff] [blame] | 399 | if (!atomic_read(&sched_core_count)) |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 400 | __sched_core_enable(); |
Peter Zijlstra | 875feb4 | 2021-03-29 10:08:58 +0200 | [diff] [blame] | 401 | |
| 402 | smp_mb__before_atomic(); |
| 403 | atomic_inc(&sched_core_count); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 404 | mutex_unlock(&sched_core_mutex); |
| 405 | } |
| 406 | |
Peter Zijlstra | 875feb4 | 2021-03-29 10:08:58 +0200 | [diff] [blame] | 407 | static void __sched_core_put(struct work_struct *work) |
| 408 | { |
| 409 | if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) { |
| 410 | __sched_core_disable(); |
| 411 | mutex_unlock(&sched_core_mutex); |
| 412 | } |
| 413 | } |
| 414 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 415 | void sched_core_put(void) |
| 416 | { |
Peter Zijlstra | 875feb4 | 2021-03-29 10:08:58 +0200 | [diff] [blame] | 417 | static DECLARE_WORK(_work, __sched_core_put); |
| 418 | |
| 419 | /* |
| 420 | * "There can be only one" |
| 421 | * |
| 422 | * Either this is the last one, or we don't actually need to do any |
| 423 | * 'work'. If it is the last *again*, we rely on |
| 424 | * WORK_STRUCT_PENDING_BIT. |
| 425 | */ |
| 426 | if (!atomic_add_unless(&sched_core_count, -1, 1)) |
| 427 | schedule_work(&_work); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 428 | } |
| 429 | |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 430 | #else /* !CONFIG_SCHED_CORE */ |
| 431 | |
| 432 | static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 433 | static inline void |
| 434 | sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 435 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 436 | #endif /* CONFIG_SCHED_CORE */ |
| 437 | |
Peter Zijlstra | fa85ae2 | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 438 | /* |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 439 | * Serialization rules: |
| 440 | * |
| 441 | * Lock order: |
| 442 | * |
| 443 | * p->pi_lock |
| 444 | * rq->lock |
| 445 | * hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) |
| 446 | * |
| 447 | * rq1->lock |
| 448 | * rq2->lock where: rq1 < rq2 |
| 449 | * |
| 450 | * Regular state: |
| 451 | * |
| 452 | * Normal scheduling state is serialized by rq->lock. __schedule() takes the |
| 453 | * local CPU's rq->lock, it optionally removes the task from the runqueue and |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 454 | * always looks at the local rq data structures to find the most eligible task |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 455 | * to run next. |
| 456 | * |
| 457 | * Task enqueue is also under rq->lock, possibly taken from another CPU. |
| 458 | * Wakeups from another LLC domain might use an IPI to transfer the enqueue to |
| 459 | * the local CPU to avoid bouncing the runqueue state around [ see |
| 460 | * ttwu_queue_wakelist() ] |
| 461 | * |
| 462 | * Task wakeup, specifically wakeups that involve migration, are horribly |
| 463 | * complicated to avoid having to take two rq->locks. |
| 464 | * |
| 465 | * Special state: |
| 466 | * |
| 467 | * System-calls and anything external will use task_rq_lock() which acquires |
| 468 | * both p->pi_lock and rq->lock. As a consequence the state they change is |
| 469 | * stable while holding either lock: |
| 470 | * |
| 471 | * - sched_setaffinity()/ |
| 472 | * set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed |
| 473 | * - set_user_nice(): p->se.load, p->*prio |
| 474 | * - __sched_setscheduler(): p->sched_class, p->policy, p->*prio, |
| 475 | * p->se.load, p->rt_priority, |
| 476 | * p->dl.dl_{runtime, deadline, period, flags, bw, density} |
| 477 | * - sched_setnuma(): p->numa_preferred_nid |
Chengming Zhou | 39c4261 | 2022-08-18 20:47:58 +0800 | [diff] [blame] | 478 | * - sched_move_task(): p->sched_task_group |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 479 | * - uclamp_update_active() p->uclamp* |
| 480 | * |
| 481 | * p->state <- TASK_*: |
| 482 | * |
| 483 | * is changed locklessly using set_current_state(), __set_current_state() or |
| 484 | * set_special_state(), see their respective comments, or by |
| 485 | * try_to_wake_up(). This latter uses p->pi_lock to serialize against |
| 486 | * concurrent self. |
| 487 | * |
| 488 | * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: |
| 489 | * |
| 490 | * is set by activate_task() and cleared by deactivate_task(), under |
| 491 | * rq->lock. Non-zero indicates the task is runnable, the special |
| 492 | * ON_RQ_MIGRATING state is used for migration without holding both |
| 493 | * rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). |
| 494 | * |
| 495 | * p->on_cpu <- { 0, 1 }: |
| 496 | * |
| 497 | * is set by prepare_task() and cleared by finish_task() such that it will be |
| 498 | * set before p is scheduled-in and cleared after p is scheduled-out, both |
| 499 | * under rq->lock. Non-zero indicates the task is running on its CPU. |
| 500 | * |
| 501 | * [ The astute reader will observe that it is possible for two tasks on one |
| 502 | * CPU to have ->on_cpu = 1 at the same time. ] |
| 503 | * |
| 504 | * task_cpu(p): is changed by set_task_cpu(), the rules are: |
| 505 | * |
| 506 | * - Don't call set_task_cpu() on a blocked task: |
| 507 | * |
| 508 | * We don't care what CPU we're not running on, this simplifies hotplug, |
| 509 | * the CPU assignment of blocked tasks isn't required to be valid. |
| 510 | * |
| 511 | * - for try_to_wake_up(), called under p->pi_lock: |
| 512 | * |
| 513 | * This allows try_to_wake_up() to only take one rq->lock, see its comment. |
| 514 | * |
| 515 | * - for migration called under rq->lock: |
| 516 | * [ see task_on_rq_migrating() in task_rq_lock() ] |
| 517 | * |
| 518 | * o move_queued_task() |
| 519 | * o detach_task() |
| 520 | * |
| 521 | * - for migration called under double_rq_lock(): |
| 522 | * |
| 523 | * o __migrate_swap_task() |
| 524 | * o push_rt_task() / pull_rt_task() |
| 525 | * o push_dl_task() / pull_dl_task() |
| 526 | * o dl_task_offline_migration() |
| 527 | * |
| 528 | */ |
| 529 | |
Peter Zijlstra | 39d371b | 2021-03-02 12:13:13 +0100 | [diff] [blame] | 530 | void raw_spin_rq_lock_nested(struct rq *rq, int subclass) |
| 531 | { |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 532 | raw_spinlock_t *lock; |
| 533 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 534 | /* Matches synchronize_rcu() in __sched_core_enable() */ |
| 535 | preempt_disable(); |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 536 | if (sched_core_disabled()) { |
| 537 | raw_spin_lock_nested(&rq->__lock, subclass); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 538 | /* preempt_count *MUST* be > 1 */ |
| 539 | preempt_enable_no_resched(); |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 540 | return; |
| 541 | } |
| 542 | |
| 543 | for (;;) { |
Peter Zijlstra | 9ef7e7e | 2021-03-03 16:45:41 +0100 | [diff] [blame] | 544 | lock = __rq_lockp(rq); |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 545 | raw_spin_lock_nested(lock, subclass); |
Peter Zijlstra | 9ef7e7e | 2021-03-03 16:45:41 +0100 | [diff] [blame] | 546 | if (likely(lock == __rq_lockp(rq))) { |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 547 | /* preempt_count *MUST* be > 1 */ |
| 548 | preempt_enable_no_resched(); |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 549 | return; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 550 | } |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 551 | raw_spin_unlock(lock); |
| 552 | } |
Peter Zijlstra | 39d371b | 2021-03-02 12:13:13 +0100 | [diff] [blame] | 553 | } |
| 554 | |
| 555 | bool raw_spin_rq_trylock(struct rq *rq) |
| 556 | { |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 557 | raw_spinlock_t *lock; |
| 558 | bool ret; |
| 559 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 560 | /* Matches synchronize_rcu() in __sched_core_enable() */ |
| 561 | preempt_disable(); |
| 562 | if (sched_core_disabled()) { |
| 563 | ret = raw_spin_trylock(&rq->__lock); |
| 564 | preempt_enable(); |
| 565 | return ret; |
| 566 | } |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 567 | |
| 568 | for (;;) { |
Peter Zijlstra | 9ef7e7e | 2021-03-03 16:45:41 +0100 | [diff] [blame] | 569 | lock = __rq_lockp(rq); |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 570 | ret = raw_spin_trylock(lock); |
Peter Zijlstra | 9ef7e7e | 2021-03-03 16:45:41 +0100 | [diff] [blame] | 571 | if (!ret || (likely(lock == __rq_lockp(rq)))) { |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 572 | preempt_enable(); |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 573 | return ret; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 574 | } |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 575 | raw_spin_unlock(lock); |
| 576 | } |
Peter Zijlstra | 39d371b | 2021-03-02 12:13:13 +0100 | [diff] [blame] | 577 | } |
| 578 | |
| 579 | void raw_spin_rq_unlock(struct rq *rq) |
| 580 | { |
| 581 | raw_spin_unlock(rq_lockp(rq)); |
| 582 | } |
| 583 | |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 584 | #ifdef CONFIG_SMP |
| 585 | /* |
| 586 | * double_rq_lock - safely lock two runqueues |
| 587 | */ |
| 588 | void double_rq_lock(struct rq *rq1, struct rq *rq2) |
| 589 | { |
| 590 | lockdep_assert_irqs_disabled(); |
| 591 | |
| 592 | if (rq_order_less(rq2, rq1)) |
| 593 | swap(rq1, rq2); |
| 594 | |
| 595 | raw_spin_rq_lock(rq1); |
Hao Jia | 2679a83 | 2022-04-30 16:58:42 +0800 | [diff] [blame] | 596 | if (__rq_lockp(rq1) != __rq_lockp(rq2)) |
| 597 | raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING); |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 598 | |
Hao Jia | 2679a83 | 2022-04-30 16:58:42 +0800 | [diff] [blame] | 599 | double_rq_clock_clear_update(rq1, rq2); |
Peter Zijlstra | d66f1b0 | 2021-03-02 12:16:48 +0100 | [diff] [blame] | 600 | } |
| 601 | #endif |
| 602 | |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 603 | /* |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 604 | * __task_rq_lock - lock the rq @p resides on. |
| 605 | */ |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 606 | struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 607 | __acquires(rq->lock) |
| 608 | { |
| 609 | struct rq *rq; |
| 610 | |
| 611 | lockdep_assert_held(&p->pi_lock); |
| 612 | |
| 613 | for (;;) { |
| 614 | rq = task_rq(p); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 615 | raw_spin_rq_lock(rq); |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 616 | if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 617 | rq_pin_lock(rq, rf); |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 618 | return rq; |
| 619 | } |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 620 | raw_spin_rq_unlock(rq); |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 621 | |
| 622 | while (unlikely(task_on_rq_migrating(p))) |
| 623 | cpu_relax(); |
| 624 | } |
| 625 | } |
| 626 | |
| 627 | /* |
| 628 | * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. |
| 629 | */ |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 630 | struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 631 | __acquires(p->pi_lock) |
| 632 | __acquires(rq->lock) |
| 633 | { |
| 634 | struct rq *rq; |
| 635 | |
| 636 | for (;;) { |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 637 | raw_spin_lock_irqsave(&p->pi_lock, rf->flags); |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 638 | rq = task_rq(p); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 639 | raw_spin_rq_lock(rq); |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 640 | /* |
| 641 | * move_queued_task() task_rq_lock() |
| 642 | * |
| 643 | * ACQUIRE (rq->lock) |
| 644 | * [S] ->on_rq = MIGRATING [L] rq = task_rq() |
| 645 | * WMB (__set_task_cpu()) ACQUIRE (rq->lock); |
| 646 | * [S] ->cpu = new_cpu [L] task_rq() |
| 647 | * [L] ->on_rq |
| 648 | * RELEASE (rq->lock) |
| 649 | * |
Andrea Parri | c546951 | 2019-01-21 16:52:40 +0100 | [diff] [blame] | 650 | * If we observe the old CPU in task_rq_lock(), the acquire of |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 651 | * the old rq->lock will fully serialize against the stores. |
| 652 | * |
Andrea Parri | c546951 | 2019-01-21 16:52:40 +0100 | [diff] [blame] | 653 | * If we observe the new CPU in task_rq_lock(), the address |
| 654 | * dependency headed by '[L] rq = task_rq()' and the acquire |
| 655 | * will pair with the WMB to ensure we then also see migrating. |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 656 | */ |
| 657 | if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 658 | rq_pin_lock(rq, rf); |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 659 | return rq; |
| 660 | } |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 661 | raw_spin_rq_unlock(rq); |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 662 | raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); |
Peter Zijlstra | 3e71a46 | 2016-04-28 16:16:33 +0200 | [diff] [blame] | 663 | |
| 664 | while (unlikely(task_on_rq_migrating(p))) |
| 665 | cpu_relax(); |
| 666 | } |
| 667 | } |
| 668 | |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 669 | /* |
| 670 | * RQ-clock updating methods: |
| 671 | */ |
| 672 | |
| 673 | static void update_rq_clock_task(struct rq *rq, s64 delta) |
| 674 | { |
| 675 | /* |
| 676 | * In theory, the compile should just see 0 here, and optimize out the call |
| 677 | * to sched_rt_avg_update. But I don't trust it... |
| 678 | */ |
Vincent Guittot | 11d4afd | 2018-09-25 11:17:42 +0200 | [diff] [blame] | 679 | s64 __maybe_unused steal = 0, irq_delta = 0; |
| 680 | |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 681 | #ifdef CONFIG_IRQ_TIME_ACCOUNTING |
| 682 | irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; |
| 683 | |
| 684 | /* |
| 685 | * Since irq_time is only updated on {soft,}irq_exit, we might run into |
| 686 | * this case when a previous update_rq_clock() happened inside a |
| 687 | * {soft,}irq region. |
| 688 | * |
| 689 | * When this happens, we stop ->clock_task and only update the |
| 690 | * prev_irq_time stamp to account for the part that fit, so that a next |
| 691 | * update will consume the rest. This ensures ->clock_task is |
| 692 | * monotonic. |
| 693 | * |
| 694 | * It does however cause some slight miss-attribution of {soft,}irq |
| 695 | * time, a more accurate solution would be to update the irq_time using |
| 696 | * the current rq->clock timestamp, except that would require using |
| 697 | * atomic ops. |
| 698 | */ |
| 699 | if (irq_delta > delta) |
| 700 | irq_delta = delta; |
| 701 | |
| 702 | rq->prev_irq_time += irq_delta; |
| 703 | delta -= irq_delta; |
Chengming Zhou | 52b1364 | 2022-08-26 00:41:08 +0800 | [diff] [blame] | 704 | psi_account_irqtime(rq->curr, irq_delta); |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 705 | #endif |
| 706 | #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING |
| 707 | if (static_key_false((¶virt_steal_rq_enabled))) { |
| 708 | steal = paravirt_steal_clock(cpu_of(rq)); |
| 709 | steal -= rq->prev_steal_time_rq; |
| 710 | |
| 711 | if (unlikely(steal > delta)) |
| 712 | steal = delta; |
| 713 | |
| 714 | rq->prev_steal_time_rq += steal; |
| 715 | delta -= steal; |
| 716 | } |
| 717 | #endif |
| 718 | |
| 719 | rq->clock_task += delta; |
| 720 | |
Vincent Guittot | 11d4afd | 2018-09-25 11:17:42 +0200 | [diff] [blame] | 721 | #ifdef CONFIG_HAVE_SCHED_AVG_IRQ |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 722 | if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) |
Vincent Guittot | 91c2749 | 2018-06-28 17:45:09 +0200 | [diff] [blame] | 723 | update_irq_load_avg(rq, irq_delta + steal); |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 724 | #endif |
Vincent Guittot | 2312729 | 2019-01-23 16:26:53 +0100 | [diff] [blame] | 725 | update_rq_clock_pelt(rq, delta); |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 726 | } |
| 727 | |
| 728 | void update_rq_clock(struct rq *rq) |
| 729 | { |
| 730 | s64 delta; |
| 731 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 732 | lockdep_assert_rq_held(rq); |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 733 | |
| 734 | if (rq->clock_update_flags & RQCF_ACT_SKIP) |
| 735 | return; |
| 736 | |
| 737 | #ifdef CONFIG_SCHED_DEBUG |
Peter Zijlstra | 26ae58d | 2016-10-03 16:53:49 +0200 | [diff] [blame] | 738 | if (sched_feat(WARN_DOUBLE_CLOCK)) |
| 739 | SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 740 | rq->clock_update_flags |= RQCF_UPDATED; |
| 741 | #endif |
Peter Zijlstra | 26ae58d | 2016-10-03 16:53:49 +0200 | [diff] [blame] | 742 | |
Ingo Molnar | 535b955 | 2017-02-01 12:29:21 +0100 | [diff] [blame] | 743 | delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; |
| 744 | if (delta < 0) |
| 745 | return; |
| 746 | rq->clock += delta; |
| 747 | update_rq_clock_task(rq, delta); |
| 748 | } |
| 749 | |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 750 | #ifdef CONFIG_SCHED_HRTICK |
| 751 | /* |
| 752 | * Use HR-timers to deliver accurate preemption points. |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 753 | */ |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 754 | |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 755 | static void hrtick_clear(struct rq *rq) |
| 756 | { |
| 757 | if (hrtimer_active(&rq->hrtick_timer)) |
| 758 | hrtimer_cancel(&rq->hrtick_timer); |
| 759 | } |
| 760 | |
| 761 | /* |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 762 | * High-resolution timer tick. |
| 763 | * Runs from hardirq context with interrupts disabled. |
| 764 | */ |
| 765 | static enum hrtimer_restart hrtick(struct hrtimer *timer) |
| 766 | { |
| 767 | struct rq *rq = container_of(timer, struct rq, hrtick_timer); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 768 | struct rq_flags rf; |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 769 | |
| 770 | WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); |
| 771 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 772 | rq_lock(rq, &rf); |
Peter Zijlstra | 3e51f33 | 2008-05-03 18:29:28 +0200 | [diff] [blame] | 773 | update_rq_clock(rq); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 774 | rq->curr->sched_class->task_tick(rq, rq->curr, 1); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 775 | rq_unlock(rq, &rf); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 776 | |
| 777 | return HRTIMER_NORESTART; |
| 778 | } |
| 779 | |
Rabin Vincent | 95e904c | 2008-05-11 05:55:33 +0530 | [diff] [blame] | 780 | #ifdef CONFIG_SMP |
Peter Zijlstra | 971ee28 | 2013-06-28 11:18:53 +0200 | [diff] [blame] | 781 | |
Thomas Gleixner | 4961b6e | 2015-04-14 21:09:05 +0000 | [diff] [blame] | 782 | static void __hrtick_restart(struct rq *rq) |
Peter Zijlstra | 971ee28 | 2013-06-28 11:18:53 +0200 | [diff] [blame] | 783 | { |
| 784 | struct hrtimer *timer = &rq->hrtick_timer; |
Juri Lelli | 156ec6f | 2021-02-08 08:35:53 +0100 | [diff] [blame] | 785 | ktime_t time = rq->hrtick_time; |
Peter Zijlstra | 971ee28 | 2013-06-28 11:18:53 +0200 | [diff] [blame] | 786 | |
Juri Lelli | 156ec6f | 2021-02-08 08:35:53 +0100 | [diff] [blame] | 787 | hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD); |
Peter Zijlstra | 971ee28 | 2013-06-28 11:18:53 +0200 | [diff] [blame] | 788 | } |
| 789 | |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 790 | /* |
| 791 | * called from hardirq (IPI) context |
| 792 | */ |
| 793 | static void __hrtick_start(void *arg) |
Peter Zijlstra | b328ca1 | 2008-04-29 10:02:46 +0200 | [diff] [blame] | 794 | { |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 795 | struct rq *rq = arg; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 796 | struct rq_flags rf; |
Peter Zijlstra | b328ca1 | 2008-04-29 10:02:46 +0200 | [diff] [blame] | 797 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 798 | rq_lock(rq, &rf); |
Peter Zijlstra | 971ee28 | 2013-06-28 11:18:53 +0200 | [diff] [blame] | 799 | __hrtick_restart(rq); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 800 | rq_unlock(rq, &rf); |
Peter Zijlstra | b328ca1 | 2008-04-29 10:02:46 +0200 | [diff] [blame] | 801 | } |
| 802 | |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 803 | /* |
| 804 | * Called to set the hrtick timer state. |
| 805 | * |
| 806 | * called with rq->lock held and irqs disabled |
| 807 | */ |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 808 | void hrtick_start(struct rq *rq, u64 delay) |
Peter Zijlstra | b328ca1 | 2008-04-29 10:02:46 +0200 | [diff] [blame] | 809 | { |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 810 | struct hrtimer *timer = &rq->hrtick_timer; |
xiaofeng.yan | 177ef2a | 2014-08-26 03:15:41 +0000 | [diff] [blame] | 811 | s64 delta; |
| 812 | |
| 813 | /* |
| 814 | * Don't schedule slices shorter than 10000ns, that just |
| 815 | * doesn't make sense and can cause timer DoS. |
| 816 | */ |
| 817 | delta = max_t(s64, delay, 10000LL); |
Juri Lelli | 156ec6f | 2021-02-08 08:35:53 +0100 | [diff] [blame] | 818 | rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta); |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 819 | |
Peter Xu | fd3eafd | 2019-12-16 16:31:25 -0500 | [diff] [blame] | 820 | if (rq == this_rq()) |
Peter Zijlstra | 971ee28 | 2013-06-28 11:18:53 +0200 | [diff] [blame] | 821 | __hrtick_restart(rq); |
Peter Xu | fd3eafd | 2019-12-16 16:31:25 -0500 | [diff] [blame] | 822 | else |
Frederic Weisbecker | c46fff2 | 2014-02-24 16:40:02 +0100 | [diff] [blame] | 823 | smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); |
Peter Zijlstra | b328ca1 | 2008-04-29 10:02:46 +0200 | [diff] [blame] | 824 | } |
| 825 | |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 826 | #else |
| 827 | /* |
| 828 | * Called to set the hrtick timer state. |
| 829 | * |
| 830 | * called with rq->lock held and irqs disabled |
| 831 | */ |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 832 | void hrtick_start(struct rq *rq, u64 delay) |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 833 | { |
Wanpeng Li | 8689333 | 2014-11-26 08:44:06 +0800 | [diff] [blame] | 834 | /* |
| 835 | * Don't schedule slices shorter than 10000ns, that just |
| 836 | * doesn't make sense. Rely on vruntime for fairness. |
| 837 | */ |
| 838 | delay = max_t(u64, delay, 10000LL); |
Thomas Gleixner | 4961b6e | 2015-04-14 21:09:05 +0000 | [diff] [blame] | 839 | hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), |
Sebastian Andrzej Siewior | d5096aa | 2019-07-26 20:30:52 +0200 | [diff] [blame] | 840 | HRTIMER_MODE_REL_PINNED_HARD); |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 841 | } |
Peter Zijlstra (Intel) | 90b5363 | 2020-03-27 11:44:56 +0100 | [diff] [blame] | 842 | |
Rabin Vincent | 95e904c | 2008-05-11 05:55:33 +0530 | [diff] [blame] | 843 | #endif /* CONFIG_SMP */ |
Peter Zijlstra | b328ca1 | 2008-04-29 10:02:46 +0200 | [diff] [blame] | 844 | |
Frederic Weisbecker | 77a021b | 2018-02-21 05:17:23 +0100 | [diff] [blame] | 845 | static void hrtick_rq_init(struct rq *rq) |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 846 | { |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 847 | #ifdef CONFIG_SMP |
Peter Zijlstra | 545b8c8 | 2020-06-15 11:29:31 +0200 | [diff] [blame] | 848 | INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 849 | #endif |
Sebastian Andrzej Siewior | d5096aa | 2019-07-26 20:30:52 +0200 | [diff] [blame] | 850 | hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 851 | rq->hrtick_timer.function = hrtick; |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 852 | } |
Andrew Morton | 006c75f | 2008-09-22 14:55:46 -0700 | [diff] [blame] | 853 | #else /* CONFIG_SCHED_HRTICK */ |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 854 | static inline void hrtick_clear(struct rq *rq) |
| 855 | { |
| 856 | } |
| 857 | |
Frederic Weisbecker | 77a021b | 2018-02-21 05:17:23 +0100 | [diff] [blame] | 858 | static inline void hrtick_rq_init(struct rq *rq) |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 859 | { |
| 860 | } |
Andrew Morton | 006c75f | 2008-09-22 14:55:46 -0700 | [diff] [blame] | 861 | #endif /* CONFIG_SCHED_HRTICK */ |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 862 | |
Frederic Weisbecker | 5529578 | 2016-03-24 15:38:01 +0100 | [diff] [blame] | 863 | /* |
| 864 | * cmpxchg based fetch_or, macro so it works for different integer types |
| 865 | */ |
| 866 | #define fetch_or(ptr, mask) \ |
| 867 | ({ \ |
| 868 | typeof(ptr) _ptr = (ptr); \ |
| 869 | typeof(mask) _mask = (mask); \ |
Uros Bizjak | c02d554 | 2022-06-29 17:15:52 +0200 | [diff] [blame] | 870 | typeof(*_ptr) _val = *_ptr; \ |
Frederic Weisbecker | 5529578 | 2016-03-24 15:38:01 +0100 | [diff] [blame] | 871 | \ |
Uros Bizjak | c02d554 | 2022-06-29 17:15:52 +0200 | [diff] [blame] | 872 | do { \ |
| 873 | } while (!try_cmpxchg(_ptr, &_val, _val | _mask)); \ |
| 874 | _val; \ |
Frederic Weisbecker | 5529578 | 2016-03-24 15:38:01 +0100 | [diff] [blame] | 875 | }) |
| 876 | |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 877 | #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) |
Peter Zijlstra | fd99f91 | 2014-04-09 15:35:08 +0200 | [diff] [blame] | 878 | /* |
| 879 | * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, |
| 880 | * this avoids any races wrt polling state changes and thereby avoids |
| 881 | * spurious IPIs. |
| 882 | */ |
Uros Bizjak | c02d554 | 2022-06-29 17:15:52 +0200 | [diff] [blame] | 883 | static inline bool set_nr_and_not_polling(struct task_struct *p) |
Peter Zijlstra | fd99f91 | 2014-04-09 15:35:08 +0200 | [diff] [blame] | 884 | { |
| 885 | struct thread_info *ti = task_thread_info(p); |
| 886 | return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); |
| 887 | } |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 888 | |
| 889 | /* |
| 890 | * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. |
| 891 | * |
| 892 | * If this returns true, then the idle task promises to call |
| 893 | * sched_ttwu_pending() and reschedule soon. |
| 894 | */ |
| 895 | static bool set_nr_if_polling(struct task_struct *p) |
| 896 | { |
| 897 | struct thread_info *ti = task_thread_info(p); |
Uros Bizjak | c02d554 | 2022-06-29 17:15:52 +0200 | [diff] [blame] | 898 | typeof(ti->flags) val = READ_ONCE(ti->flags); |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 899 | |
| 900 | for (;;) { |
| 901 | if (!(val & _TIF_POLLING_NRFLAG)) |
| 902 | return false; |
| 903 | if (val & _TIF_NEED_RESCHED) |
| 904 | return true; |
Uros Bizjak | c02d554 | 2022-06-29 17:15:52 +0200 | [diff] [blame] | 905 | if (try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED)) |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 906 | break; |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 907 | } |
| 908 | return true; |
| 909 | } |
| 910 | |
Peter Zijlstra | fd99f91 | 2014-04-09 15:35:08 +0200 | [diff] [blame] | 911 | #else |
Uros Bizjak | c02d554 | 2022-06-29 17:15:52 +0200 | [diff] [blame] | 912 | static inline bool set_nr_and_not_polling(struct task_struct *p) |
Peter Zijlstra | fd99f91 | 2014-04-09 15:35:08 +0200 | [diff] [blame] | 913 | { |
| 914 | set_tsk_need_resched(p); |
| 915 | return true; |
| 916 | } |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 917 | |
| 918 | #ifdef CONFIG_SMP |
Uros Bizjak | c02d554 | 2022-06-29 17:15:52 +0200 | [diff] [blame] | 919 | static inline bool set_nr_if_polling(struct task_struct *p) |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 920 | { |
| 921 | return false; |
| 922 | } |
| 923 | #endif |
Peter Zijlstra | fd99f91 | 2014-04-09 15:35:08 +0200 | [diff] [blame] | 924 | #endif |
| 925 | |
Davidlohr Bueso | 07879c6 | 2018-12-18 11:53:52 -0800 | [diff] [blame] | 926 | static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) |
| 927 | { |
| 928 | struct wake_q_node *node = &task->wake_q; |
| 929 | |
| 930 | /* |
| 931 | * Atomically grab the task, if ->wake_q is !nil already it means |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 932 | * it's already queued (either by us or someone else) and will get the |
Davidlohr Bueso | 07879c6 | 2018-12-18 11:53:52 -0800 | [diff] [blame] | 933 | * wakeup due to that. |
| 934 | * |
| 935 | * In order to ensure that a pending wakeup will observe our pending |
| 936 | * state, even in the failed case, an explicit smp_mb() must be used. |
| 937 | */ |
| 938 | smp_mb__before_atomic(); |
| 939 | if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) |
| 940 | return false; |
| 941 | |
| 942 | /* |
| 943 | * The head is context local, there can be no concurrency. |
| 944 | */ |
| 945 | *head->lastp = node; |
| 946 | head->lastp = &node->next; |
| 947 | return true; |
| 948 | } |
| 949 | |
Peter Zijlstra | e6018c0 | 2018-12-17 10:14:53 +0100 | [diff] [blame] | 950 | /** |
| 951 | * wake_q_add() - queue a wakeup for 'later' waking. |
| 952 | * @head: the wake_q_head to add @task to |
| 953 | * @task: the task to queue for 'later' wakeup |
| 954 | * |
| 955 | * Queue a task for later wakeup, most likely by the wake_up_q() call in the |
| 956 | * same context, _HOWEVER_ this is not guaranteed, the wakeup can come |
| 957 | * instantly. |
| 958 | * |
| 959 | * This function must be used as-if it were wake_up_process(); IOW the task |
| 960 | * must be ready to be woken at this location. |
| 961 | */ |
Peter Zijlstra | 7675104 | 2015-05-01 08:27:50 -0700 | [diff] [blame] | 962 | void wake_q_add(struct wake_q_head *head, struct task_struct *task) |
| 963 | { |
Davidlohr Bueso | 07879c6 | 2018-12-18 11:53:52 -0800 | [diff] [blame] | 964 | if (__wake_q_add(head, task)) |
| 965 | get_task_struct(task); |
| 966 | } |
Peter Zijlstra | 7675104 | 2015-05-01 08:27:50 -0700 | [diff] [blame] | 967 | |
Davidlohr Bueso | 07879c6 | 2018-12-18 11:53:52 -0800 | [diff] [blame] | 968 | /** |
| 969 | * wake_q_add_safe() - safely queue a wakeup for 'later' waking. |
| 970 | * @head: the wake_q_head to add @task to |
| 971 | * @task: the task to queue for 'later' wakeup |
| 972 | * |
| 973 | * Queue a task for later wakeup, most likely by the wake_up_q() call in the |
| 974 | * same context, _HOWEVER_ this is not guaranteed, the wakeup can come |
| 975 | * instantly. |
| 976 | * |
| 977 | * This function must be used as-if it were wake_up_process(); IOW the task |
| 978 | * must be ready to be woken at this location. |
| 979 | * |
| 980 | * This function is essentially a task-safe equivalent to wake_q_add(). Callers |
| 981 | * that already hold reference to @task can call the 'safe' version and trust |
| 982 | * wake_q to do the right thing depending whether or not the @task is already |
| 983 | * queued for wakeup. |
| 984 | */ |
| 985 | void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) |
| 986 | { |
| 987 | if (!__wake_q_add(head, task)) |
| 988 | put_task_struct(task); |
Peter Zijlstra | 7675104 | 2015-05-01 08:27:50 -0700 | [diff] [blame] | 989 | } |
| 990 | |
| 991 | void wake_up_q(struct wake_q_head *head) |
| 992 | { |
| 993 | struct wake_q_node *node = head->first; |
| 994 | |
| 995 | while (node != WAKE_Q_TAIL) { |
| 996 | struct task_struct *task; |
| 997 | |
| 998 | task = container_of(node, struct task_struct, wake_q); |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 999 | /* Task can safely be re-inserted now: */ |
Peter Zijlstra | 7675104 | 2015-05-01 08:27:50 -0700 | [diff] [blame] | 1000 | node = node->next; |
| 1001 | task->wake_q.next = NULL; |
| 1002 | |
| 1003 | /* |
Andrea Parri | 7696f99 | 2018-07-16 11:06:03 -0700 | [diff] [blame] | 1004 | * wake_up_process() executes a full barrier, which pairs with |
| 1005 | * the queueing in wake_q_add() so as not to miss wakeups. |
Peter Zijlstra | 7675104 | 2015-05-01 08:27:50 -0700 | [diff] [blame] | 1006 | */ |
| 1007 | wake_up_process(task); |
| 1008 | put_task_struct(task); |
| 1009 | } |
| 1010 | } |
| 1011 | |
Peter Zijlstra | fd99f91 | 2014-04-09 15:35:08 +0200 | [diff] [blame] | 1012 | /* |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 1013 | * resched_curr - mark rq's current task 'to be rescheduled now'. |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1014 | * |
| 1015 | * On UP this means the setting of the need_resched flag, on SMP it |
| 1016 | * might also involve a cross-CPU call to trigger the scheduler on |
| 1017 | * the target CPU. |
| 1018 | */ |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 1019 | void resched_curr(struct rq *rq) |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1020 | { |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 1021 | struct task_struct *curr = rq->curr; |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1022 | int cpu; |
| 1023 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 1024 | lockdep_assert_rq_held(rq); |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1025 | |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 1026 | if (test_tsk_need_resched(curr)) |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1027 | return; |
| 1028 | |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 1029 | cpu = cpu_of(rq); |
Peter Zijlstra | fd99f91 | 2014-04-09 15:35:08 +0200 | [diff] [blame] | 1030 | |
Peter Zijlstra | f27dde8 | 2013-08-14 14:55:31 +0200 | [diff] [blame] | 1031 | if (cpu == smp_processor_id()) { |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 1032 | set_tsk_need_resched(curr); |
Peter Zijlstra | f27dde8 | 2013-08-14 14:55:31 +0200 | [diff] [blame] | 1033 | set_preempt_need_resched(); |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1034 | return; |
Peter Zijlstra | f27dde8 | 2013-08-14 14:55:31 +0200 | [diff] [blame] | 1035 | } |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1036 | |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 1037 | if (set_nr_and_not_polling(curr)) |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1038 | smp_send_reschedule(cpu); |
Andy Lutomirski | dfc68f2 | 2014-06-04 10:31:15 -0700 | [diff] [blame] | 1039 | else |
| 1040 | trace_sched_wake_idle_without_ipi(cpu); |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1041 | } |
| 1042 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 1043 | void resched_cpu(int cpu) |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1044 | { |
| 1045 | struct rq *rq = cpu_rq(cpu); |
| 1046 | unsigned long flags; |
| 1047 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 1048 | raw_spin_rq_lock_irqsave(rq, flags); |
Paul E. McKenney | a0982df | 2017-10-13 16:24:28 -0700 | [diff] [blame] | 1049 | if (cpu_online(cpu) || cpu == smp_processor_id()) |
| 1050 | resched_curr(rq); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 1051 | raw_spin_rq_unlock_irqrestore(rq, flags); |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1052 | } |
Thomas Gleixner | 06d8308 | 2008-03-22 09:20:24 +0100 | [diff] [blame] | 1053 | |
Peter Zijlstra | b021fe3 | 2013-09-17 09:30:55 +0200 | [diff] [blame] | 1054 | #ifdef CONFIG_SMP |
Frederic Weisbecker | 3451d02 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 1055 | #ifdef CONFIG_NO_HZ_COMMON |
Thomas Gleixner | 06d8308 | 2008-03-22 09:20:24 +0100 | [diff] [blame] | 1056 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 1057 | * In the semi idle case, use the nearest busy CPU for migrating timers |
| 1058 | * from an idle CPU. This is good for power-savings. |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1059 | * |
| 1060 | * We don't do similar optimization for completely idle system, as |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 1061 | * selecting an idle CPU will add more delays to the timers than intended |
| 1062 | * (as that CPU's timer base may not be uptodate wrt jiffies etc). |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1063 | */ |
Thomas Gleixner | bc7a34b | 2015-05-26 22:50:33 +0000 | [diff] [blame] | 1064 | int get_nohz_timer_target(void) |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1065 | { |
Wanpeng Li | e938b9c | 2020-01-13 08:50:27 +0800 | [diff] [blame] | 1066 | int i, cpu = smp_processor_id(), default_cpu = -1; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1067 | struct sched_domain *sd; |
Yuan ZhaoXiong | 031e3bd | 2021-06-06 21:11:55 +0800 | [diff] [blame] | 1068 | const struct cpumask *hk_mask; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1069 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 1070 | if (housekeeping_cpu(cpu, HK_TYPE_TIMER)) { |
Wanpeng Li | e938b9c | 2020-01-13 08:50:27 +0800 | [diff] [blame] | 1071 | if (!idle_cpu(cpu)) |
| 1072 | return cpu; |
| 1073 | default_cpu = cpu; |
| 1074 | } |
Viresh Kumar | 6201b4d | 2014-03-18 16:26:07 +0530 | [diff] [blame] | 1075 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 1076 | hk_mask = housekeeping_cpumask(HK_TYPE_TIMER); |
Yuan ZhaoXiong | 031e3bd | 2021-06-06 21:11:55 +0800 | [diff] [blame] | 1077 | |
Peter Zijlstra | 057f3fa | 2011-04-18 11:24:34 +0200 | [diff] [blame] | 1078 | rcu_read_lock(); |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1079 | for_each_domain(cpu, sd) { |
Yuan ZhaoXiong | 031e3bd | 2021-06-06 21:11:55 +0800 | [diff] [blame] | 1080 | for_each_cpu_and(i, sched_domain_span(sd), hk_mask) { |
Wanpeng Li | 4449692 | 2016-05-04 14:45:34 +0800 | [diff] [blame] | 1081 | if (cpu == i) |
| 1082 | continue; |
| 1083 | |
Wanpeng Li | e938b9c | 2020-01-13 08:50:27 +0800 | [diff] [blame] | 1084 | if (!idle_cpu(i)) { |
Peter Zijlstra | 057f3fa | 2011-04-18 11:24:34 +0200 | [diff] [blame] | 1085 | cpu = i; |
| 1086 | goto unlock; |
| 1087 | } |
| 1088 | } |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1089 | } |
Vatika Harlalka | 9642d18e | 2015-09-01 16:50:59 +0200 | [diff] [blame] | 1090 | |
Wanpeng Li | e938b9c | 2020-01-13 08:50:27 +0800 | [diff] [blame] | 1091 | if (default_cpu == -1) |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 1092 | default_cpu = housekeeping_any_cpu(HK_TYPE_TIMER); |
Wanpeng Li | e938b9c | 2020-01-13 08:50:27 +0800 | [diff] [blame] | 1093 | cpu = default_cpu; |
Peter Zijlstra | 057f3fa | 2011-04-18 11:24:34 +0200 | [diff] [blame] | 1094 | unlock: |
| 1095 | rcu_read_unlock(); |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1096 | return cpu; |
| 1097 | } |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 1098 | |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 1099 | /* |
Thomas Gleixner | 06d8308 | 2008-03-22 09:20:24 +0100 | [diff] [blame] | 1100 | * When add_timer_on() enqueues a timer into the timer wheel of an |
| 1101 | * idle CPU then this timer might expire before the next timer event |
| 1102 | * which is scheduled to wake up that CPU. In case of a completely |
| 1103 | * idle system the next event might even be infinite time into the |
| 1104 | * future. wake_up_idle_cpu() ensures that the CPU is woken up and |
| 1105 | * leaves the inner idle loop so the newly added timer is taken into |
| 1106 | * account when the CPU goes back to idle and evaluates the timer |
| 1107 | * wheel for the next timer event. |
| 1108 | */ |
Frederic Weisbecker | 1c20091 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 1109 | static void wake_up_idle_cpu(int cpu) |
Thomas Gleixner | 06d8308 | 2008-03-22 09:20:24 +0100 | [diff] [blame] | 1110 | { |
| 1111 | struct rq *rq = cpu_rq(cpu); |
| 1112 | |
| 1113 | if (cpu == smp_processor_id()) |
| 1114 | return; |
| 1115 | |
Andy Lutomirski | 67b9ca7 | 2014-06-04 10:31:17 -0700 | [diff] [blame] | 1116 | if (set_nr_and_not_polling(rq->idle)) |
Thomas Gleixner | 06d8308 | 2008-03-22 09:20:24 +0100 | [diff] [blame] | 1117 | smp_send_reschedule(cpu); |
Andy Lutomirski | dfc68f2 | 2014-06-04 10:31:15 -0700 | [diff] [blame] | 1118 | else |
| 1119 | trace_sched_wake_idle_without_ipi(cpu); |
Thomas Gleixner | 06d8308 | 2008-03-22 09:20:24 +0100 | [diff] [blame] | 1120 | } |
Mike Galbraith | 39c0cbe | 2010-03-11 17:17:13 +0100 | [diff] [blame] | 1121 | |
Frederic Weisbecker | c5bfece | 2013-04-12 16:45:34 +0200 | [diff] [blame] | 1122 | static bool wake_up_full_nohz_cpu(int cpu) |
Frederic Weisbecker | 1c20091 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 1123 | { |
Frederic Weisbecker | 53c5fa1 | 2014-06-04 16:20:21 +0200 | [diff] [blame] | 1124 | /* |
| 1125 | * We just need the target to call irq_exit() and re-evaluate |
| 1126 | * the next tick. The nohz full kick at least implies that. |
| 1127 | * If needed we can still optimize that later with an |
| 1128 | * empty IRQ. |
| 1129 | */ |
Paul E. McKenney | 379d9ec | 2016-06-30 10:37:20 -0700 | [diff] [blame] | 1130 | if (cpu_is_offline(cpu)) |
| 1131 | return true; /* Don't try to wake offline CPUs. */ |
Frederic Weisbecker | c5bfece | 2013-04-12 16:45:34 +0200 | [diff] [blame] | 1132 | if (tick_nohz_full_cpu(cpu)) { |
Frederic Weisbecker | 1c20091 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 1133 | if (cpu != smp_processor_id() || |
| 1134 | tick_nohz_tick_stopped()) |
Frederic Weisbecker | 53c5fa1 | 2014-06-04 16:20:21 +0200 | [diff] [blame] | 1135 | tick_nohz_full_kick_cpu(cpu); |
Frederic Weisbecker | 1c20091 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 1136 | return true; |
| 1137 | } |
| 1138 | |
| 1139 | return false; |
| 1140 | } |
| 1141 | |
Paul E. McKenney | 379d9ec | 2016-06-30 10:37:20 -0700 | [diff] [blame] | 1142 | /* |
| 1143 | * Wake up the specified CPU. If the CPU is going offline, it is the |
| 1144 | * caller's responsibility to deal with the lost wakeup, for example, |
| 1145 | * by hooking into the CPU_DEAD notifier like timers and hrtimers do. |
| 1146 | */ |
Frederic Weisbecker | 1c20091 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 1147 | void wake_up_nohz_cpu(int cpu) |
| 1148 | { |
Frederic Weisbecker | c5bfece | 2013-04-12 16:45:34 +0200 | [diff] [blame] | 1149 | if (!wake_up_full_nohz_cpu(cpu)) |
Frederic Weisbecker | 1c20091 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 1150 | wake_up_idle_cpu(cpu); |
| 1151 | } |
| 1152 | |
Peter Zijlstra (Intel) | 90b5363 | 2020-03-27 11:44:56 +0100 | [diff] [blame] | 1153 | static void nohz_csd_func(void *info) |
Suresh Siddha | ca38062 | 2011-10-03 15:09:00 -0700 | [diff] [blame] | 1154 | { |
Peter Zijlstra (Intel) | 90b5363 | 2020-03-27 11:44:56 +0100 | [diff] [blame] | 1155 | struct rq *rq = info; |
Peter Zijlstra | 19a1f5e | 2020-05-26 18:10:58 +0200 | [diff] [blame] | 1156 | int cpu = cpu_of(rq); |
| 1157 | unsigned int flags; |
Vincent Guittot | 873b4c6 | 2013-06-05 10:13:11 +0200 | [diff] [blame] | 1158 | |
| 1159 | /* |
Peter Zijlstra | 19a1f5e | 2020-05-26 18:10:58 +0200 | [diff] [blame] | 1160 | * Release the rq::nohz_csd. |
Vincent Guittot | 873b4c6 | 2013-06-05 10:13:11 +0200 | [diff] [blame] | 1161 | */ |
Vincent Guittot | c6f8865 | 2021-02-24 14:30:06 +0100 | [diff] [blame] | 1162 | flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu)); |
Peter Zijlstra | 19a1f5e | 2020-05-26 18:10:58 +0200 | [diff] [blame] | 1163 | WARN_ON(!(flags & NOHZ_KICK_MASK)); |
Suresh Siddha | ca38062 | 2011-10-03 15:09:00 -0700 | [diff] [blame] | 1164 | |
Peter Zijlstra | 19a1f5e | 2020-05-26 18:10:58 +0200 | [diff] [blame] | 1165 | rq->idle_balance = idle_cpu(cpu); |
| 1166 | if (rq->idle_balance && !need_resched()) { |
| 1167 | rq->nohz_idle_balance = flags; |
Peter Zijlstra (Intel) | 90b5363 | 2020-03-27 11:44:56 +0100 | [diff] [blame] | 1168 | raise_softirq_irqoff(SCHED_SOFTIRQ); |
| 1169 | } |
Suresh Siddha | ca38062 | 2011-10-03 15:09:00 -0700 | [diff] [blame] | 1170 | } |
| 1171 | |
Frederic Weisbecker | 3451d02 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 1172 | #endif /* CONFIG_NO_HZ_COMMON */ |
Thomas Gleixner | 06d8308 | 2008-03-22 09:20:24 +0100 | [diff] [blame] | 1173 | |
Frederic Weisbecker | ce831b3 | 2013-04-20 15:15:35 +0200 | [diff] [blame] | 1174 | #ifdef CONFIG_NO_HZ_FULL |
Frederic Weisbecker | 76d92ac | 2015-07-17 22:25:49 +0200 | [diff] [blame] | 1175 | bool sched_can_stop_tick(struct rq *rq) |
Frederic Weisbecker | ce831b3 | 2013-04-20 15:15:35 +0200 | [diff] [blame] | 1176 | { |
Frederic Weisbecker | 76d92ac | 2015-07-17 22:25:49 +0200 | [diff] [blame] | 1177 | int fifo_nr_running; |
| 1178 | |
| 1179 | /* Deadline tasks, even if single, need the tick */ |
| 1180 | if (rq->dl.dl_nr_running) |
| 1181 | return false; |
| 1182 | |
Frederic Weisbecker | 3882ec6 | 2014-03-18 22:54:04 +0100 | [diff] [blame] | 1183 | /* |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 1184 | * If there are more than one RR tasks, we need the tick to affect the |
Peter Zijlstra | 2548d54 | 2016-04-21 18:03:15 +0200 | [diff] [blame] | 1185 | * actual RR behaviour. |
Rik van Riel | 1e78cdb | 2015-02-16 15:23:49 -0500 | [diff] [blame] | 1186 | */ |
Frederic Weisbecker | 76d92ac | 2015-07-17 22:25:49 +0200 | [diff] [blame] | 1187 | if (rq->rt.rr_nr_running) { |
| 1188 | if (rq->rt.rr_nr_running == 1) |
| 1189 | return true; |
| 1190 | else |
| 1191 | return false; |
Rik van Riel | 1e78cdb | 2015-02-16 15:23:49 -0500 | [diff] [blame] | 1192 | } |
| 1193 | |
Peter Zijlstra | 2548d54 | 2016-04-21 18:03:15 +0200 | [diff] [blame] | 1194 | /* |
| 1195 | * If there's no RR tasks, but FIFO tasks, we can skip the tick, no |
| 1196 | * forced preemption between FIFO tasks. |
| 1197 | */ |
| 1198 | fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; |
| 1199 | if (fifo_nr_running) |
| 1200 | return true; |
| 1201 | |
| 1202 | /* |
| 1203 | * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; |
| 1204 | * if there's more than one we need the tick for involuntary |
| 1205 | * preemption. |
| 1206 | */ |
| 1207 | if (rq->nr_running > 1) |
Viresh Kumar | 541b826 | 2014-06-24 14:04:12 +0530 | [diff] [blame] | 1208 | return false; |
Frederic Weisbecker | ce831b3 | 2013-04-20 15:15:35 +0200 | [diff] [blame] | 1209 | |
Viresh Kumar | 541b826 | 2014-06-24 14:04:12 +0530 | [diff] [blame] | 1210 | return true; |
Frederic Weisbecker | ce831b3 | 2013-04-20 15:15:35 +0200 | [diff] [blame] | 1211 | } |
| 1212 | #endif /* CONFIG_NO_HZ_FULL */ |
Dhaval Giani | 6d6bc0a | 2008-05-30 14:23:45 +0200 | [diff] [blame] | 1213 | #endif /* CONFIG_SMP */ |
Ingo Molnar | c24d20d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1214 | |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 1215 | #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ |
| 1216 | (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) |
Peter Zijlstra | eb75580 | 2008-08-19 12:33:05 +0200 | [diff] [blame] | 1217 | /* |
Paul Turner | 8277434 | 2011-07-21 09:43:35 -0700 | [diff] [blame] | 1218 | * Iterate task_group tree rooted at *from, calling @down when first entering a |
| 1219 | * node and @up when leaving it for the final time. |
| 1220 | * |
| 1221 | * Caller must hold rcu_lock or sufficient equivalent. |
Peter Zijlstra | eb75580 | 2008-08-19 12:33:05 +0200 | [diff] [blame] | 1222 | */ |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 1223 | int walk_tg_tree_from(struct task_group *from, |
Paul Turner | 8277434 | 2011-07-21 09:43:35 -0700 | [diff] [blame] | 1224 | tg_visitor down, tg_visitor up, void *data) |
Peter Zijlstra | eb75580 | 2008-08-19 12:33:05 +0200 | [diff] [blame] | 1225 | { |
| 1226 | struct task_group *parent, *child; |
| 1227 | int ret; |
| 1228 | |
Paul Turner | 8277434 | 2011-07-21 09:43:35 -0700 | [diff] [blame] | 1229 | parent = from; |
| 1230 | |
Peter Zijlstra | eb75580 | 2008-08-19 12:33:05 +0200 | [diff] [blame] | 1231 | down: |
| 1232 | ret = (*down)(parent, data); |
| 1233 | if (ret) |
Paul Turner | 8277434 | 2011-07-21 09:43:35 -0700 | [diff] [blame] | 1234 | goto out; |
Peter Zijlstra | eb75580 | 2008-08-19 12:33:05 +0200 | [diff] [blame] | 1235 | list_for_each_entry_rcu(child, &parent->children, siblings) { |
| 1236 | parent = child; |
| 1237 | goto down; |
| 1238 | |
| 1239 | up: |
| 1240 | continue; |
| 1241 | } |
| 1242 | ret = (*up)(parent, data); |
Paul Turner | 8277434 | 2011-07-21 09:43:35 -0700 | [diff] [blame] | 1243 | if (ret || parent == from) |
| 1244 | goto out; |
Peter Zijlstra | eb75580 | 2008-08-19 12:33:05 +0200 | [diff] [blame] | 1245 | |
| 1246 | child = parent; |
| 1247 | parent = parent->parent; |
| 1248 | if (parent) |
| 1249 | goto up; |
Paul Turner | 8277434 | 2011-07-21 09:43:35 -0700 | [diff] [blame] | 1250 | out: |
Peter Zijlstra | eb75580 | 2008-08-19 12:33:05 +0200 | [diff] [blame] | 1251 | return ret; |
| 1252 | } |
| 1253 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 1254 | int tg_nop(struct task_group *tg, void *data) |
Peter Zijlstra | eb75580 | 2008-08-19 12:33:05 +0200 | [diff] [blame] | 1255 | { |
| 1256 | return 0; |
| 1257 | } |
| 1258 | #endif |
| 1259 | |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 1260 | static void set_load_weight(struct task_struct *p, bool update_load) |
Ingo Molnar | 45bf76d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1261 | { |
Nikhil Rao | f05998d | 2011-05-18 10:09:38 -0700 | [diff] [blame] | 1262 | int prio = p->static_prio - MAX_RT_PRIO; |
| 1263 | struct load_weight *load = &p->se.load; |
| 1264 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1265 | /* |
| 1266 | * SCHED_IDLE tasks get minimal weight: |
| 1267 | */ |
Viresh Kumar | 1da1843 | 2018-11-05 16:51:55 +0530 | [diff] [blame] | 1268 | if (task_has_idle_policy(p)) { |
Nikhil Rao | c8b2811 | 2011-05-18 14:37:48 -0700 | [diff] [blame] | 1269 | load->weight = scale_load(WEIGHT_IDLEPRIO); |
Nikhil Rao | f05998d | 2011-05-18 10:09:38 -0700 | [diff] [blame] | 1270 | load->inv_weight = WMULT_IDLEPRIO; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1271 | return; |
| 1272 | } |
| 1273 | |
Vincent Guittot | 9059393 | 2017-05-17 11:50:45 +0200 | [diff] [blame] | 1274 | /* |
| 1275 | * SCHED_OTHER tasks have to update their load when changing their |
| 1276 | * weight |
| 1277 | */ |
| 1278 | if (update_load && p->sched_class == &fair_sched_class) { |
| 1279 | reweight_task(p, prio); |
| 1280 | } else { |
| 1281 | load->weight = scale_load(sched_prio_to_weight[prio]); |
| 1282 | load->inv_weight = sched_prio_to_wmult[prio]; |
| 1283 | } |
Ingo Molnar | 45bf76d | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 1284 | } |
| 1285 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1286 | #ifdef CONFIG_UCLAMP_TASK |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 1287 | /* |
| 1288 | * Serializes updates of utilization clamp values |
| 1289 | * |
| 1290 | * The (slow-path) user-space triggers utilization clamp value updates which |
| 1291 | * can require updates on (fast-path) scheduler's data structures used to |
| 1292 | * support enqueue/dequeue operations. |
| 1293 | * While the per-CPU rq lock protects fast-path update operations, user-space |
| 1294 | * requests are serialized using a mutex to reduce the risk of conflicting |
| 1295 | * updates or API abuses. |
| 1296 | */ |
| 1297 | static DEFINE_MUTEX(uclamp_mutex); |
| 1298 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1299 | /* Max allowed minimum utilization */ |
YueHaibing | 494dcdf | 2022-04-27 21:10:02 +0800 | [diff] [blame] | 1300 | static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1301 | |
| 1302 | /* Max allowed maximum utilization */ |
YueHaibing | 494dcdf | 2022-04-27 21:10:02 +0800 | [diff] [blame] | 1303 | static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1304 | |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1305 | /* |
| 1306 | * By default RT tasks run at the maximum performance point/capacity of the |
| 1307 | * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to |
| 1308 | * SCHED_CAPACITY_SCALE. |
| 1309 | * |
| 1310 | * This knob allows admins to change the default behavior when uclamp is being |
| 1311 | * used. In battery powered devices, particularly, running at the maximum |
| 1312 | * capacity and frequency will increase energy consumption and shorten the |
| 1313 | * battery life. |
| 1314 | * |
| 1315 | * This knob only affects RT tasks that their uclamp_se->user_defined == false. |
| 1316 | * |
| 1317 | * This knob will not override the system default sched_util_clamp_min defined |
| 1318 | * above. |
| 1319 | */ |
Zhen Ni | 3267e01 | 2022-02-15 19:46:02 +0800 | [diff] [blame] | 1320 | static unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1321 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1322 | /* All clamps are required to be less or equal than these values */ |
| 1323 | static struct uclamp_se uclamp_default[UCLAMP_CNT]; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1324 | |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1325 | /* |
| 1326 | * This static key is used to reduce the uclamp overhead in the fast path. It |
| 1327 | * primarily disables the call to uclamp_rq_{inc, dec}() in |
| 1328 | * enqueue/dequeue_task(). |
| 1329 | * |
| 1330 | * This allows users to continue to enable uclamp in their kernel config with |
| 1331 | * minimum uclamp overhead in the fast path. |
| 1332 | * |
| 1333 | * As soon as userspace modifies any of the uclamp knobs, the static key is |
| 1334 | * enabled, since we have an actual users that make use of uclamp |
| 1335 | * functionality. |
| 1336 | * |
| 1337 | * The knobs that would enable this static key are: |
| 1338 | * |
| 1339 | * * A task modifying its uclamp value with sched_setattr(). |
| 1340 | * * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs. |
| 1341 | * * An admin modifying the cgroup cpu.uclamp.{min, max} |
| 1342 | */ |
| 1343 | DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); |
| 1344 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1345 | /* Integer rounded range for each bucket */ |
| 1346 | #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS) |
| 1347 | |
| 1348 | #define for_each_clamp_id(clamp_id) \ |
| 1349 | for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++) |
| 1350 | |
| 1351 | static inline unsigned int uclamp_bucket_id(unsigned int clamp_value) |
| 1352 | { |
Quentin Perret | 6d2f890 | 2021-04-30 15:14:12 +0000 | [diff] [blame] | 1353 | return min_t(unsigned int, clamp_value / UCLAMP_BUCKET_DELTA, UCLAMP_BUCKETS - 1); |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1354 | } |
| 1355 | |
Valentin Schneider | 7763baa | 2019-11-15 10:39:08 +0000 | [diff] [blame] | 1356 | static inline unsigned int uclamp_none(enum uclamp_id clamp_id) |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1357 | { |
| 1358 | if (clamp_id == UCLAMP_MIN) |
| 1359 | return 0; |
| 1360 | return SCHED_CAPACITY_SCALE; |
| 1361 | } |
| 1362 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1363 | static inline void uclamp_se_set(struct uclamp_se *uc_se, |
| 1364 | unsigned int value, bool user_defined) |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1365 | { |
| 1366 | uc_se->value = value; |
| 1367 | uc_se->bucket_id = uclamp_bucket_id(value); |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1368 | uc_se->user_defined = user_defined; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1369 | } |
| 1370 | |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1371 | static inline unsigned int |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1372 | uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1373 | unsigned int clamp_value) |
| 1374 | { |
| 1375 | /* |
| 1376 | * Avoid blocked utilization pushing up the frequency when we go |
| 1377 | * idle (which drops the max-clamp) by retaining the last known |
| 1378 | * max-clamp. |
| 1379 | */ |
| 1380 | if (clamp_id == UCLAMP_MAX) { |
| 1381 | rq->uclamp_flags |= UCLAMP_FLAG_IDLE; |
| 1382 | return clamp_value; |
| 1383 | } |
| 1384 | |
| 1385 | return uclamp_none(UCLAMP_MIN); |
| 1386 | } |
| 1387 | |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1388 | static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1389 | unsigned int clamp_value) |
| 1390 | { |
| 1391 | /* Reset max-clamp retention only on idle exit */ |
| 1392 | if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) |
| 1393 | return; |
| 1394 | |
Qais Yousef | 2442260 | 2022-08-04 15:36:03 +0100 | [diff] [blame] | 1395 | uclamp_rq_set(rq, clamp_id, clamp_value); |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1396 | } |
| 1397 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1398 | static inline |
Valentin Schneider | 7763baa | 2019-11-15 10:39:08 +0000 | [diff] [blame] | 1399 | unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1400 | unsigned int clamp_value) |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1401 | { |
| 1402 | struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; |
| 1403 | int bucket_id = UCLAMP_BUCKETS - 1; |
| 1404 | |
| 1405 | /* |
| 1406 | * Since both min and max clamps are max aggregated, find the |
| 1407 | * top most bucket with tasks in. |
| 1408 | */ |
| 1409 | for ( ; bucket_id >= 0; bucket_id--) { |
| 1410 | if (!bucket[bucket_id].tasks) |
| 1411 | continue; |
| 1412 | return bucket[bucket_id].value; |
| 1413 | } |
| 1414 | |
| 1415 | /* No tasks -- default clamp values */ |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1416 | return uclamp_idle_value(rq, clamp_id, clamp_value); |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1417 | } |
| 1418 | |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1419 | static void __uclamp_update_util_min_rt_default(struct task_struct *p) |
| 1420 | { |
| 1421 | unsigned int default_util_min; |
| 1422 | struct uclamp_se *uc_se; |
| 1423 | |
| 1424 | lockdep_assert_held(&p->pi_lock); |
| 1425 | |
| 1426 | uc_se = &p->uclamp_req[UCLAMP_MIN]; |
| 1427 | |
| 1428 | /* Only sync if user didn't override the default */ |
| 1429 | if (uc_se->user_defined) |
| 1430 | return; |
| 1431 | |
| 1432 | default_util_min = sysctl_sched_uclamp_util_min_rt_default; |
| 1433 | uclamp_se_set(uc_se, default_util_min, false); |
| 1434 | } |
| 1435 | |
| 1436 | static void uclamp_update_util_min_rt_default(struct task_struct *p) |
| 1437 | { |
| 1438 | struct rq_flags rf; |
| 1439 | struct rq *rq; |
| 1440 | |
| 1441 | if (!rt_task(p)) |
| 1442 | return; |
| 1443 | |
| 1444 | /* Protect updates to p->uclamp_* */ |
| 1445 | rq = task_rq_lock(p, &rf); |
| 1446 | __uclamp_update_util_min_rt_default(p); |
| 1447 | task_rq_unlock(rq, p, &rf); |
| 1448 | } |
| 1449 | |
Patrick Bellasi | 3eac870 | 2019-08-22 14:28:09 +0100 | [diff] [blame] | 1450 | static inline struct uclamp_se |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1451 | uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) |
Patrick Bellasi | 3eac870 | 2019-08-22 14:28:09 +0100 | [diff] [blame] | 1452 | { |
Qais Yousef | 0213b70 | 2021-06-17 17:51:55 +0100 | [diff] [blame] | 1453 | /* Copy by value as we could modify it */ |
Patrick Bellasi | 3eac870 | 2019-08-22 14:28:09 +0100 | [diff] [blame] | 1454 | struct uclamp_se uc_req = p->uclamp_req[clamp_id]; |
| 1455 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
Qais Yousef | 0213b70 | 2021-06-17 17:51:55 +0100 | [diff] [blame] | 1456 | unsigned int tg_min, tg_max, value; |
Patrick Bellasi | 3eac870 | 2019-08-22 14:28:09 +0100 | [diff] [blame] | 1457 | |
| 1458 | /* |
| 1459 | * Tasks in autogroups or root task group will be |
| 1460 | * restricted by system defaults. |
| 1461 | */ |
| 1462 | if (task_group_is_autogroup(task_group(p))) |
| 1463 | return uc_req; |
| 1464 | if (task_group(p) == &root_task_group) |
| 1465 | return uc_req; |
| 1466 | |
Qais Yousef | 0213b70 | 2021-06-17 17:51:55 +0100 | [diff] [blame] | 1467 | tg_min = task_group(p)->uclamp[UCLAMP_MIN].value; |
| 1468 | tg_max = task_group(p)->uclamp[UCLAMP_MAX].value; |
| 1469 | value = uc_req.value; |
| 1470 | value = clamp(value, tg_min, tg_max); |
| 1471 | uclamp_se_set(&uc_req, value, false); |
Patrick Bellasi | 3eac870 | 2019-08-22 14:28:09 +0100 | [diff] [blame] | 1472 | #endif |
| 1473 | |
| 1474 | return uc_req; |
| 1475 | } |
| 1476 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1477 | /* |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1478 | * The effective clamp bucket index of a task depends on, by increasing |
| 1479 | * priority: |
| 1480 | * - the task specific clamp value, when explicitly requested from userspace |
Patrick Bellasi | 3eac870 | 2019-08-22 14:28:09 +0100 | [diff] [blame] | 1481 | * - the task group effective clamp value, for tasks not either in the root |
| 1482 | * group or in an autogroup |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1483 | * - the system default clamp value, defined by the sysadmin |
| 1484 | */ |
| 1485 | static inline struct uclamp_se |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1486 | uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1487 | { |
Patrick Bellasi | 3eac870 | 2019-08-22 14:28:09 +0100 | [diff] [blame] | 1488 | struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1489 | struct uclamp_se uc_max = uclamp_default[clamp_id]; |
| 1490 | |
| 1491 | /* System default restrictions always apply */ |
| 1492 | if (unlikely(uc_req.value > uc_max.value)) |
| 1493 | return uc_max; |
| 1494 | |
| 1495 | return uc_req; |
| 1496 | } |
| 1497 | |
Valentin Schneider | 686516b | 2019-12-11 11:38:48 +0000 | [diff] [blame] | 1498 | unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) |
Patrick Bellasi | 9d20ad7 | 2019-06-21 09:42:11 +0100 | [diff] [blame] | 1499 | { |
| 1500 | struct uclamp_se uc_eff; |
| 1501 | |
| 1502 | /* Task currently refcounted: use back-annotated (effective) value */ |
| 1503 | if (p->uclamp[clamp_id].active) |
Valentin Schneider | 686516b | 2019-12-11 11:38:48 +0000 | [diff] [blame] | 1504 | return (unsigned long)p->uclamp[clamp_id].value; |
Patrick Bellasi | 9d20ad7 | 2019-06-21 09:42:11 +0100 | [diff] [blame] | 1505 | |
| 1506 | uc_eff = uclamp_eff_get(p, clamp_id); |
| 1507 | |
Valentin Schneider | 686516b | 2019-12-11 11:38:48 +0000 | [diff] [blame] | 1508 | return (unsigned long)uc_eff.value; |
Patrick Bellasi | 9d20ad7 | 2019-06-21 09:42:11 +0100 | [diff] [blame] | 1509 | } |
| 1510 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1511 | /* |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1512 | * When a task is enqueued on a rq, the clamp bucket currently defined by the |
| 1513 | * task's uclamp::bucket_id is refcounted on that rq. This also immediately |
| 1514 | * updates the rq's clamp value if required. |
Patrick Bellasi | 60daf9c | 2019-06-21 09:42:03 +0100 | [diff] [blame] | 1515 | * |
| 1516 | * Tasks can have a task-specific value requested from user-space, track |
| 1517 | * within each bucket the maximum value for tasks refcounted in it. |
| 1518 | * This "local max aggregation" allows to track the exact "requested" value |
| 1519 | * for each bucket when all its RUNNABLE tasks require the same clamp. |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1520 | */ |
| 1521 | static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1522 | enum uclamp_id clamp_id) |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1523 | { |
| 1524 | struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; |
| 1525 | struct uclamp_se *uc_se = &p->uclamp[clamp_id]; |
| 1526 | struct uclamp_bucket *bucket; |
| 1527 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 1528 | lockdep_assert_rq_held(rq); |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1529 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1530 | /* Update task effective clamp */ |
| 1531 | p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); |
| 1532 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1533 | bucket = &uc_rq->bucket[uc_se->bucket_id]; |
| 1534 | bucket->tasks++; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1535 | uc_se->active = true; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1536 | |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1537 | uclamp_idle_reset(rq, clamp_id, uc_se->value); |
| 1538 | |
Patrick Bellasi | 60daf9c | 2019-06-21 09:42:03 +0100 | [diff] [blame] | 1539 | /* |
| 1540 | * Local max aggregation: rq buckets always track the max |
| 1541 | * "requested" clamp value of its RUNNABLE tasks. |
| 1542 | */ |
| 1543 | if (bucket->tasks == 1 || uc_se->value > bucket->value) |
| 1544 | bucket->value = uc_se->value; |
| 1545 | |
Qais Yousef | 2442260 | 2022-08-04 15:36:03 +0100 | [diff] [blame] | 1546 | if (uc_se->value > uclamp_rq_get(rq, clamp_id)) |
| 1547 | uclamp_rq_set(rq, clamp_id, uc_se->value); |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1548 | } |
| 1549 | |
| 1550 | /* |
| 1551 | * When a task is dequeued from a rq, the clamp bucket refcounted by the task |
| 1552 | * is released. If this is the last task reference counting the rq's max |
| 1553 | * active clamp value, then the rq's clamp value is updated. |
| 1554 | * |
| 1555 | * Both refcounted tasks and rq's cached clamp values are expected to be |
| 1556 | * always valid. If it's detected they are not, as defensive programming, |
| 1557 | * enforce the expected state and warn. |
| 1558 | */ |
| 1559 | static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1560 | enum uclamp_id clamp_id) |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1561 | { |
| 1562 | struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; |
| 1563 | struct uclamp_se *uc_se = &p->uclamp[clamp_id]; |
| 1564 | struct uclamp_bucket *bucket; |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1565 | unsigned int bkt_clamp; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1566 | unsigned int rq_clamp; |
| 1567 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 1568 | lockdep_assert_rq_held(rq); |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1569 | |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1570 | /* |
| 1571 | * If sched_uclamp_used was enabled after task @p was enqueued, |
| 1572 | * we could end up with unbalanced call to uclamp_rq_dec_id(). |
| 1573 | * |
| 1574 | * In this case the uc_se->active flag should be false since no uclamp |
| 1575 | * accounting was performed at enqueue time and we can just return |
| 1576 | * here. |
| 1577 | * |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 1578 | * Need to be careful of the following enqueue/dequeue ordering |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1579 | * problem too |
| 1580 | * |
| 1581 | * enqueue(taskA) |
| 1582 | * // sched_uclamp_used gets enabled |
| 1583 | * enqueue(taskB) |
| 1584 | * dequeue(taskA) |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 1585 | * // Must not decrement bucket->tasks here |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1586 | * dequeue(taskB) |
| 1587 | * |
| 1588 | * where we could end up with stale data in uc_se and |
| 1589 | * bucket[uc_se->bucket_id]. |
| 1590 | * |
| 1591 | * The following check here eliminates the possibility of such race. |
| 1592 | */ |
| 1593 | if (unlikely(!uc_se->active)) |
| 1594 | return; |
| 1595 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1596 | bucket = &uc_rq->bucket[uc_se->bucket_id]; |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1597 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1598 | SCHED_WARN_ON(!bucket->tasks); |
| 1599 | if (likely(bucket->tasks)) |
| 1600 | bucket->tasks--; |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1601 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1602 | uc_se->active = false; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1603 | |
Patrick Bellasi | 60daf9c | 2019-06-21 09:42:03 +0100 | [diff] [blame] | 1604 | /* |
| 1605 | * Keep "local max aggregation" simple and accept to (possibly) |
| 1606 | * overboost some RUNNABLE tasks in the same bucket. |
| 1607 | * The rq clamp bucket value is reset to its base value whenever |
| 1608 | * there are no more RUNNABLE tasks refcounting it. |
| 1609 | */ |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1610 | if (likely(bucket->tasks)) |
| 1611 | return; |
| 1612 | |
Qais Yousef | 2442260 | 2022-08-04 15:36:03 +0100 | [diff] [blame] | 1613 | rq_clamp = uclamp_rq_get(rq, clamp_id); |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1614 | /* |
| 1615 | * Defensive programming: this should never happen. If it happens, |
| 1616 | * e.g. due to future modification, warn and fixup the expected value. |
| 1617 | */ |
| 1618 | SCHED_WARN_ON(bucket->value > rq_clamp); |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1619 | if (bucket->value >= rq_clamp) { |
| 1620 | bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); |
Qais Yousef | 2442260 | 2022-08-04 15:36:03 +0100 | [diff] [blame] | 1621 | uclamp_rq_set(rq, clamp_id, bkt_clamp); |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1622 | } |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1623 | } |
| 1624 | |
| 1625 | static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) |
| 1626 | { |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1627 | enum uclamp_id clamp_id; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1628 | |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1629 | /* |
| 1630 | * Avoid any overhead until uclamp is actually used by the userspace. |
| 1631 | * |
| 1632 | * The condition is constructed such that a NOP is generated when |
| 1633 | * sched_uclamp_used is disabled. |
| 1634 | */ |
| 1635 | if (!static_branch_unlikely(&sched_uclamp_used)) |
| 1636 | return; |
| 1637 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1638 | if (unlikely(!p->sched_class->uclamp_enabled)) |
| 1639 | return; |
| 1640 | |
| 1641 | for_each_clamp_id(clamp_id) |
| 1642 | uclamp_rq_inc_id(rq, p, clamp_id); |
Patrick Bellasi | e496187 | 2019-06-21 09:42:04 +0100 | [diff] [blame] | 1643 | |
| 1644 | /* Reset clamp idle holding when there is one RUNNABLE task */ |
| 1645 | if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) |
| 1646 | rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1647 | } |
| 1648 | |
| 1649 | static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) |
| 1650 | { |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1651 | enum uclamp_id clamp_id; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1652 | |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1653 | /* |
| 1654 | * Avoid any overhead until uclamp is actually used by the userspace. |
| 1655 | * |
| 1656 | * The condition is constructed such that a NOP is generated when |
| 1657 | * sched_uclamp_used is disabled. |
| 1658 | */ |
| 1659 | if (!static_branch_unlikely(&sched_uclamp_used)) |
| 1660 | return; |
| 1661 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1662 | if (unlikely(!p->sched_class->uclamp_enabled)) |
| 1663 | return; |
| 1664 | |
| 1665 | for_each_clamp_id(clamp_id) |
| 1666 | uclamp_rq_dec_id(rq, p, clamp_id); |
| 1667 | } |
| 1668 | |
Quentin Perret | ca4984a | 2021-08-05 11:21:53 +0100 | [diff] [blame] | 1669 | static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, |
| 1670 | enum uclamp_id clamp_id) |
| 1671 | { |
| 1672 | if (!p->uclamp[clamp_id].active) |
| 1673 | return; |
| 1674 | |
| 1675 | uclamp_rq_dec_id(rq, p, clamp_id); |
| 1676 | uclamp_rq_inc_id(rq, p, clamp_id); |
| 1677 | |
| 1678 | /* |
| 1679 | * Make sure to clear the idle flag if we've transiently reached 0 |
| 1680 | * active tasks on rq. |
| 1681 | */ |
| 1682 | if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE)) |
| 1683 | rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; |
| 1684 | } |
| 1685 | |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 1686 | static inline void |
Qais Yousef | 0213b70 | 2021-06-17 17:51:55 +0100 | [diff] [blame] | 1687 | uclamp_update_active(struct task_struct *p) |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 1688 | { |
Qais Yousef | 0213b70 | 2021-06-17 17:51:55 +0100 | [diff] [blame] | 1689 | enum uclamp_id clamp_id; |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 1690 | struct rq_flags rf; |
| 1691 | struct rq *rq; |
| 1692 | |
| 1693 | /* |
| 1694 | * Lock the task and the rq where the task is (or was) queued. |
| 1695 | * |
| 1696 | * We might lock the (previous) rq of a !RUNNABLE task, but that's the |
| 1697 | * price to pay to safely serialize util_{min,max} updates with |
| 1698 | * enqueues, dequeues and migration operations. |
| 1699 | * This is the same locking schema used by __set_cpus_allowed_ptr(). |
| 1700 | */ |
| 1701 | rq = task_rq_lock(p, &rf); |
| 1702 | |
| 1703 | /* |
| 1704 | * Setting the clamp bucket is serialized by task_rq_lock(). |
| 1705 | * If the task is not yet RUNNABLE and its task_struct is not |
| 1706 | * affecting a valid clamp bucket, the next time it's enqueued, |
| 1707 | * it will already see the updated clamp bucket value. |
| 1708 | */ |
Quentin Perret | ca4984a | 2021-08-05 11:21:53 +0100 | [diff] [blame] | 1709 | for_each_clamp_id(clamp_id) |
| 1710 | uclamp_rq_reinc_id(rq, p, clamp_id); |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 1711 | |
| 1712 | task_rq_unlock(rq, p, &rf); |
| 1713 | } |
| 1714 | |
Qais Yousef | e3b8b6a | 2019-11-05 11:22:12 +0000 | [diff] [blame] | 1715 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 1716 | static inline void |
Qais Yousef | 0213b70 | 2021-06-17 17:51:55 +0100 | [diff] [blame] | 1717 | uclamp_update_active_tasks(struct cgroup_subsys_state *css) |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 1718 | { |
| 1719 | struct css_task_iter it; |
| 1720 | struct task_struct *p; |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 1721 | |
| 1722 | css_task_iter_start(css, 0, &it); |
Qais Yousef | 0213b70 | 2021-06-17 17:51:55 +0100 | [diff] [blame] | 1723 | while ((p = css_task_iter_next(&it))) |
| 1724 | uclamp_update_active(p); |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 1725 | css_task_iter_end(&it); |
| 1726 | } |
| 1727 | |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1728 | static void cpu_util_update_eff(struct cgroup_subsys_state *css); |
YueHaibing | 494dcdf | 2022-04-27 21:10:02 +0800 | [diff] [blame] | 1729 | #endif |
| 1730 | |
| 1731 | #ifdef CONFIG_SYSCTL |
| 1732 | #ifdef CONFIG_UCLAMP_TASK |
| 1733 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1734 | static void uclamp_update_root_tg(void) |
| 1735 | { |
| 1736 | struct task_group *tg = &root_task_group; |
| 1737 | |
| 1738 | uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], |
| 1739 | sysctl_sched_uclamp_util_min, false); |
| 1740 | uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], |
| 1741 | sysctl_sched_uclamp_util_max, false); |
| 1742 | |
| 1743 | rcu_read_lock(); |
| 1744 | cpu_util_update_eff(&root_task_group.css); |
| 1745 | rcu_read_unlock(); |
| 1746 | } |
| 1747 | #else |
| 1748 | static void uclamp_update_root_tg(void) { } |
| 1749 | #endif |
| 1750 | |
YueHaibing | 494dcdf | 2022-04-27 21:10:02 +0800 | [diff] [blame] | 1751 | static void uclamp_sync_util_min_rt_default(void) |
| 1752 | { |
| 1753 | struct task_struct *g, *p; |
| 1754 | |
| 1755 | /* |
| 1756 | * copy_process() sysctl_uclamp |
| 1757 | * uclamp_min_rt = X; |
| 1758 | * write_lock(&tasklist_lock) read_lock(&tasklist_lock) |
| 1759 | * // link thread smp_mb__after_spinlock() |
| 1760 | * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); |
| 1761 | * sched_post_fork() for_each_process_thread() |
| 1762 | * __uclamp_sync_rt() __uclamp_sync_rt() |
| 1763 | * |
| 1764 | * Ensures that either sched_post_fork() will observe the new |
| 1765 | * uclamp_min_rt or for_each_process_thread() will observe the new |
| 1766 | * task. |
| 1767 | */ |
| 1768 | read_lock(&tasklist_lock); |
| 1769 | smp_mb__after_spinlock(); |
| 1770 | read_unlock(&tasklist_lock); |
| 1771 | |
| 1772 | rcu_read_lock(); |
| 1773 | for_each_process_thread(g, p) |
| 1774 | uclamp_update_util_min_rt_default(p); |
| 1775 | rcu_read_unlock(); |
| 1776 | } |
| 1777 | |
Zhen Ni | 3267e01 | 2022-02-15 19:46:02 +0800 | [diff] [blame] | 1778 | static int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, |
Christoph Hellwig | 3292739 | 2020-04-24 08:43:38 +0200 | [diff] [blame] | 1779 | void *buffer, size_t *lenp, loff_t *ppos) |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1780 | { |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1781 | bool update_root_tg = false; |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1782 | int old_min, old_max, old_min_rt; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1783 | int result; |
| 1784 | |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 1785 | mutex_lock(&uclamp_mutex); |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1786 | old_min = sysctl_sched_uclamp_util_min; |
| 1787 | old_max = sysctl_sched_uclamp_util_max; |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1788 | old_min_rt = sysctl_sched_uclamp_util_min_rt_default; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1789 | |
| 1790 | result = proc_dointvec(table, write, buffer, lenp, ppos); |
| 1791 | if (result) |
| 1792 | goto undo; |
| 1793 | if (!write) |
| 1794 | goto done; |
| 1795 | |
| 1796 | if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1797 | sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE || |
| 1798 | sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) { |
| 1799 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1800 | result = -EINVAL; |
| 1801 | goto undo; |
| 1802 | } |
| 1803 | |
| 1804 | if (old_min != sysctl_sched_uclamp_util_min) { |
| 1805 | uclamp_se_set(&uclamp_default[UCLAMP_MIN], |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1806 | sysctl_sched_uclamp_util_min, false); |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1807 | update_root_tg = true; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1808 | } |
| 1809 | if (old_max != sysctl_sched_uclamp_util_max) { |
| 1810 | uclamp_se_set(&uclamp_default[UCLAMP_MAX], |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1811 | sysctl_sched_uclamp_util_max, false); |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1812 | update_root_tg = true; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1813 | } |
| 1814 | |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1815 | if (update_root_tg) { |
| 1816 | static_branch_enable(&sched_uclamp_used); |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1817 | uclamp_update_root_tg(); |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 1818 | } |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1819 | |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1820 | if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) { |
| 1821 | static_branch_enable(&sched_uclamp_used); |
| 1822 | uclamp_sync_util_min_rt_default(); |
| 1823 | } |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1824 | |
| 1825 | /* |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1826 | * We update all RUNNABLE tasks only when task groups are in use. |
| 1827 | * Otherwise, keep it simple and do just a lazy update at each next |
| 1828 | * task enqueue time. |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1829 | */ |
Patrick Bellasi | 7274a5c | 2019-08-22 14:28:08 +0100 | [diff] [blame] | 1830 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1831 | goto done; |
| 1832 | |
| 1833 | undo: |
| 1834 | sysctl_sched_uclamp_util_min = old_min; |
| 1835 | sysctl_sched_uclamp_util_max = old_max; |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1836 | sysctl_sched_uclamp_util_min_rt_default = old_min_rt; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1837 | done: |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 1838 | mutex_unlock(&uclamp_mutex); |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1839 | |
| 1840 | return result; |
| 1841 | } |
YueHaibing | 494dcdf | 2022-04-27 21:10:02 +0800 | [diff] [blame] | 1842 | #endif |
| 1843 | #endif |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1844 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1845 | static int uclamp_validate(struct task_struct *p, |
| 1846 | const struct sched_attr *attr) |
| 1847 | { |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1848 | int util_min = p->uclamp_req[UCLAMP_MIN].value; |
| 1849 | int util_max = p->uclamp_req[UCLAMP_MAX].value; |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1850 | |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1851 | if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) { |
| 1852 | util_min = attr->sched_util_min; |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1853 | |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1854 | if (util_min + 1 > SCHED_CAPACITY_SCALE + 1) |
| 1855 | return -EINVAL; |
| 1856 | } |
| 1857 | |
| 1858 | if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) { |
| 1859 | util_max = attr->sched_util_max; |
| 1860 | |
| 1861 | if (util_max + 1 > SCHED_CAPACITY_SCALE + 1) |
| 1862 | return -EINVAL; |
| 1863 | } |
| 1864 | |
| 1865 | if (util_min != -1 && util_max != -1 && util_min > util_max) |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1866 | return -EINVAL; |
| 1867 | |
Qais Yousef | e65855a | 2020-07-16 12:03:47 +0100 | [diff] [blame] | 1868 | /* |
| 1869 | * We have valid uclamp attributes; make sure uclamp is enabled. |
| 1870 | * |
| 1871 | * We need to do that here, because enabling static branches is a |
| 1872 | * blocking operation which obviously cannot be done while holding |
| 1873 | * scheduler locks. |
| 1874 | */ |
| 1875 | static_branch_enable(&sched_uclamp_used); |
| 1876 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1877 | return 0; |
| 1878 | } |
| 1879 | |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1880 | static bool uclamp_reset(const struct sched_attr *attr, |
| 1881 | enum uclamp_id clamp_id, |
| 1882 | struct uclamp_se *uc_se) |
| 1883 | { |
| 1884 | /* Reset on sched class change for a non user-defined clamp value. */ |
| 1885 | if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)) && |
| 1886 | !uc_se->user_defined) |
| 1887 | return true; |
| 1888 | |
| 1889 | /* Reset on sched_util_{min,max} == -1. */ |
| 1890 | if (clamp_id == UCLAMP_MIN && |
| 1891 | attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN && |
| 1892 | attr->sched_util_min == -1) { |
| 1893 | return true; |
| 1894 | } |
| 1895 | |
| 1896 | if (clamp_id == UCLAMP_MAX && |
| 1897 | attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX && |
| 1898 | attr->sched_util_max == -1) { |
| 1899 | return true; |
| 1900 | } |
| 1901 | |
| 1902 | return false; |
| 1903 | } |
| 1904 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1905 | static void __setscheduler_uclamp(struct task_struct *p, |
| 1906 | const struct sched_attr *attr) |
| 1907 | { |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1908 | enum uclamp_id clamp_id; |
Patrick Bellasi | 1a00d99 | 2019-06-21 09:42:09 +0100 | [diff] [blame] | 1909 | |
Patrick Bellasi | 1a00d99 | 2019-06-21 09:42:09 +0100 | [diff] [blame] | 1910 | for_each_clamp_id(clamp_id) { |
| 1911 | struct uclamp_se *uc_se = &p->uclamp_req[clamp_id]; |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1912 | unsigned int value; |
Patrick Bellasi | 1a00d99 | 2019-06-21 09:42:09 +0100 | [diff] [blame] | 1913 | |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1914 | if (!uclamp_reset(attr, clamp_id, uc_se)) |
Patrick Bellasi | 1a00d99 | 2019-06-21 09:42:09 +0100 | [diff] [blame] | 1915 | continue; |
| 1916 | |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1917 | /* |
| 1918 | * RT by default have a 100% boost value that could be modified |
| 1919 | * at runtime. |
| 1920 | */ |
Patrick Bellasi | 1a00d99 | 2019-06-21 09:42:09 +0100 | [diff] [blame] | 1921 | if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN)) |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1922 | value = sysctl_sched_uclamp_util_min_rt_default; |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1923 | else |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1924 | value = uclamp_none(clamp_id); |
| 1925 | |
| 1926 | uclamp_se_set(uc_se, value, false); |
Patrick Bellasi | 1a00d99 | 2019-06-21 09:42:09 +0100 | [diff] [blame] | 1927 | |
Patrick Bellasi | 1a00d99 | 2019-06-21 09:42:09 +0100 | [diff] [blame] | 1928 | } |
| 1929 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1930 | if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP))) |
| 1931 | return; |
| 1932 | |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1933 | if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN && |
| 1934 | attr->sched_util_min != -1) { |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1935 | uclamp_se_set(&p->uclamp_req[UCLAMP_MIN], |
| 1936 | attr->sched_util_min, true); |
| 1937 | } |
| 1938 | |
Dietmar Eggemann | 480a6ca | 2020-11-13 12:34:54 +0100 | [diff] [blame] | 1939 | if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX && |
| 1940 | attr->sched_util_max != -1) { |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1941 | uclamp_se_set(&p->uclamp_req[UCLAMP_MAX], |
| 1942 | attr->sched_util_max, true); |
| 1943 | } |
| 1944 | } |
| 1945 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1946 | static void uclamp_fork(struct task_struct *p) |
| 1947 | { |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1948 | enum uclamp_id clamp_id; |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1949 | |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1950 | /* |
| 1951 | * We don't need to hold task_rq_lock() when updating p->uclamp_* here |
| 1952 | * as the task is still at its early fork stages. |
| 1953 | */ |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1954 | for_each_clamp_id(clamp_id) |
| 1955 | p->uclamp[clamp_id].active = false; |
Patrick Bellasi | a87498a | 2019-06-21 09:42:08 +0100 | [diff] [blame] | 1956 | |
| 1957 | if (likely(!p->sched_reset_on_fork)) |
| 1958 | return; |
| 1959 | |
| 1960 | for_each_clamp_id(clamp_id) { |
Quentin Perret | eaf5a92 | 2020-04-16 09:59:56 +0100 | [diff] [blame] | 1961 | uclamp_se_set(&p->uclamp_req[clamp_id], |
| 1962 | uclamp_none(clamp_id), false); |
Patrick Bellasi | a87498a | 2019-06-21 09:42:08 +0100 | [diff] [blame] | 1963 | } |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1964 | } |
| 1965 | |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 1966 | static void uclamp_post_fork(struct task_struct *p) |
| 1967 | { |
| 1968 | uclamp_update_util_min_rt_default(p); |
| 1969 | } |
| 1970 | |
Qais Yousef | d81ae8aa | 2020-06-30 12:21:22 +0100 | [diff] [blame] | 1971 | static void __init init_uclamp_rq(struct rq *rq) |
| 1972 | { |
| 1973 | enum uclamp_id clamp_id; |
| 1974 | struct uclamp_rq *uc_rq = rq->uclamp; |
| 1975 | |
| 1976 | for_each_clamp_id(clamp_id) { |
| 1977 | uc_rq[clamp_id] = (struct uclamp_rq) { |
| 1978 | .value = uclamp_none(clamp_id) |
| 1979 | }; |
| 1980 | } |
| 1981 | |
Qais Yousef | 315c4f8 | 2021-12-02 11:20:33 +0000 | [diff] [blame] | 1982 | rq->uclamp_flags = UCLAMP_FLAG_IDLE; |
Qais Yousef | d81ae8aa | 2020-06-30 12:21:22 +0100 | [diff] [blame] | 1983 | } |
| 1984 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1985 | static void __init init_uclamp(void) |
| 1986 | { |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1987 | struct uclamp_se uc_max = {}; |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 1988 | enum uclamp_id clamp_id; |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1989 | int cpu; |
| 1990 | |
Qais Yousef | d81ae8aa | 2020-06-30 12:21:22 +0100 | [diff] [blame] | 1991 | for_each_possible_cpu(cpu) |
| 1992 | init_uclamp_rq(cpu_rq(cpu)); |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1993 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1994 | for_each_clamp_id(clamp_id) { |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1995 | uclamp_se_set(&init_task.uclamp_req[clamp_id], |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 1996 | uclamp_none(clamp_id), false); |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 1997 | } |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 1998 | |
| 1999 | /* System defaults allow max clamp values for both indexes */ |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 2000 | uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 2001 | for_each_clamp_id(clamp_id) { |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 2002 | uclamp_default[clamp_id] = uc_max; |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 2003 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
| 2004 | root_task_group.uclamp_req[clamp_id] = uc_max; |
Patrick Bellasi | 0b60ba2 | 2019-08-22 14:28:07 +0100 | [diff] [blame] | 2005 | root_task_group.uclamp[clamp_id] = uc_max; |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 2006 | #endif |
| 2007 | } |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 2008 | } |
| 2009 | |
| 2010 | #else /* CONFIG_UCLAMP_TASK */ |
| 2011 | static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } |
| 2012 | static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 2013 | static inline int uclamp_validate(struct task_struct *p, |
| 2014 | const struct sched_attr *attr) |
| 2015 | { |
| 2016 | return -EOPNOTSUPP; |
| 2017 | } |
| 2018 | static void __setscheduler_uclamp(struct task_struct *p, |
| 2019 | const struct sched_attr *attr) { } |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 2020 | static inline void uclamp_fork(struct task_struct *p) { } |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 2021 | static inline void uclamp_post_fork(struct task_struct *p) { } |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 2022 | static inline void init_uclamp(void) { } |
| 2023 | #endif /* CONFIG_UCLAMP_TASK */ |
| 2024 | |
Marcelo Tosatti | a1dfb63 | 2021-05-13 01:29:22 +0200 | [diff] [blame] | 2025 | bool sched_task_on_rq(struct task_struct *p) |
| 2026 | { |
| 2027 | return task_on_rq_queued(p); |
| 2028 | } |
| 2029 | |
Kees Cook | 42a20f8 | 2021-09-29 15:02:14 -0700 | [diff] [blame] | 2030 | unsigned long get_wchan(struct task_struct *p) |
| 2031 | { |
| 2032 | unsigned long ip = 0; |
| 2033 | unsigned int state; |
| 2034 | |
| 2035 | if (!p || p == current) |
| 2036 | return 0; |
| 2037 | |
| 2038 | /* Only get wchan if task is blocked and we can keep it that way. */ |
| 2039 | raw_spin_lock_irq(&p->pi_lock); |
| 2040 | state = READ_ONCE(p->__state); |
| 2041 | smp_rmb(); /* see try_to_wake_up() */ |
| 2042 | if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq) |
| 2043 | ip = __get_wchan(p); |
| 2044 | raw_spin_unlock_irq(&p->pi_lock); |
| 2045 | |
| 2046 | return ip; |
| 2047 | } |
| 2048 | |
Peter Zijlstra | 1de6444 | 2015-09-30 17:44:13 +0200 | [diff] [blame] | 2049 | static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) |
Gregory Haskins | 2087a1a | 2008-06-27 14:30:00 -0600 | [diff] [blame] | 2050 | { |
Peter Zijlstra | 0a67d1e | 2016-10-04 16:29:45 +0200 | [diff] [blame] | 2051 | if (!(flags & ENQUEUE_NOCLOCK)) |
| 2052 | update_rq_clock(rq); |
| 2053 | |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 2054 | if (!(flags & ENQUEUE_RESTORE)) { |
Peter Zijlstra | 4e29fb7 | 2021-05-04 22:43:45 +0200 | [diff] [blame] | 2055 | sched_info_enqueue(rq, p); |
Chengming Zhou | 52b33d8 | 2022-09-26 16:19:31 +0800 | [diff] [blame] | 2056 | psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED)); |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 2057 | } |
Peter Zijlstra | 0a67d1e | 2016-10-04 16:29:45 +0200 | [diff] [blame] | 2058 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 2059 | uclamp_rq_inc(rq, p); |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 2060 | p->sched_class->enqueue_task(rq, p, flags); |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 2061 | |
| 2062 | if (sched_core_enabled(rq)) |
| 2063 | sched_core_enqueue(rq, p); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 2064 | } |
| 2065 | |
Peter Zijlstra | 1de6444 | 2015-09-30 17:44:13 +0200 | [diff] [blame] | 2066 | static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 2067 | { |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 2068 | if (sched_core_enabled(rq)) |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 2069 | sched_core_dequeue(rq, p, flags); |
Peter Zijlstra | 8a311c7 | 2020-11-17 18:19:36 -0500 | [diff] [blame] | 2070 | |
Peter Zijlstra | 0a67d1e | 2016-10-04 16:29:45 +0200 | [diff] [blame] | 2071 | if (!(flags & DEQUEUE_NOCLOCK)) |
| 2072 | update_rq_clock(rq); |
| 2073 | |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 2074 | if (!(flags & DEQUEUE_SAVE)) { |
Peter Zijlstra | 4e29fb7 | 2021-05-04 22:43:45 +0200 | [diff] [blame] | 2075 | sched_info_dequeue(rq, p); |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 2076 | psi_dequeue(p, flags & DEQUEUE_SLEEP); |
| 2077 | } |
Peter Zijlstra | 0a67d1e | 2016-10-04 16:29:45 +0200 | [diff] [blame] | 2078 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 2079 | uclamp_rq_dec(rq, p); |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 2080 | p->sched_class->dequeue_task(rq, p, flags); |
Ingo Molnar | 71f8bd4 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 2081 | } |
| 2082 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 2083 | void activate_task(struct rq *rq, struct task_struct *p, int flags) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 2084 | { |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 2085 | enqueue_task(rq, p, flags); |
Peter Zijlstra | 7dd7788 | 2019-04-09 09:59:05 +0200 | [diff] [blame] | 2086 | |
| 2087 | p->on_rq = TASK_ON_RQ_QUEUED; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 2088 | } |
| 2089 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 2090 | void deactivate_task(struct rq *rq, struct task_struct *p, int flags) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 2091 | { |
Peter Zijlstra | 7dd7788 | 2019-04-09 09:59:05 +0200 | [diff] [blame] | 2092 | p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; |
| 2093 | |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 2094 | dequeue_task(rq, p, flags); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 2095 | } |
| 2096 | |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 2097 | static inline int __normal_prio(int policy, int rt_prio, int nice) |
Ingo Molnar | 1453118 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 2098 | { |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 2099 | int prio; |
| 2100 | |
| 2101 | if (dl_policy(policy)) |
| 2102 | prio = MAX_DL_PRIO - 1; |
| 2103 | else if (rt_policy(policy)) |
| 2104 | prio = MAX_RT_PRIO - 1 - rt_prio; |
| 2105 | else |
| 2106 | prio = NICE_TO_PRIO(nice); |
| 2107 | |
| 2108 | return prio; |
Ingo Molnar | 1453118 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 2109 | } |
| 2110 | |
| 2111 | /* |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 2112 | * Calculate the expected normal priority: i.e. priority |
| 2113 | * without taking RT-inheritance into account. Might be |
| 2114 | * boosted by interactivity modifiers. Changes upon fork, |
| 2115 | * setprio syscalls, and whenever the interactivity |
| 2116 | * estimator recalculates. |
| 2117 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 2118 | static inline int normal_prio(struct task_struct *p) |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 2119 | { |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 2120 | return __normal_prio(p->policy, p->rt_priority, PRIO_TO_NICE(p->static_prio)); |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 2121 | } |
| 2122 | |
| 2123 | /* |
| 2124 | * Calculate the current priority, i.e. the priority |
| 2125 | * taken into account by the scheduler. This value might |
| 2126 | * be boosted by RT tasks, or might be boosted by |
| 2127 | * interactivity modifiers. Will be RT if the task got |
| 2128 | * RT-boosted. If not then it returns p->normal_prio. |
| 2129 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 2130 | static int effective_prio(struct task_struct *p) |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 2131 | { |
| 2132 | p->normal_prio = normal_prio(p); |
| 2133 | /* |
| 2134 | * If we are RT tasks or we were boosted to RT priority, |
| 2135 | * keep the priority unchanged. Otherwise, update priority |
| 2136 | * to the normal priority: |
| 2137 | */ |
| 2138 | if (!rt_prio(p->prio)) |
| 2139 | return p->normal_prio; |
| 2140 | return p->prio; |
| 2141 | } |
| 2142 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2143 | /** |
| 2144 | * task_curr - is this task currently executing on a CPU? |
| 2145 | * @p: the task in question. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 2146 | * |
| 2147 | * Return: 1 if the task is currently executing. 0 otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2148 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 2149 | inline int task_curr(const struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2150 | { |
| 2151 | return cpu_curr(task_cpu(p)) == p; |
| 2152 | } |
| 2153 | |
Kirill Tkhai | 67dfa1b | 2014-10-27 17:40:52 +0300 | [diff] [blame] | 2154 | /* |
Peter Zijlstra | 4c9a4bc | 2015-06-11 14:46:39 +0200 | [diff] [blame] | 2155 | * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, |
| 2156 | * use the balance_callback list if you want balancing. |
| 2157 | * |
| 2158 | * this means any call to check_class_changed() must be followed by a call to |
| 2159 | * balance_callback(). |
Kirill Tkhai | 67dfa1b | 2014-10-27 17:40:52 +0300 | [diff] [blame] | 2160 | */ |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 2161 | static inline void check_class_changed(struct rq *rq, struct task_struct *p, |
| 2162 | const struct sched_class *prev_class, |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 2163 | int oldprio) |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 2164 | { |
| 2165 | if (prev_class != p->sched_class) { |
| 2166 | if (prev_class->switched_from) |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 2167 | prev_class->switched_from(rq, p); |
Peter Zijlstra | 4c9a4bc | 2015-06-11 14:46:39 +0200 | [diff] [blame] | 2168 | |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 2169 | p->sched_class->switched_to(rq, p); |
Dario Faggioli | 2d3d891 | 2013-11-07 14:43:44 +0100 | [diff] [blame] | 2170 | } else if (oldprio != p->prio || dl_task(p)) |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 2171 | p->sched_class->prio_changed(rq, p, oldprio); |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 2172 | } |
| 2173 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 2174 | void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) |
Peter Zijlstra | 1e5a740 | 2010-10-31 12:37:04 +0100 | [diff] [blame] | 2175 | { |
Kirill Tkhai | aa93cd5 | 2019-12-19 16:44:55 -0500 | [diff] [blame] | 2176 | if (p->sched_class == rq->curr->sched_class) |
Peter Zijlstra | 1e5a740 | 2010-10-31 12:37:04 +0100 | [diff] [blame] | 2177 | rq->curr->sched_class->check_preempt_curr(rq, p, flags); |
Peter Zijlstra | 546a3fe | 2022-05-17 13:46:54 +0200 | [diff] [blame] | 2178 | else if (sched_class_above(p->sched_class, rq->curr->sched_class)) |
Kirill Tkhai | aa93cd5 | 2019-12-19 16:44:55 -0500 | [diff] [blame] | 2179 | resched_curr(rq); |
Peter Zijlstra | 1e5a740 | 2010-10-31 12:37:04 +0100 | [diff] [blame] | 2180 | |
| 2181 | /* |
| 2182 | * A queue event has occurred, and we're going to schedule. In |
| 2183 | * this case, we can save a useless back to back clock update. |
| 2184 | */ |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 2185 | if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) |
Davidlohr Bueso | adcc8da | 2018-04-04 09:15:39 -0700 | [diff] [blame] | 2186 | rq_clock_skip_update(rq); |
Peter Zijlstra | 1e5a740 | 2010-10-31 12:37:04 +0100 | [diff] [blame] | 2187 | } |
| 2188 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2189 | #ifdef CONFIG_SMP |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 2190 | |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2191 | static void |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2192 | __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx); |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2193 | |
| 2194 | static int __set_cpus_allowed_ptr(struct task_struct *p, |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2195 | struct affinity_context *ctx); |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2196 | |
| 2197 | static void migrate_disable_switch(struct rq *rq, struct task_struct *p) |
| 2198 | { |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2199 | struct affinity_context ac = { |
| 2200 | .new_mask = cpumask_of(rq->cpu), |
| 2201 | .flags = SCA_MIGRATE_DISABLE, |
| 2202 | }; |
| 2203 | |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2204 | if (likely(!p->migration_disabled)) |
| 2205 | return; |
| 2206 | |
| 2207 | if (p->cpus_ptr != &p->cpus_mask) |
| 2208 | return; |
| 2209 | |
| 2210 | /* |
| 2211 | * Violates locking rules! see comment in __do_set_cpus_allowed(). |
| 2212 | */ |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2213 | __do_set_cpus_allowed(p, &ac); |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2214 | } |
| 2215 | |
| 2216 | void migrate_disable(void) |
| 2217 | { |
Thomas Gleixner | 3015ef4 | 2020-08-26 14:08:10 +0200 | [diff] [blame] | 2218 | struct task_struct *p = current; |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2219 | |
Thomas Gleixner | 3015ef4 | 2020-08-26 14:08:10 +0200 | [diff] [blame] | 2220 | if (p->migration_disabled) { |
| 2221 | p->migration_disabled++; |
| 2222 | return; |
| 2223 | } |
| 2224 | |
| 2225 | preempt_disable(); |
| 2226 | this_rq()->nr_pinned++; |
| 2227 | p->migration_disabled = 1; |
| 2228 | preempt_enable(); |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2229 | } |
| 2230 | EXPORT_SYMBOL_GPL(migrate_disable); |
| 2231 | |
| 2232 | void migrate_enable(void) |
| 2233 | { |
| 2234 | struct task_struct *p = current; |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2235 | struct affinity_context ac = { |
| 2236 | .new_mask = &p->cpus_mask, |
| 2237 | .flags = SCA_MIGRATE_ENABLE, |
| 2238 | }; |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2239 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2240 | if (p->migration_disabled > 1) { |
| 2241 | p->migration_disabled--; |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2242 | return; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2243 | } |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2244 | |
Sebastian Andrzej Siewior | 9d0df37 | 2021-11-29 18:46:44 +0100 | [diff] [blame] | 2245 | if (WARN_ON_ONCE(!p->migration_disabled)) |
| 2246 | return; |
| 2247 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2248 | /* |
| 2249 | * Ensure stop_task runs either before or after this, and that |
| 2250 | * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule(). |
| 2251 | */ |
| 2252 | preempt_disable(); |
| 2253 | if (p->cpus_ptr != &p->cpus_mask) |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2254 | __set_cpus_allowed_ptr(p, &ac); |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2255 | /* |
| 2256 | * Mustn't clear migration_disabled() until cpus_ptr points back at the |
| 2257 | * regular cpus_mask, otherwise things that race (eg. |
| 2258 | * select_fallback_rq) get confused. |
| 2259 | */ |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2260 | barrier(); |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2261 | p->migration_disabled = 0; |
Thomas Gleixner | 3015ef4 | 2020-08-26 14:08:10 +0200 | [diff] [blame] | 2262 | this_rq()->nr_pinned--; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2263 | preempt_enable(); |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2264 | } |
| 2265 | EXPORT_SYMBOL_GPL(migrate_enable); |
| 2266 | |
Thomas Gleixner | 3015ef4 | 2020-08-26 14:08:10 +0200 | [diff] [blame] | 2267 | static inline bool rq_has_pinned_tasks(struct rq *rq) |
| 2268 | { |
| 2269 | return rq->nr_pinned; |
| 2270 | } |
| 2271 | |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 2272 | /* |
Joel Savitz | bee9853 | 2019-03-06 20:13:33 -0500 | [diff] [blame] | 2273 | * Per-CPU kthreads are allowed to run on !active && online CPUs, see |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 2274 | * __set_cpus_allowed_ptr() and select_fallback_rq(). |
| 2275 | */ |
| 2276 | static inline bool is_cpu_allowed(struct task_struct *p, int cpu) |
| 2277 | { |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 2278 | /* When not in the task's cpumask, no point in looking further. */ |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 2279 | if (!cpumask_test_cpu(cpu, p->cpus_ptr)) |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 2280 | return false; |
| 2281 | |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 2282 | /* migrate_disabled() must be allowed to finish. */ |
| 2283 | if (is_migration_disabled(p)) |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 2284 | return cpu_online(cpu); |
| 2285 | |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 2286 | /* Non kernel threads are not allowed during either online or offline. */ |
| 2287 | if (!(p->flags & PF_KTHREAD)) |
Will Deacon | 9ae606b | 2021-07-30 12:24:28 +0100 | [diff] [blame] | 2288 | return cpu_active(cpu) && task_cpu_possible(cpu, p); |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 2289 | |
| 2290 | /* KTHREAD_IS_PER_CPU is always allowed. */ |
| 2291 | if (kthread_is_per_cpu(p)) |
| 2292 | return cpu_online(cpu); |
| 2293 | |
| 2294 | /* Regular kernel threads don't get to stay during offline. */ |
Peter Zijlstra | b5c4477 | 2021-01-21 16:09:32 +0100 | [diff] [blame] | 2295 | if (cpu_dying(cpu)) |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 2296 | return false; |
| 2297 | |
| 2298 | /* But are allowed during online. */ |
| 2299 | return cpu_online(cpu); |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 2300 | } |
| 2301 | |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2302 | /* |
| 2303 | * This is how migration works: |
| 2304 | * |
| 2305 | * 1) we invoke migration_cpu_stop() on the target CPU using |
| 2306 | * stop_one_cpu(). |
| 2307 | * 2) stopper starts to run (implicitly forcing the migrated thread |
| 2308 | * off the CPU) |
| 2309 | * 3) it checks whether the migrated task is still in the wrong runqueue. |
| 2310 | * 4) if it's in the wrong runqueue then the migration thread removes |
| 2311 | * it and puts it into the right queue. |
| 2312 | * 5) stopper completes and stop_one_cpu() returns and the migration |
| 2313 | * is done. |
| 2314 | */ |
| 2315 | |
| 2316 | /* |
| 2317 | * move_queued_task - move a queued task to new rq. |
| 2318 | * |
| 2319 | * Returns (locked) new rq. Old rq's lock is released. |
| 2320 | */ |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 2321 | static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, |
| 2322 | struct task_struct *p, int new_cpu) |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2323 | { |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 2324 | lockdep_assert_rq_held(rq); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2325 | |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 2326 | deactivate_task(rq, p, DEQUEUE_NOCLOCK); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2327 | set_task_cpu(p, new_cpu); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 2328 | rq_unlock(rq, rf); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2329 | |
| 2330 | rq = cpu_rq(new_cpu); |
| 2331 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 2332 | rq_lock(rq, rf); |
Ingo Molnar | 09348d7 | 2022-08-11 08:54:52 +0200 | [diff] [blame] | 2333 | WARN_ON_ONCE(task_cpu(p) != new_cpu); |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 2334 | activate_task(rq, p, 0); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2335 | check_preempt_curr(rq, p, 0); |
| 2336 | |
| 2337 | return rq; |
| 2338 | } |
| 2339 | |
| 2340 | struct migration_arg { |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2341 | struct task_struct *task; |
| 2342 | int dest_cpu; |
| 2343 | struct set_affinity_pending *pending; |
| 2344 | }; |
| 2345 | |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2346 | /* |
| 2347 | * @refs: number of wait_for_completion() |
| 2348 | * @stop_pending: is @stop_work in use |
| 2349 | */ |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2350 | struct set_affinity_pending { |
| 2351 | refcount_t refs; |
Peter Zijlstra | 9e81889 | 2021-02-24 11:31:09 +0100 | [diff] [blame] | 2352 | unsigned int stop_pending; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2353 | struct completion done; |
| 2354 | struct cpu_stop_work stop_work; |
| 2355 | struct migration_arg arg; |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2356 | }; |
| 2357 | |
| 2358 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 2359 | * Move (not current) task off this CPU, onto the destination CPU. We're doing |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2360 | * this because either it can't run here any more (set_cpus_allowed() |
| 2361 | * away from this CPU, or CPU going down), or because we're |
| 2362 | * attempting to rebalance this task on exec (sched_exec). |
| 2363 | * |
| 2364 | * So we race with normal scheduler movements, but that's OK, as long |
| 2365 | * as the task is no longer on this CPU. |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2366 | */ |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 2367 | static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, |
| 2368 | struct task_struct *p, int dest_cpu) |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2369 | { |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2370 | /* Affinity changed (again). */ |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 2371 | if (!is_cpu_allowed(p, dest_cpu)) |
Peter Zijlstra | 5e16bbc | 2015-06-11 14:46:51 +0200 | [diff] [blame] | 2372 | return rq; |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2373 | |
Peter Zijlstra | 15ff991 | 2016-10-05 17:59:32 +0200 | [diff] [blame] | 2374 | update_rq_clock(rq); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 2375 | rq = move_queued_task(rq, rf, p, dest_cpu); |
Peter Zijlstra | 5e16bbc | 2015-06-11 14:46:51 +0200 | [diff] [blame] | 2376 | |
| 2377 | return rq; |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2378 | } |
| 2379 | |
| 2380 | /* |
| 2381 | * migration_cpu_stop - this will be executed by a highprio stopper thread |
| 2382 | * and performs thread migration by bumping thread off CPU then |
| 2383 | * 'pushing' onto another runqueue. |
| 2384 | */ |
| 2385 | static int migration_cpu_stop(void *data) |
| 2386 | { |
| 2387 | struct migration_arg *arg = data; |
Peter Zijlstra | c20cf06 | 2021-02-24 11:50:39 +0100 | [diff] [blame] | 2388 | struct set_affinity_pending *pending = arg->pending; |
Peter Zijlstra | 5e16bbc | 2015-06-11 14:46:51 +0200 | [diff] [blame] | 2389 | struct task_struct *p = arg->task; |
| 2390 | struct rq *rq = this_rq(); |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2391 | bool complete = false; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 2392 | struct rq_flags rf; |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2393 | |
| 2394 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 2395 | * The original target CPU might have gone down and we might |
| 2396 | * be on another CPU but it doesn't matter. |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2397 | */ |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2398 | local_irq_save(rf.flags); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2399 | /* |
| 2400 | * We need to explicitly wake pending tasks before running |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 2401 | * __migrate_task() such that we will not miss enforcing cpus_ptr |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2402 | * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. |
| 2403 | */ |
Thomas Gleixner | 16bf5a5 | 2022-04-13 15:31:03 +0200 | [diff] [blame] | 2404 | flush_smp_call_function_queue(); |
Peter Zijlstra | 5e16bbc | 2015-06-11 14:46:51 +0200 | [diff] [blame] | 2405 | |
| 2406 | raw_spin_lock(&p->pi_lock); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 2407 | rq_lock(rq, &rf); |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2408 | |
Peter Zijlstra | 5e16bbc | 2015-06-11 14:46:51 +0200 | [diff] [blame] | 2409 | /* |
Valentin Schneider | e140749 | 2021-02-25 10:22:30 +0100 | [diff] [blame] | 2410 | * If we were passed a pending, then ->stop_pending was set, thus |
| 2411 | * p->migration_pending must have remained stable. |
| 2412 | */ |
| 2413 | WARN_ON_ONCE(pending && pending != p->migration_pending); |
| 2414 | |
| 2415 | /* |
Peter Zijlstra | 5e16bbc | 2015-06-11 14:46:51 +0200 | [diff] [blame] | 2416 | * If task_rq(p) != rq, it cannot be migrated here, because we're |
| 2417 | * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because |
| 2418 | * we're holding p->pi_lock. |
| 2419 | */ |
Cheng Chao | bf89a30 | 2016-09-14 10:01:50 +0800 | [diff] [blame] | 2420 | if (task_rq(p) == rq) { |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2421 | if (is_migration_disabled(p)) |
| 2422 | goto out; |
Peter Zijlstra | 5e16bbc | 2015-06-11 14:46:51 +0200 | [diff] [blame] | 2423 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2424 | if (pending) { |
Valentin Schneider | e140749 | 2021-02-25 10:22:30 +0100 | [diff] [blame] | 2425 | p->migration_pending = NULL; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2426 | complete = true; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2427 | |
Peter Zijlstra | 3f1bc11 | 2021-02-24 11:21:35 +0100 | [diff] [blame] | 2428 | if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) |
| 2429 | goto out; |
Peter Zijlstra | 3f1bc11 | 2021-02-24 11:21:35 +0100 | [diff] [blame] | 2430 | } |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2431 | |
| 2432 | if (task_on_rq_queued(p)) |
Valentin Schneider | 475ea6c | 2021-05-26 21:57:50 +0100 | [diff] [blame] | 2433 | rq = __migrate_task(rq, &rf, p, arg->dest_cpu); |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2434 | else |
Valentin Schneider | 475ea6c | 2021-05-26 21:57:50 +0100 | [diff] [blame] | 2435 | p->wake_cpu = arg->dest_cpu; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2436 | |
Peter Zijlstra | 3f1bc11 | 2021-02-24 11:21:35 +0100 | [diff] [blame] | 2437 | /* |
| 2438 | * XXX __migrate_task() can fail, at which point we might end |
| 2439 | * up running on a dodgy CPU, AFAICT this can only happen |
| 2440 | * during CPU hotplug, at which point we'll get pushed out |
| 2441 | * anyway, so it's probably not a big deal. |
| 2442 | */ |
| 2443 | |
Peter Zijlstra | c20cf06 | 2021-02-24 11:50:39 +0100 | [diff] [blame] | 2444 | } else if (pending) { |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2445 | /* |
| 2446 | * This happens when we get migrated between migrate_enable()'s |
| 2447 | * preempt_enable() and scheduling the stopper task. At that |
| 2448 | * point we're a regular task again and not current anymore. |
| 2449 | * |
| 2450 | * A !PREEMPT kernel has a giant hole here, which makes it far |
| 2451 | * more likely. |
| 2452 | */ |
| 2453 | |
| 2454 | /* |
Valentin Schneider | d707faa | 2020-11-13 11:24:14 +0000 | [diff] [blame] | 2455 | * The task moved before the stopper got to run. We're holding |
| 2456 | * ->pi_lock, so the allowed mask is stable - if it got |
| 2457 | * somewhere allowed, we're done. |
| 2458 | */ |
Peter Zijlstra | c20cf06 | 2021-02-24 11:50:39 +0100 | [diff] [blame] | 2459 | if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { |
Valentin Schneider | e140749 | 2021-02-25 10:22:30 +0100 | [diff] [blame] | 2460 | p->migration_pending = NULL; |
Valentin Schneider | d707faa | 2020-11-13 11:24:14 +0000 | [diff] [blame] | 2461 | complete = true; |
| 2462 | goto out; |
| 2463 | } |
| 2464 | |
| 2465 | /* |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2466 | * When migrate_enable() hits a rq mis-match we can't reliably |
| 2467 | * determine is_migration_disabled() and so have to chase after |
| 2468 | * it. |
| 2469 | */ |
Peter Zijlstra | 9e81889 | 2021-02-24 11:31:09 +0100 | [diff] [blame] | 2470 | WARN_ON_ONCE(!pending->stop_pending); |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2471 | task_rq_unlock(rq, p, &rf); |
| 2472 | stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, |
| 2473 | &pending->arg, &pending->stop_work); |
| 2474 | return 0; |
| 2475 | } |
| 2476 | out: |
Peter Zijlstra | 9e81889 | 2021-02-24 11:31:09 +0100 | [diff] [blame] | 2477 | if (pending) |
| 2478 | pending->stop_pending = false; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2479 | task_rq_unlock(rq, p, &rf); |
| 2480 | |
| 2481 | if (complete) |
| 2482 | complete_all(&pending->done); |
| 2483 | |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2484 | return 0; |
| 2485 | } |
| 2486 | |
Peter Zijlstra | a7c8155 | 2020-09-28 17:06:07 +0200 | [diff] [blame] | 2487 | int push_cpu_stop(void *arg) |
| 2488 | { |
| 2489 | struct rq *lowest_rq = NULL, *rq = this_rq(); |
| 2490 | struct task_struct *p = arg; |
| 2491 | |
| 2492 | raw_spin_lock_irq(&p->pi_lock); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 2493 | raw_spin_rq_lock(rq); |
Peter Zijlstra | a7c8155 | 2020-09-28 17:06:07 +0200 | [diff] [blame] | 2494 | |
| 2495 | if (task_rq(p) != rq) |
| 2496 | goto out_unlock; |
| 2497 | |
| 2498 | if (is_migration_disabled(p)) { |
| 2499 | p->migration_flags |= MDF_PUSH; |
| 2500 | goto out_unlock; |
| 2501 | } |
| 2502 | |
| 2503 | p->migration_flags &= ~MDF_PUSH; |
| 2504 | |
| 2505 | if (p->sched_class->find_lock_rq) |
| 2506 | lowest_rq = p->sched_class->find_lock_rq(p, rq); |
| 2507 | |
| 2508 | if (!lowest_rq) |
| 2509 | goto out_unlock; |
| 2510 | |
| 2511 | // XXX validate p is still the highest prio task |
| 2512 | if (task_rq(p) == rq) { |
| 2513 | deactivate_task(rq, p, 0); |
| 2514 | set_task_cpu(p, lowest_rq->cpu); |
| 2515 | activate_task(lowest_rq, p, 0); |
| 2516 | resched_curr(lowest_rq); |
| 2517 | } |
| 2518 | |
| 2519 | double_unlock_balance(rq, lowest_rq); |
| 2520 | |
| 2521 | out_unlock: |
| 2522 | rq->push_busy = false; |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 2523 | raw_spin_rq_unlock(rq); |
Peter Zijlstra | a7c8155 | 2020-09-28 17:06:07 +0200 | [diff] [blame] | 2524 | raw_spin_unlock_irq(&p->pi_lock); |
| 2525 | |
| 2526 | put_task_struct(p); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2527 | return 0; |
| 2528 | } |
| 2529 | |
Peter Zijlstra | c5b2803 | 2015-05-15 17:43:35 +0200 | [diff] [blame] | 2530 | /* |
| 2531 | * sched_class::set_cpus_allowed must do the below, but is not required to |
| 2532 | * actually call this function. |
| 2533 | */ |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2534 | void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx) |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2535 | { |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2536 | if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { |
| 2537 | p->cpus_ptr = ctx->new_mask; |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2538 | return; |
| 2539 | } |
| 2540 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2541 | cpumask_copy(&p->cpus_mask, ctx->new_mask); |
| 2542 | p->nr_cpus_allowed = cpumask_weight(ctx->new_mask); |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 2543 | |
| 2544 | /* |
| 2545 | * Swap in a new user_cpus_ptr if SCA_USER flag set |
| 2546 | */ |
| 2547 | if (ctx->flags & SCA_USER) |
| 2548 | swap(p->user_cpus_ptr, ctx->user_mask); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2549 | } |
| 2550 | |
Peter Zijlstra | 9cfc3e1 | 2020-09-16 14:59:08 +0200 | [diff] [blame] | 2551 | static void |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2552 | __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2553 | { |
Peter Zijlstra | 6c37067 | 2015-05-15 17:43:36 +0200 | [diff] [blame] | 2554 | struct rq *rq = task_rq(p); |
| 2555 | bool queued, running; |
| 2556 | |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2557 | /* |
| 2558 | * This here violates the locking rules for affinity, since we're only |
| 2559 | * supposed to change these variables while holding both rq->lock and |
| 2560 | * p->pi_lock. |
| 2561 | * |
| 2562 | * HOWEVER, it magically works, because ttwu() is the only code that |
| 2563 | * accesses these variables under p->pi_lock and only does so after |
| 2564 | * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule() |
| 2565 | * before finish_task(). |
| 2566 | * |
| 2567 | * XXX do further audits, this smells like something putrid. |
| 2568 | */ |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2569 | if (ctx->flags & SCA_MIGRATE_DISABLE) |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2570 | SCHED_WARN_ON(!p->on_cpu); |
| 2571 | else |
| 2572 | lockdep_assert_held(&p->pi_lock); |
Peter Zijlstra | 6c37067 | 2015-05-15 17:43:36 +0200 | [diff] [blame] | 2573 | |
| 2574 | queued = task_on_rq_queued(p); |
| 2575 | running = task_current(rq, p); |
| 2576 | |
| 2577 | if (queued) { |
| 2578 | /* |
| 2579 | * Because __kthread_bind() calls this on blocked tasks without |
| 2580 | * holding rq->lock. |
| 2581 | */ |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 2582 | lockdep_assert_rq_held(rq); |
Peter Zijlstra | 7a57f32 | 2017-02-21 14:47:02 +0100 | [diff] [blame] | 2583 | dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); |
Peter Zijlstra | 6c37067 | 2015-05-15 17:43:36 +0200 | [diff] [blame] | 2584 | } |
| 2585 | if (running) |
| 2586 | put_prev_task(rq, p); |
| 2587 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2588 | p->sched_class->set_cpus_allowed(p, ctx); |
Peter Zijlstra | 6c37067 | 2015-05-15 17:43:36 +0200 | [diff] [blame] | 2589 | |
Peter Zijlstra | 6c37067 | 2015-05-15 17:43:36 +0200 | [diff] [blame] | 2590 | if (queued) |
Peter Zijlstra | 7134b3e | 2017-02-21 14:23:38 +0100 | [diff] [blame] | 2591 | enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); |
Vincent Guittot | a399d23 | 2016-09-12 09:47:52 +0200 | [diff] [blame] | 2592 | if (running) |
Peter Zijlstra | 03b7fad | 2019-05-29 20:36:41 +0000 | [diff] [blame] | 2593 | set_next_task(rq, p); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2594 | } |
| 2595 | |
Waiman Long | 851a723 | 2022-09-22 14:00:41 -0400 | [diff] [blame] | 2596 | /* |
| 2597 | * Used for kthread_bind() and select_fallback_rq(), in both cases the user |
| 2598 | * affinity (if any) should be destroyed too. |
| 2599 | */ |
Peter Zijlstra | 9cfc3e1 | 2020-09-16 14:59:08 +0200 | [diff] [blame] | 2600 | void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) |
| 2601 | { |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2602 | struct affinity_context ac = { |
| 2603 | .new_mask = new_mask, |
Waiman Long | 851a723 | 2022-09-22 14:00:41 -0400 | [diff] [blame] | 2604 | .user_mask = NULL, |
| 2605 | .flags = SCA_USER, /* clear the user requested mask */ |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2606 | }; |
Waiman Long | 9a5418b | 2022-12-30 23:11:20 -0500 | [diff] [blame] | 2607 | union cpumask_rcuhead { |
| 2608 | cpumask_t cpumask; |
| 2609 | struct rcu_head rcu; |
| 2610 | }; |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2611 | |
| 2612 | __do_set_cpus_allowed(p, &ac); |
Waiman Long | 9a5418b | 2022-12-30 23:11:20 -0500 | [diff] [blame] | 2613 | |
| 2614 | /* |
| 2615 | * Because this is called with p->pi_lock held, it is not possible |
| 2616 | * to use kfree() here (when PREEMPT_RT=y), therefore punt to using |
| 2617 | * kfree_rcu(). |
| 2618 | */ |
| 2619 | kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu); |
| 2620 | } |
| 2621 | |
| 2622 | static cpumask_t *alloc_user_cpus_ptr(int node) |
| 2623 | { |
| 2624 | /* |
| 2625 | * See do_set_cpus_allowed() above for the rcu_head usage. |
| 2626 | */ |
| 2627 | int size = max_t(int, cpumask_size(), sizeof(struct rcu_head)); |
| 2628 | |
| 2629 | return kmalloc_node(size, GFP_KERNEL, node); |
Peter Zijlstra | 9cfc3e1 | 2020-09-16 14:59:08 +0200 | [diff] [blame] | 2630 | } |
| 2631 | |
Will Deacon | b90ca8b | 2021-07-30 12:24:33 +0100 | [diff] [blame] | 2632 | int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, |
| 2633 | int node) |
| 2634 | { |
Waiman Long | 87ca4f9 | 2022-12-30 23:11:19 -0500 | [diff] [blame] | 2635 | cpumask_t *user_mask; |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 2636 | unsigned long flags; |
| 2637 | |
Waiman Long | 87ca4f9 | 2022-12-30 23:11:19 -0500 | [diff] [blame] | 2638 | /* |
| 2639 | * Always clear dst->user_cpus_ptr first as their user_cpus_ptr's |
| 2640 | * may differ by now due to racing. |
| 2641 | */ |
| 2642 | dst->user_cpus_ptr = NULL; |
| 2643 | |
| 2644 | /* |
| 2645 | * This check is racy and losing the race is a valid situation. |
| 2646 | * It is not worth the extra overhead of taking the pi_lock on |
| 2647 | * every fork/clone. |
| 2648 | */ |
| 2649 | if (data_race(!src->user_cpus_ptr)) |
Will Deacon | b90ca8b | 2021-07-30 12:24:33 +0100 | [diff] [blame] | 2650 | return 0; |
| 2651 | |
Waiman Long | 9a5418b | 2022-12-30 23:11:20 -0500 | [diff] [blame] | 2652 | user_mask = alloc_user_cpus_ptr(node); |
Waiman Long | 87ca4f9 | 2022-12-30 23:11:19 -0500 | [diff] [blame] | 2653 | if (!user_mask) |
Will Deacon | b90ca8b | 2021-07-30 12:24:33 +0100 | [diff] [blame] | 2654 | return -ENOMEM; |
| 2655 | |
Waiman Long | 87ca4f9 | 2022-12-30 23:11:19 -0500 | [diff] [blame] | 2656 | /* |
| 2657 | * Use pi_lock to protect content of user_cpus_ptr |
| 2658 | * |
| 2659 | * Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent |
| 2660 | * do_set_cpus_allowed(). |
| 2661 | */ |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 2662 | raw_spin_lock_irqsave(&src->pi_lock, flags); |
Waiman Long | 87ca4f9 | 2022-12-30 23:11:19 -0500 | [diff] [blame] | 2663 | if (src->user_cpus_ptr) { |
| 2664 | swap(dst->user_cpus_ptr, user_mask); |
| 2665 | cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr); |
| 2666 | } |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 2667 | raw_spin_unlock_irqrestore(&src->pi_lock, flags); |
Waiman Long | 87ca4f9 | 2022-12-30 23:11:19 -0500 | [diff] [blame] | 2668 | |
| 2669 | if (unlikely(user_mask)) |
| 2670 | kfree(user_mask); |
| 2671 | |
Will Deacon | b90ca8b | 2021-07-30 12:24:33 +0100 | [diff] [blame] | 2672 | return 0; |
| 2673 | } |
| 2674 | |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 2675 | static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p) |
| 2676 | { |
| 2677 | struct cpumask *user_mask = NULL; |
| 2678 | |
| 2679 | swap(p->user_cpus_ptr, user_mask); |
| 2680 | |
| 2681 | return user_mask; |
| 2682 | } |
| 2683 | |
Will Deacon | b90ca8b | 2021-07-30 12:24:33 +0100 | [diff] [blame] | 2684 | void release_user_cpus_ptr(struct task_struct *p) |
| 2685 | { |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 2686 | kfree(clear_user_cpus_ptr(p)); |
Will Deacon | b90ca8b | 2021-07-30 12:24:33 +0100 | [diff] [blame] | 2687 | } |
| 2688 | |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2689 | /* |
Valentin Schneider | c777d84 | 2020-10-13 15:01:16 +0100 | [diff] [blame] | 2690 | * This function is wildly self concurrent; here be dragons. |
| 2691 | * |
| 2692 | * |
| 2693 | * When given a valid mask, __set_cpus_allowed_ptr() must block until the |
| 2694 | * designated task is enqueued on an allowed CPU. If that task is currently |
| 2695 | * running, we have to kick it out using the CPU stopper. |
| 2696 | * |
| 2697 | * Migrate-Disable comes along and tramples all over our nice sandcastle. |
| 2698 | * Consider: |
| 2699 | * |
| 2700 | * Initial conditions: P0->cpus_mask = [0, 1] |
| 2701 | * |
| 2702 | * P0@CPU0 P1 |
| 2703 | * |
| 2704 | * migrate_disable(); |
| 2705 | * <preempted> |
| 2706 | * set_cpus_allowed_ptr(P0, [1]); |
| 2707 | * |
| 2708 | * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes |
| 2709 | * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region). |
| 2710 | * This means we need the following scheme: |
| 2711 | * |
| 2712 | * P0@CPU0 P1 |
| 2713 | * |
| 2714 | * migrate_disable(); |
| 2715 | * <preempted> |
| 2716 | * set_cpus_allowed_ptr(P0, [1]); |
| 2717 | * <blocks> |
| 2718 | * <resumes> |
| 2719 | * migrate_enable(); |
| 2720 | * __set_cpus_allowed_ptr(); |
| 2721 | * <wakes local stopper> |
| 2722 | * `--> <woken on migration completion> |
| 2723 | * |
| 2724 | * Now the fun stuff: there may be several P1-like tasks, i.e. multiple |
| 2725 | * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any |
| 2726 | * task p are serialized by p->pi_lock, which we can leverage: the one that |
| 2727 | * should come into effect at the end of the Migrate-Disable region is the last |
| 2728 | * one. This means we only need to track a single cpumask (i.e. p->cpus_mask), |
| 2729 | * but we still need to properly signal those waiting tasks at the appropriate |
| 2730 | * moment. |
| 2731 | * |
| 2732 | * This is implemented using struct set_affinity_pending. The first |
| 2733 | * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will |
| 2734 | * setup an instance of that struct and install it on the targeted task_struct. |
| 2735 | * Any and all further callers will reuse that instance. Those then wait for |
| 2736 | * a completion signaled at the tail of the CPU stopper callback (1), triggered |
| 2737 | * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()). |
| 2738 | * |
| 2739 | * |
| 2740 | * (1) In the cases covered above. There is one more where the completion is |
| 2741 | * signaled within affine_move_task() itself: when a subsequent affinity request |
Valentin Schneider | e140749 | 2021-02-25 10:22:30 +0100 | [diff] [blame] | 2742 | * occurs after the stopper bailed out due to the targeted task still being |
| 2743 | * Migrate-Disable. Consider: |
Valentin Schneider | c777d84 | 2020-10-13 15:01:16 +0100 | [diff] [blame] | 2744 | * |
| 2745 | * Initial conditions: P0->cpus_mask = [0, 1] |
| 2746 | * |
Valentin Schneider | e140749 | 2021-02-25 10:22:30 +0100 | [diff] [blame] | 2747 | * CPU0 P1 P2 |
| 2748 | * <P0> |
| 2749 | * migrate_disable(); |
| 2750 | * <preempted> |
Valentin Schneider | c777d84 | 2020-10-13 15:01:16 +0100 | [diff] [blame] | 2751 | * set_cpus_allowed_ptr(P0, [1]); |
| 2752 | * <blocks> |
Valentin Schneider | e140749 | 2021-02-25 10:22:30 +0100 | [diff] [blame] | 2753 | * <migration/0> |
| 2754 | * migration_cpu_stop() |
| 2755 | * is_migration_disabled() |
| 2756 | * <bails> |
Valentin Schneider | c777d84 | 2020-10-13 15:01:16 +0100 | [diff] [blame] | 2757 | * set_cpus_allowed_ptr(P0, [0, 1]); |
| 2758 | * <signal completion> |
| 2759 | * <awakes> |
| 2760 | * |
| 2761 | * Note that the above is safe vs a concurrent migrate_enable(), as any |
| 2762 | * pending affinity completion is preceded by an uninstallation of |
| 2763 | * p->migration_pending done with p->pi_lock held. |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2764 | */ |
| 2765 | static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, |
| 2766 | int dest_cpu, unsigned int flags) |
Waiman Long | 5584e8a | 2022-09-22 14:00:37 -0400 | [diff] [blame] | 2767 | __releases(rq->lock) |
| 2768 | __releases(p->pi_lock) |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2769 | { |
| 2770 | struct set_affinity_pending my_pending = { }, *pending = NULL; |
Peter Zijlstra | 9e81889 | 2021-02-24 11:31:09 +0100 | [diff] [blame] | 2771 | bool stop_pending, complete = false; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2772 | |
| 2773 | /* Can the task run on the task's current CPU? If so, we're done */ |
| 2774 | if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { |
Peter Zijlstra | a7c8155 | 2020-09-28 17:06:07 +0200 | [diff] [blame] | 2775 | struct task_struct *push_task = NULL; |
| 2776 | |
| 2777 | if ((flags & SCA_MIGRATE_ENABLE) && |
| 2778 | (p->migration_flags & MDF_PUSH) && !rq->push_busy) { |
| 2779 | rq->push_busy = true; |
| 2780 | push_task = get_task_struct(p); |
| 2781 | } |
| 2782 | |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2783 | /* |
| 2784 | * If there are pending waiters, but no pending stop_work, |
| 2785 | * then complete now. |
| 2786 | */ |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2787 | pending = p->migration_pending; |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2788 | if (pending && !pending->stop_pending) { |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2789 | p->migration_pending = NULL; |
| 2790 | complete = true; |
| 2791 | } |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2792 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2793 | task_rq_unlock(rq, p, rf); |
| 2794 | |
Peter Zijlstra | a7c8155 | 2020-09-28 17:06:07 +0200 | [diff] [blame] | 2795 | if (push_task) { |
| 2796 | stop_one_cpu_nowait(rq->cpu, push_cpu_stop, |
| 2797 | p, &rq->push_work); |
| 2798 | } |
| 2799 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2800 | if (complete) |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2801 | complete_all(&pending->done); |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2802 | |
| 2803 | return 0; |
| 2804 | } |
| 2805 | |
| 2806 | if (!(flags & SCA_MIGRATE_ENABLE)) { |
| 2807 | /* serialized by p->pi_lock */ |
| 2808 | if (!p->migration_pending) { |
Valentin Schneider | c777d84 | 2020-10-13 15:01:16 +0100 | [diff] [blame] | 2809 | /* Install the request */ |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2810 | refcount_set(&my_pending.refs, 1); |
| 2811 | init_completion(&my_pending.done); |
Peter Zijlstra | 8a6edb5 | 2021-02-13 13:10:35 +0100 | [diff] [blame] | 2812 | my_pending.arg = (struct migration_arg) { |
| 2813 | .task = p, |
Valentin Schneider | 475ea6c | 2021-05-26 21:57:50 +0100 | [diff] [blame] | 2814 | .dest_cpu = dest_cpu, |
Peter Zijlstra | 8a6edb5 | 2021-02-13 13:10:35 +0100 | [diff] [blame] | 2815 | .pending = &my_pending, |
| 2816 | }; |
| 2817 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2818 | p->migration_pending = &my_pending; |
| 2819 | } else { |
| 2820 | pending = p->migration_pending; |
| 2821 | refcount_inc(&pending->refs); |
Valentin Schneider | 475ea6c | 2021-05-26 21:57:50 +0100 | [diff] [blame] | 2822 | /* |
| 2823 | * Affinity has changed, but we've already installed a |
| 2824 | * pending. migration_cpu_stop() *must* see this, else |
| 2825 | * we risk a completion of the pending despite having a |
| 2826 | * task on a disallowed CPU. |
| 2827 | * |
| 2828 | * Serialized by p->pi_lock, so this is safe. |
| 2829 | */ |
| 2830 | pending->arg.dest_cpu = dest_cpu; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2831 | } |
| 2832 | } |
| 2833 | pending = p->migration_pending; |
| 2834 | /* |
| 2835 | * - !MIGRATE_ENABLE: |
| 2836 | * we'll have installed a pending if there wasn't one already. |
| 2837 | * |
| 2838 | * - MIGRATE_ENABLE: |
| 2839 | * we're here because the current CPU isn't matching anymore, |
| 2840 | * the only way that can happen is because of a concurrent |
| 2841 | * set_cpus_allowed_ptr() call, which should then still be |
| 2842 | * pending completion. |
| 2843 | * |
| 2844 | * Either way, we really should have a @pending here. |
| 2845 | */ |
| 2846 | if (WARN_ON_ONCE(!pending)) { |
| 2847 | task_rq_unlock(rq, p, rf); |
| 2848 | return -EINVAL; |
| 2849 | } |
| 2850 | |
Peter Zijlstra | 0b9d46f | 2022-09-06 12:33:04 +0200 | [diff] [blame] | 2851 | if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { |
Valentin Schneider | c777d84 | 2020-10-13 15:01:16 +0100 | [diff] [blame] | 2852 | /* |
Peter Zijlstra | 58b1a45 | 2021-02-24 11:15:23 +0100 | [diff] [blame] | 2853 | * MIGRATE_ENABLE gets here because 'p == current', but for |
| 2854 | * anything else we cannot do is_migration_disabled(), punt |
| 2855 | * and have the stopper function handle it all race-free. |
Valentin Schneider | c777d84 | 2020-10-13 15:01:16 +0100 | [diff] [blame] | 2856 | */ |
Peter Zijlstra | 9e81889 | 2021-02-24 11:31:09 +0100 | [diff] [blame] | 2857 | stop_pending = pending->stop_pending; |
| 2858 | if (!stop_pending) |
| 2859 | pending->stop_pending = true; |
Peter Zijlstra | 58b1a45 | 2021-02-24 11:15:23 +0100 | [diff] [blame] | 2860 | |
Peter Zijlstra | 58b1a45 | 2021-02-24 11:15:23 +0100 | [diff] [blame] | 2861 | if (flags & SCA_MIGRATE_ENABLE) |
| 2862 | p->migration_flags &= ~MDF_PUSH; |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2863 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2864 | task_rq_unlock(rq, p, rf); |
Peter Zijlstra | 8a6edb5 | 2021-02-13 13:10:35 +0100 | [diff] [blame] | 2865 | |
Peter Zijlstra | 9e81889 | 2021-02-24 11:31:09 +0100 | [diff] [blame] | 2866 | if (!stop_pending) { |
| 2867 | stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, |
| 2868 | &pending->arg, &pending->stop_work); |
| 2869 | } |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2870 | |
Peter Zijlstra | 58b1a45 | 2021-02-24 11:15:23 +0100 | [diff] [blame] | 2871 | if (flags & SCA_MIGRATE_ENABLE) |
| 2872 | return 0; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2873 | } else { |
| 2874 | |
| 2875 | if (!is_migration_disabled(p)) { |
| 2876 | if (task_on_rq_queued(p)) |
| 2877 | rq = move_queued_task(rq, rf, p, dest_cpu); |
| 2878 | |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2879 | if (!pending->stop_pending) { |
| 2880 | p->migration_pending = NULL; |
| 2881 | complete = true; |
| 2882 | } |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2883 | } |
| 2884 | task_rq_unlock(rq, p, rf); |
| 2885 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2886 | if (complete) |
| 2887 | complete_all(&pending->done); |
| 2888 | } |
| 2889 | |
| 2890 | wait_for_completion(&pending->done); |
| 2891 | |
| 2892 | if (refcount_dec_and_test(&pending->refs)) |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2893 | wake_up_var(&pending->refs); /* No UaF, just an address */ |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2894 | |
Valentin Schneider | c777d84 | 2020-10-13 15:01:16 +0100 | [diff] [blame] | 2895 | /* |
| 2896 | * Block the original owner of &pending until all subsequent callers |
| 2897 | * have seen the completion and decremented the refcount |
| 2898 | */ |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2899 | wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); |
| 2900 | |
Peter Zijlstra | 50caf9c | 2021-02-24 11:42:08 +0100 | [diff] [blame] | 2901 | /* ARGH */ |
| 2902 | WARN_ON_ONCE(my_pending.stop_pending); |
| 2903 | |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 2904 | return 0; |
| 2905 | } |
| 2906 | |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2907 | /* |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 2908 | * Called with both p->pi_lock and rq->lock held; drops both before returning. |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2909 | */ |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 2910 | static int __set_cpus_allowed_ptr_locked(struct task_struct *p, |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2911 | struct affinity_context *ctx, |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 2912 | struct rq *rq, |
| 2913 | struct rq_flags *rf) |
| 2914 | __releases(rq->lock) |
| 2915 | __releases(p->pi_lock) |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2916 | { |
Will Deacon | 234a503 | 2021-07-30 12:24:32 +0100 | [diff] [blame] | 2917 | const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p); |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 2918 | const struct cpumask *cpu_valid_mask = cpu_active_mask; |
Will Deacon | 234a503 | 2021-07-30 12:24:32 +0100 | [diff] [blame] | 2919 | bool kthread = p->flags & PF_KTHREAD; |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2920 | unsigned int dest_cpu; |
| 2921 | int ret = 0; |
| 2922 | |
Wanpeng Li | a499c3e | 2017-02-21 23:52:55 -0800 | [diff] [blame] | 2923 | update_rq_clock(rq); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2924 | |
Will Deacon | 234a503 | 2021-07-30 12:24:32 +0100 | [diff] [blame] | 2925 | if (kthread || is_migration_disabled(p)) { |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 2926 | /* |
Peter Zijlstra | 741ba80 | 2021-01-16 11:56:37 +0100 | [diff] [blame] | 2927 | * Kernel threads are allowed on online && !active CPUs, |
| 2928 | * however, during cpu-hot-unplug, even these might get pushed |
| 2929 | * away if not KTHREAD_IS_PER_CPU. |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 2930 | * |
| 2931 | * Specifically, migration_disabled() tasks must not fail the |
| 2932 | * cpumask_any_and_distribute() pick below, esp. so on |
| 2933 | * SCA_MIGRATE_ENABLE, otherwise we'll not call |
| 2934 | * set_cpus_allowed_common() and actually reset p->cpus_ptr. |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 2935 | */ |
| 2936 | cpu_valid_mask = cpu_online_mask; |
| 2937 | } |
| 2938 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2939 | if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) { |
Will Deacon | 234a503 | 2021-07-30 12:24:32 +0100 | [diff] [blame] | 2940 | ret = -EINVAL; |
| 2941 | goto out; |
| 2942 | } |
| 2943 | |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 2944 | /* |
| 2945 | * Must re-check here, to close a race against __kthread_bind(), |
| 2946 | * sched_setaffinity() is not guaranteed to observe the flag. |
| 2947 | */ |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2948 | if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 2949 | ret = -EINVAL; |
| 2950 | goto out; |
| 2951 | } |
| 2952 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2953 | if (!(ctx->flags & SCA_MIGRATE_ENABLE)) { |
| 2954 | if (cpumask_equal(&p->cpus_mask, ctx->new_mask)) |
Valentin Schneider | 885b3ba | 2020-10-13 15:01:15 +0100 | [diff] [blame] | 2955 | goto out; |
| 2956 | |
| 2957 | if (WARN_ON_ONCE(p == current && |
| 2958 | is_migration_disabled(p) && |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2959 | !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) { |
Valentin Schneider | 885b3ba | 2020-10-13 15:01:15 +0100 | [diff] [blame] | 2960 | ret = -EBUSY; |
| 2961 | goto out; |
| 2962 | } |
| 2963 | } |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2964 | |
Paul Turner | 46a87b3 | 2020-03-10 18:01:13 -0700 | [diff] [blame] | 2965 | /* |
| 2966 | * Picking a ~random cpu helps in cases where we are changing affinity |
| 2967 | * for groups of tasks (ie. cpuset), so that load balancing is not |
| 2968 | * immediately required to distribute the tasks within their new mask. |
| 2969 | */ |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2970 | dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask); |
KeMeng Shi | 714e501 | 2019-09-16 06:53:28 +0000 | [diff] [blame] | 2971 | if (dest_cpu >= nr_cpu_ids) { |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2972 | ret = -EINVAL; |
| 2973 | goto out; |
| 2974 | } |
| 2975 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2976 | __do_set_cpus_allowed(p, ctx); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 2977 | |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 2978 | return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 2979 | |
| 2980 | out: |
| 2981 | task_rq_unlock(rq, p, rf); |
| 2982 | |
| 2983 | return ret; |
| 2984 | } |
| 2985 | |
| 2986 | /* |
| 2987 | * Change a given task's CPU affinity. Migrate the thread to a |
| 2988 | * proper CPU and schedule it away if the CPU it's executing on |
| 2989 | * is removed from the allowed bitmask. |
| 2990 | * |
| 2991 | * NOTE: the caller must have a valid reference to the task, the |
| 2992 | * task must not exit() & deallocate itself prematurely. The |
| 2993 | * call is not atomic; no spinlocks may be held. |
| 2994 | */ |
| 2995 | static int __set_cpus_allowed_ptr(struct task_struct *p, |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 2996 | struct affinity_context *ctx) |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 2997 | { |
| 2998 | struct rq_flags rf; |
| 2999 | struct rq *rq; |
| 3000 | |
| 3001 | rq = task_rq_lock(p, &rf); |
Waiman Long | da01903 | 2022-09-22 14:00:39 -0400 | [diff] [blame] | 3002 | /* |
| 3003 | * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_* |
| 3004 | * flags are set. |
| 3005 | */ |
| 3006 | if (p->user_cpus_ptr && |
| 3007 | !(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) && |
| 3008 | cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) |
| 3009 | ctx->new_mask = rq->scratch_mask; |
| 3010 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 3011 | return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf); |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 3012 | } |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 3013 | |
| 3014 | int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) |
| 3015 | { |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 3016 | struct affinity_context ac = { |
| 3017 | .new_mask = new_mask, |
| 3018 | .flags = 0, |
| 3019 | }; |
| 3020 | |
| 3021 | return __set_cpus_allowed_ptr(p, &ac); |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 3022 | } |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 3023 | EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); |
| 3024 | |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3025 | /* |
| 3026 | * Change a given task's CPU affinity to the intersection of its current |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 3027 | * affinity mask and @subset_mask, writing the resulting mask to @new_mask. |
| 3028 | * If user_cpus_ptr is defined, use it as the basis for restricting CPU |
| 3029 | * affinity or use cpu_online_mask instead. |
| 3030 | * |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3031 | * If the resulting mask is empty, leave the affinity unchanged and return |
| 3032 | * -EINVAL. |
| 3033 | */ |
| 3034 | static int restrict_cpus_allowed_ptr(struct task_struct *p, |
| 3035 | struct cpumask *new_mask, |
| 3036 | const struct cpumask *subset_mask) |
| 3037 | { |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 3038 | struct affinity_context ac = { |
| 3039 | .new_mask = new_mask, |
| 3040 | .flags = 0, |
| 3041 | }; |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3042 | struct rq_flags rf; |
| 3043 | struct rq *rq; |
| 3044 | int err; |
| 3045 | |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3046 | rq = task_rq_lock(p, &rf); |
| 3047 | |
| 3048 | /* |
| 3049 | * Forcefully restricting the affinity of a deadline task is |
| 3050 | * likely to cause problems, so fail and noisily override the |
| 3051 | * mask entirely. |
| 3052 | */ |
| 3053 | if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { |
| 3054 | err = -EPERM; |
| 3055 | goto err_unlock; |
| 3056 | } |
| 3057 | |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 3058 | if (!cpumask_and(new_mask, task_user_cpus(p), subset_mask)) { |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3059 | err = -EINVAL; |
| 3060 | goto err_unlock; |
| 3061 | } |
| 3062 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 3063 | return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf); |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3064 | |
| 3065 | err_unlock: |
| 3066 | task_rq_unlock(rq, p, &rf); |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3067 | return err; |
| 3068 | } |
| 3069 | |
| 3070 | /* |
| 3071 | * Restrict the CPU affinity of task @p so that it is a subset of |
Waiman Long | 5584e8a | 2022-09-22 14:00:37 -0400 | [diff] [blame] | 3072 | * task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3073 | * old affinity mask. If the resulting mask is empty, we warn and walk |
| 3074 | * up the cpuset hierarchy until we find a suitable mask. |
| 3075 | */ |
| 3076 | void force_compatible_cpus_allowed_ptr(struct task_struct *p) |
| 3077 | { |
| 3078 | cpumask_var_t new_mask; |
| 3079 | const struct cpumask *override_mask = task_cpu_possible_mask(p); |
| 3080 | |
| 3081 | alloc_cpumask_var(&new_mask, GFP_KERNEL); |
| 3082 | |
| 3083 | /* |
| 3084 | * __migrate_task() can fail silently in the face of concurrent |
| 3085 | * offlining of the chosen destination CPU, so take the hotplug |
| 3086 | * lock to ensure that the migration succeeds. |
| 3087 | */ |
| 3088 | cpus_read_lock(); |
| 3089 | if (!cpumask_available(new_mask)) |
| 3090 | goto out_set_mask; |
| 3091 | |
| 3092 | if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask)) |
| 3093 | goto out_free_mask; |
| 3094 | |
| 3095 | /* |
| 3096 | * We failed to find a valid subset of the affinity mask for the |
| 3097 | * task, so override it based on its cpuset hierarchy. |
| 3098 | */ |
| 3099 | cpuset_cpus_allowed(p, new_mask); |
| 3100 | override_mask = new_mask; |
| 3101 | |
| 3102 | out_set_mask: |
| 3103 | if (printk_ratelimit()) { |
| 3104 | printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n", |
| 3105 | task_pid_nr(p), p->comm, |
| 3106 | cpumask_pr_args(override_mask)); |
| 3107 | } |
| 3108 | |
| 3109 | WARN_ON(set_cpus_allowed_ptr(p, override_mask)); |
| 3110 | out_free_mask: |
| 3111 | cpus_read_unlock(); |
| 3112 | free_cpumask_var(new_mask); |
| 3113 | } |
| 3114 | |
| 3115 | static int |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 3116 | __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx); |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3117 | |
| 3118 | /* |
| 3119 | * Restore the affinity of a task @p which was previously restricted by a |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 3120 | * call to force_compatible_cpus_allowed_ptr(). |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3121 | * |
| 3122 | * It is the caller's responsibility to serialise this with any calls to |
| 3123 | * force_compatible_cpus_allowed_ptr(@p). |
| 3124 | */ |
| 3125 | void relax_compatible_cpus_allowed_ptr(struct task_struct *p) |
| 3126 | { |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 3127 | struct affinity_context ac = { |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 3128 | .new_mask = task_user_cpus(p), |
| 3129 | .flags = 0, |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 3130 | }; |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 3131 | int ret; |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3132 | |
| 3133 | /* |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 3134 | * Try to restore the old affinity mask with __sched_setaffinity(). |
| 3135 | * Cpuset masking will be done there too. |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3136 | */ |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 3137 | ret = __sched_setaffinity(p, &ac); |
| 3138 | WARN_ON_ONCE(ret); |
Will Deacon | 07ec77a | 2021-07-30 12:24:35 +0100 | [diff] [blame] | 3139 | } |
| 3140 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 3141 | void set_task_cpu(struct task_struct *p, unsigned int new_cpu) |
Ingo Molnar | c65cc87 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3142 | { |
Peter Zijlstra | e291200 | 2009-12-16 18:04:36 +0100 | [diff] [blame] | 3143 | #ifdef CONFIG_SCHED_DEBUG |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 3144 | unsigned int state = READ_ONCE(p->__state); |
| 3145 | |
Peter Zijlstra | e291200 | 2009-12-16 18:04:36 +0100 | [diff] [blame] | 3146 | /* |
| 3147 | * We should never call set_task_cpu() on a blocked task, |
| 3148 | * ttwu() will sort out the placement. |
| 3149 | */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 3150 | WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq); |
Peter Zijlstra | 0122ec5 | 2011-04-05 17:23:51 +0200 | [diff] [blame] | 3151 | |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 3152 | /* |
| 3153 | * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, |
| 3154 | * because schedstat_wait_{start,end} rebase migrating task's wait_start |
| 3155 | * time relying on p->on_rq. |
| 3156 | */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 3157 | WARN_ON_ONCE(state == TASK_RUNNING && |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 3158 | p->sched_class == &fair_sched_class && |
| 3159 | (p->on_rq && !task_on_rq_migrating(p))); |
| 3160 | |
Peter Zijlstra | 0122ec5 | 2011-04-05 17:23:51 +0200 | [diff] [blame] | 3161 | #ifdef CONFIG_LOCKDEP |
Peter Zijlstra | 6c6c54e | 2011-06-03 17:37:07 +0200 | [diff] [blame] | 3162 | /* |
| 3163 | * The caller should hold either p->pi_lock or rq->lock, when changing |
| 3164 | * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. |
| 3165 | * |
| 3166 | * sched_move_task() holds both and thus holding either pins the cgroup, |
Peter Zijlstra | 8323f26 | 2012-06-22 13:36:05 +0200 | [diff] [blame] | 3167 | * see task_group(). |
Peter Zijlstra | 6c6c54e | 2011-06-03 17:37:07 +0200 | [diff] [blame] | 3168 | * |
| 3169 | * Furthermore, all task_rq users should acquire both locks, see |
| 3170 | * task_rq_lock(). |
| 3171 | */ |
Peter Zijlstra | 0122ec5 | 2011-04-05 17:23:51 +0200 | [diff] [blame] | 3172 | WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || |
Peter Zijlstra | 9ef7e7e | 2021-03-03 16:45:41 +0100 | [diff] [blame] | 3173 | lockdep_is_held(__rq_lockp(task_rq(p))))); |
Peter Zijlstra | 0122ec5 | 2011-04-05 17:23:51 +0200 | [diff] [blame] | 3174 | #endif |
Peter Zijlstra | 4ff9083 | 2017-09-07 17:03:52 +0200 | [diff] [blame] | 3175 | /* |
| 3176 | * Clearly, migrating tasks to offline CPUs is a fairly daft thing. |
| 3177 | */ |
| 3178 | WARN_ON_ONCE(!cpu_online(new_cpu)); |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 3179 | |
| 3180 | WARN_ON_ONCE(is_migration_disabled(p)); |
Peter Zijlstra | e291200 | 2009-12-16 18:04:36 +0100 | [diff] [blame] | 3181 | #endif |
| 3182 | |
Mathieu Desnoyers | de1d728 | 2009-05-05 16:49:59 +0800 | [diff] [blame] | 3183 | trace_sched_migrate_task(p, new_cpu); |
Peter Zijlstra | cbc34ed | 2008-12-10 08:08:22 +0100 | [diff] [blame] | 3184 | |
Peter Zijlstra | 0c69774 | 2009-12-22 15:43:19 +0100 | [diff] [blame] | 3185 | if (task_cpu(p) != new_cpu) { |
Paul Turner | 0a74bef | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3186 | if (p->sched_class->migrate_task_rq) |
Srikar Dronamraju | 1327237a5 | 2018-09-21 23:18:57 +0530 | [diff] [blame] | 3187 | p->sched_class->migrate_task_rq(p, new_cpu); |
Peter Zijlstra | 0c69774 | 2009-12-22 15:43:19 +0100 | [diff] [blame] | 3188 | p->se.nr_migrations++; |
Mathieu Desnoyers | d7822b1 | 2018-06-02 08:43:54 -0400 | [diff] [blame] | 3189 | rseq_migrate(p); |
Peter Zijlstra | ff303e6 | 2015-04-17 20:05:30 +0200 | [diff] [blame] | 3190 | perf_event_task_migrate(p); |
Peter Zijlstra | 0c69774 | 2009-12-22 15:43:19 +0100 | [diff] [blame] | 3191 | } |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 3192 | |
| 3193 | __set_task_cpu(p, new_cpu); |
Ingo Molnar | c65cc87 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3194 | } |
| 3195 | |
Srikar Dronamraju | 0ad4e3d | 2018-06-20 22:32:50 +0530 | [diff] [blame] | 3196 | #ifdef CONFIG_NUMA_BALANCING |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3197 | static void __migrate_swap_task(struct task_struct *p, int cpu) |
| 3198 | { |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 3199 | if (task_on_rq_queued(p)) { |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3200 | struct rq *src_rq, *dst_rq; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 3201 | struct rq_flags srf, drf; |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3202 | |
| 3203 | src_rq = task_rq(p); |
| 3204 | dst_rq = cpu_rq(cpu); |
| 3205 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 3206 | rq_pin_lock(src_rq, &srf); |
| 3207 | rq_pin_lock(dst_rq, &drf); |
| 3208 | |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3209 | deactivate_task(src_rq, p, 0); |
| 3210 | set_task_cpu(p, cpu); |
| 3211 | activate_task(dst_rq, p, 0); |
| 3212 | check_preempt_curr(dst_rq, p, 0); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 3213 | |
| 3214 | rq_unpin_lock(dst_rq, &drf); |
| 3215 | rq_unpin_lock(src_rq, &srf); |
| 3216 | |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3217 | } else { |
| 3218 | /* |
| 3219 | * Task isn't running anymore; make it appear like we migrated |
| 3220 | * it before it went to sleep. This means on wakeup we make the |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 3221 | * previous CPU our target instead of where it really is. |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3222 | */ |
| 3223 | p->wake_cpu = cpu; |
| 3224 | } |
| 3225 | } |
| 3226 | |
| 3227 | struct migration_swap_arg { |
| 3228 | struct task_struct *src_task, *dst_task; |
| 3229 | int src_cpu, dst_cpu; |
| 3230 | }; |
| 3231 | |
| 3232 | static int migrate_swap_stop(void *data) |
| 3233 | { |
| 3234 | struct migration_swap_arg *arg = data; |
| 3235 | struct rq *src_rq, *dst_rq; |
| 3236 | int ret = -EAGAIN; |
| 3237 | |
Peter Zijlstra | 62694cd | 2015-10-09 18:36:29 +0200 | [diff] [blame] | 3238 | if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) |
| 3239 | return -EAGAIN; |
| 3240 | |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3241 | src_rq = cpu_rq(arg->src_cpu); |
| 3242 | dst_rq = cpu_rq(arg->dst_cpu); |
| 3243 | |
Peter Zijlstra | 7460231 | 2013-10-10 20:17:22 +0200 | [diff] [blame] | 3244 | double_raw_lock(&arg->src_task->pi_lock, |
| 3245 | &arg->dst_task->pi_lock); |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3246 | double_rq_lock(src_rq, dst_rq); |
Peter Zijlstra | 62694cd | 2015-10-09 18:36:29 +0200 | [diff] [blame] | 3247 | |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3248 | if (task_cpu(arg->dst_task) != arg->dst_cpu) |
| 3249 | goto unlock; |
| 3250 | |
| 3251 | if (task_cpu(arg->src_task) != arg->src_cpu) |
| 3252 | goto unlock; |
| 3253 | |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3254 | if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3255 | goto unlock; |
| 3256 | |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3257 | if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3258 | goto unlock; |
| 3259 | |
| 3260 | __migrate_swap_task(arg->src_task, arg->dst_cpu); |
| 3261 | __migrate_swap_task(arg->dst_task, arg->src_cpu); |
| 3262 | |
| 3263 | ret = 0; |
| 3264 | |
| 3265 | unlock: |
| 3266 | double_rq_unlock(src_rq, dst_rq); |
Peter Zijlstra | 7460231 | 2013-10-10 20:17:22 +0200 | [diff] [blame] | 3267 | raw_spin_unlock(&arg->dst_task->pi_lock); |
| 3268 | raw_spin_unlock(&arg->src_task->pi_lock); |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3269 | |
| 3270 | return ret; |
| 3271 | } |
| 3272 | |
| 3273 | /* |
| 3274 | * Cross migrate two tasks |
| 3275 | */ |
Srikar Dronamraju | 0ad4e3d | 2018-06-20 22:32:50 +0530 | [diff] [blame] | 3276 | int migrate_swap(struct task_struct *cur, struct task_struct *p, |
| 3277 | int target_cpu, int curr_cpu) |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3278 | { |
| 3279 | struct migration_swap_arg arg; |
| 3280 | int ret = -EINVAL; |
| 3281 | |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3282 | arg = (struct migration_swap_arg){ |
| 3283 | .src_task = cur, |
Srikar Dronamraju | 0ad4e3d | 2018-06-20 22:32:50 +0530 | [diff] [blame] | 3284 | .src_cpu = curr_cpu, |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3285 | .dst_task = p, |
Srikar Dronamraju | 0ad4e3d | 2018-06-20 22:32:50 +0530 | [diff] [blame] | 3286 | .dst_cpu = target_cpu, |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3287 | }; |
| 3288 | |
| 3289 | if (arg.src_cpu == arg.dst_cpu) |
| 3290 | goto out; |
| 3291 | |
Peter Zijlstra | 6acce3e | 2013-10-11 14:38:20 +0200 | [diff] [blame] | 3292 | /* |
| 3293 | * These three tests are all lockless; this is OK since all of them |
| 3294 | * will be re-checked with proper locks held further down the line. |
| 3295 | */ |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3296 | if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) |
| 3297 | goto out; |
| 3298 | |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3299 | if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3300 | goto out; |
| 3301 | |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3302 | if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3303 | goto out; |
| 3304 | |
Mel Gorman | 286549d | 2014-01-21 15:51:03 -0800 | [diff] [blame] | 3305 | trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3306 | ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); |
| 3307 | |
| 3308 | out: |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3309 | return ret; |
| 3310 | } |
Srikar Dronamraju | 0ad4e3d | 2018-06-20 22:32:50 +0530 | [diff] [blame] | 3311 | #endif /* CONFIG_NUMA_BALANCING */ |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 3312 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3313 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3314 | * wait_task_inactive - wait for a thread to unschedule. |
| 3315 | * |
Peter Zijlstra | f9fc8ca | 2022-09-06 12:39:55 +0200 | [diff] [blame] | 3316 | * Wait for the thread to block in any of the states set in @match_state. |
| 3317 | * If it changes, i.e. @p might have woken up, then return zero. When we |
| 3318 | * succeed in waiting for @p to be off its CPU, we return a positive number |
| 3319 | * (its total switch count). If a second call a short while later returns the |
| 3320 | * same number, the caller can be sure that @p has remained unscheduled the |
| 3321 | * whole time. |
Roland McGrath | 85ba2d8 | 2008-07-25 19:45:58 -0700 | [diff] [blame] | 3322 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3323 | * The caller must ensure that the task *will* unschedule sometime soon, |
| 3324 | * else this function might spin for a *long* time. This function can't |
| 3325 | * be called with interrupts off, or it may introduce deadlock with |
| 3326 | * smp_call_function() if an IPI is sent by the same process we are |
| 3327 | * waiting to become inactive. |
| 3328 | */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 3329 | unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3330 | { |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 3331 | int running, queued; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 3332 | struct rq_flags rf; |
Roland McGrath | 85ba2d8 | 2008-07-25 19:45:58 -0700 | [diff] [blame] | 3333 | unsigned long ncsw; |
Ingo Molnar | 70b97a7 | 2006-07-03 00:25:42 -0700 | [diff] [blame] | 3334 | struct rq *rq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3335 | |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3336 | for (;;) { |
| 3337 | /* |
| 3338 | * We do the initial early heuristics without holding |
| 3339 | * any task-queue locks at all. We'll only try to get |
| 3340 | * the runqueue lock when things look like they will |
| 3341 | * work out! |
| 3342 | */ |
| 3343 | rq = task_rq(p); |
Linus Torvalds | fa490cf | 2007-06-18 09:34:40 -0700 | [diff] [blame] | 3344 | |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3345 | /* |
| 3346 | * If the task is actively running on another CPU |
| 3347 | * still, just relax and busy-wait without holding |
| 3348 | * any locks. |
| 3349 | * |
| 3350 | * NOTE! Since we don't hold any locks, it's not |
| 3351 | * even sure that "rq" stays as the right runqueue! |
Peter Zijlstra | 0b9d46f | 2022-09-06 12:33:04 +0200 | [diff] [blame] | 3352 | * But we don't care, since "task_on_cpu()" will |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3353 | * return false if the runqueue has changed and p |
| 3354 | * is actually now running somewhere else! |
| 3355 | */ |
Peter Zijlstra | 0b9d46f | 2022-09-06 12:33:04 +0200 | [diff] [blame] | 3356 | while (task_on_cpu(rq, p)) { |
Peter Zijlstra | f9fc8ca | 2022-09-06 12:39:55 +0200 | [diff] [blame] | 3357 | if (!(READ_ONCE(p->__state) & match_state)) |
Roland McGrath | 85ba2d8 | 2008-07-25 19:45:58 -0700 | [diff] [blame] | 3358 | return 0; |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3359 | cpu_relax(); |
Roland McGrath | 85ba2d8 | 2008-07-25 19:45:58 -0700 | [diff] [blame] | 3360 | } |
Linus Torvalds | fa490cf | 2007-06-18 09:34:40 -0700 | [diff] [blame] | 3361 | |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3362 | /* |
| 3363 | * Ok, time to look more closely! We need the rq |
| 3364 | * lock now, to be *sure*. If we're wrong, we'll |
| 3365 | * just go back and repeat. |
| 3366 | */ |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 3367 | rq = task_rq_lock(p, &rf); |
Peter Zijlstra | 27a9da6 | 2010-05-04 20:36:56 +0200 | [diff] [blame] | 3368 | trace_sched_wait_task(p); |
Peter Zijlstra | 0b9d46f | 2022-09-06 12:33:04 +0200 | [diff] [blame] | 3369 | running = task_on_cpu(rq, p); |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 3370 | queued = task_on_rq_queued(p); |
Roland McGrath | 85ba2d8 | 2008-07-25 19:45:58 -0700 | [diff] [blame] | 3371 | ncsw = 0; |
Peter Zijlstra | f9fc8ca | 2022-09-06 12:39:55 +0200 | [diff] [blame] | 3372 | if (READ_ONCE(p->__state) & match_state) |
Oleg Nesterov | 93dcf55 | 2008-08-20 16:54:44 -0700 | [diff] [blame] | 3373 | ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 3374 | task_rq_unlock(rq, p, &rf); |
Linus Torvalds | fa490cf | 2007-06-18 09:34:40 -0700 | [diff] [blame] | 3375 | |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3376 | /* |
Roland McGrath | 85ba2d8 | 2008-07-25 19:45:58 -0700 | [diff] [blame] | 3377 | * If it changed from the expected state, bail out now. |
| 3378 | */ |
| 3379 | if (unlikely(!ncsw)) |
| 3380 | break; |
| 3381 | |
| 3382 | /* |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3383 | * Was it really running after all now that we |
| 3384 | * checked with the proper locks actually held? |
| 3385 | * |
| 3386 | * Oops. Go back and try again.. |
| 3387 | */ |
| 3388 | if (unlikely(running)) { |
| 3389 | cpu_relax(); |
| 3390 | continue; |
| 3391 | } |
| 3392 | |
| 3393 | /* |
| 3394 | * It's not enough that it's not actively running, |
| 3395 | * it must be off the runqueue _entirely_, and not |
| 3396 | * preempted! |
| 3397 | * |
Luis Henriques | 80dd99b | 2009-03-16 19:58:09 +0000 | [diff] [blame] | 3398 | * So if it was still runnable (but just not actively |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3399 | * running right now), it's preempted, and we should |
| 3400 | * yield - it could be a while. |
| 3401 | */ |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 3402 | if (unlikely(queued)) { |
Thomas Gleixner | 8b0e195 | 2016-12-25 12:30:41 +0100 | [diff] [blame] | 3403 | ktime_t to = NSEC_PER_SEC / HZ; |
Thomas Gleixner | 8eb90c3 | 2011-02-23 23:52:21 +0000 | [diff] [blame] | 3404 | |
| 3405 | set_current_state(TASK_UNINTERRUPTIBLE); |
Sebastian Andrzej Siewior | c33627e | 2021-08-26 19:04:08 +0200 | [diff] [blame] | 3406 | schedule_hrtimeout(&to, HRTIMER_MODE_REL_HARD); |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 3407 | continue; |
| 3408 | } |
| 3409 | |
| 3410 | /* |
| 3411 | * Ahh, all good. It wasn't running, and it wasn't |
| 3412 | * runnable, which means that it will never become |
| 3413 | * running in the future either. We're all done! |
| 3414 | */ |
| 3415 | break; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3416 | } |
Roland McGrath | 85ba2d8 | 2008-07-25 19:45:58 -0700 | [diff] [blame] | 3417 | |
| 3418 | return ncsw; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3419 | } |
| 3420 | |
| 3421 | /*** |
| 3422 | * kick_process - kick a running thread to enter/exit the kernel |
| 3423 | * @p: the to-be-kicked thread |
| 3424 | * |
| 3425 | * Cause a process which is running on another CPU to enter |
| 3426 | * kernel-mode, without any delay. (to get signals handled.) |
| 3427 | * |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 3428 | * NOTE: this function doesn't have to take the runqueue lock, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3429 | * because all it wants to ensure is that the remote task enters |
| 3430 | * the kernel. If the IPI races and the task has been migrated |
| 3431 | * to another CPU then no harm is done and the purpose has been |
| 3432 | * achieved as well. |
| 3433 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 3434 | void kick_process(struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3435 | { |
| 3436 | int cpu; |
| 3437 | |
| 3438 | preempt_disable(); |
| 3439 | cpu = task_cpu(p); |
| 3440 | if ((cpu != smp_processor_id()) && task_curr(p)) |
| 3441 | smp_send_reschedule(cpu); |
| 3442 | preempt_enable(); |
| 3443 | } |
Rusty Russell | b43e352 | 2009-06-12 22:27:00 -0600 | [diff] [blame] | 3444 | EXPORT_SYMBOL_GPL(kick_process); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3445 | |
Oleg Nesterov | 30da688 | 2010-03-15 10:10:19 +0100 | [diff] [blame] | 3446 | /* |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3447 | * ->cpus_ptr is protected by both rq->lock and p->pi_lock |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 3448 | * |
| 3449 | * A few notes on cpu_active vs cpu_online: |
| 3450 | * |
| 3451 | * - cpu_active must be a subset of cpu_online |
| 3452 | * |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 3453 | * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 3454 | * see __set_cpus_allowed_ptr(). At this point the newly online |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 3455 | * CPU isn't yet part of the sched domains, and balancing will not |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 3456 | * see it. |
| 3457 | * |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 3458 | * - on CPU-down we clear cpu_active() to mask the sched domains and |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 3459 | * avoid the load balancer to place new tasks on the to be removed |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 3460 | * CPU. Existing tasks will remain running there and will be taken |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 3461 | * off. |
| 3462 | * |
| 3463 | * This means that fallback selection must not select !active CPUs. |
| 3464 | * And can assume that any active CPU must be online. Conversely |
| 3465 | * select_task_rq() below may allow selection of !active CPUs in order |
| 3466 | * to satisfy the above rules. |
Oleg Nesterov | 30da688 | 2010-03-15 10:10:19 +0100 | [diff] [blame] | 3467 | */ |
Peter Zijlstra | 5da9a0f | 2009-12-16 18:04:38 +0100 | [diff] [blame] | 3468 | static int select_fallback_rq(int cpu, struct task_struct *p) |
| 3469 | { |
Tang Chen | aa00d89 | 2013-02-22 16:33:33 -0800 | [diff] [blame] | 3470 | int nid = cpu_to_node(cpu); |
| 3471 | const struct cpumask *nodemask = NULL; |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3472 | enum { cpuset, possible, fail } state = cpuset; |
| 3473 | int dest_cpu; |
Peter Zijlstra | 5da9a0f | 2009-12-16 18:04:38 +0100 | [diff] [blame] | 3474 | |
Tang Chen | aa00d89 | 2013-02-22 16:33:33 -0800 | [diff] [blame] | 3475 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 3476 | * If the node that the CPU is on has been offlined, cpu_to_node() |
| 3477 | * will return -1. There is no CPU on the node, and we should |
| 3478 | * select the CPU on the other node. |
Tang Chen | aa00d89 | 2013-02-22 16:33:33 -0800 | [diff] [blame] | 3479 | */ |
| 3480 | if (nid != -1) { |
| 3481 | nodemask = cpumask_of_node(nid); |
| 3482 | |
| 3483 | /* Look for allowed, online CPU in same node. */ |
| 3484 | for_each_cpu(dest_cpu, nodemask) { |
Will Deacon | 9ae606b | 2021-07-30 12:24:28 +0100 | [diff] [blame] | 3485 | if (is_cpu_allowed(p, dest_cpu)) |
Tang Chen | aa00d89 | 2013-02-22 16:33:33 -0800 | [diff] [blame] | 3486 | return dest_cpu; |
| 3487 | } |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3488 | } |
Peter Zijlstra | 5da9a0f | 2009-12-16 18:04:38 +0100 | [diff] [blame] | 3489 | |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3490 | for (;;) { |
| 3491 | /* Any allowed, online CPU? */ |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3492 | for_each_cpu(dest_cpu, p->cpus_ptr) { |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 3493 | if (!is_cpu_allowed(p, dest_cpu)) |
Tejun Heo | feb245e | 2016-06-16 15:35:04 -0400 | [diff] [blame] | 3494 | continue; |
Peter Zijlstra | 175f0e2 | 2017-07-25 18:58:21 +0200 | [diff] [blame] | 3495 | |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3496 | goto out; |
| 3497 | } |
Peter Zijlstra | 5da9a0f | 2009-12-16 18:04:38 +0100 | [diff] [blame] | 3498 | |
Oleg Nesterov | e73e85f | 2015-10-10 20:53:15 +0200 | [diff] [blame] | 3499 | /* No more Mr. Nice Guy. */ |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3500 | switch (state) { |
| 3501 | case cpuset: |
Will Deacon | 97c0054 | 2021-07-30 12:24:31 +0100 | [diff] [blame] | 3502 | if (cpuset_cpus_allowed_fallback(p)) { |
Oleg Nesterov | e73e85f | 2015-10-10 20:53:15 +0200 | [diff] [blame] | 3503 | state = possible; |
| 3504 | break; |
| 3505 | } |
Gustavo A. R. Silva | df561f66 | 2020-08-23 17:36:59 -0500 | [diff] [blame] | 3506 | fallthrough; |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3507 | case possible: |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 3508 | /* |
| 3509 | * XXX When called from select_task_rq() we only |
| 3510 | * hold p->pi_lock and again violate locking order. |
| 3511 | * |
| 3512 | * More yuck to audit. |
| 3513 | */ |
Will Deacon | 9ae606b | 2021-07-30 12:24:28 +0100 | [diff] [blame] | 3514 | do_set_cpus_allowed(p, task_cpu_possible_mask(p)); |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3515 | state = fail; |
| 3516 | break; |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3517 | case fail: |
| 3518 | BUG(); |
| 3519 | break; |
| 3520 | } |
| 3521 | } |
| 3522 | |
| 3523 | out: |
| 3524 | if (state != cpuset) { |
| 3525 | /* |
| 3526 | * Don't tell them about moving exiting tasks or |
| 3527 | * kernel threads (both mm NULL), since they never |
| 3528 | * leave kernel. |
| 3529 | */ |
| 3530 | if (p->mm && printk_ratelimit()) { |
John Stultz | aac74dc | 2014-06-04 16:11:40 -0700 | [diff] [blame] | 3531 | printk_deferred("process %d (%s) no longer affine to cpu%d\n", |
Peter Zijlstra | 2baab4e | 2012-03-20 15:57:01 +0100 | [diff] [blame] | 3532 | task_pid_nr(p), p->comm, cpu); |
| 3533 | } |
Peter Zijlstra | 5da9a0f | 2009-12-16 18:04:38 +0100 | [diff] [blame] | 3534 | } |
| 3535 | |
| 3536 | return dest_cpu; |
| 3537 | } |
| 3538 | |
Peter Zijlstra | e291200 | 2009-12-16 18:04:36 +0100 | [diff] [blame] | 3539 | /* |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3540 | * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. |
Peter Zijlstra | e291200 | 2009-12-16 18:04:36 +0100 | [diff] [blame] | 3541 | */ |
Peter Zijlstra | 970b13b | 2009-11-25 13:31:39 +0100 | [diff] [blame] | 3542 | static inline |
Valentin Schneider | 3aef1551 | 2020-11-02 18:45:13 +0000 | [diff] [blame] | 3543 | int select_task_rq(struct task_struct *p, int cpu, int wake_flags) |
Peter Zijlstra | 970b13b | 2009-11-25 13:31:39 +0100 | [diff] [blame] | 3544 | { |
Peter Zijlstra | cbce1a6 | 2015-06-11 14:46:54 +0200 | [diff] [blame] | 3545 | lockdep_assert_held(&p->pi_lock); |
| 3546 | |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 3547 | if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) |
Valentin Schneider | 3aef1551 | 2020-11-02 18:45:13 +0000 | [diff] [blame] | 3548 | cpu = p->sched_class->select_task_rq(p, cpu, wake_flags); |
Peter Zijlstra (Intel) | e9d867a | 2016-03-10 12:54:08 +0100 | [diff] [blame] | 3549 | else |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3550 | cpu = cpumask_any(p->cpus_ptr); |
Peter Zijlstra | e291200 | 2009-12-16 18:04:36 +0100 | [diff] [blame] | 3551 | |
| 3552 | /* |
| 3553 | * In order not to call set_task_cpu() on a blocking task we need |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 3554 | * to rely on ttwu() to place the task on a valid ->cpus_ptr |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 3555 | * CPU. |
Peter Zijlstra | e291200 | 2009-12-16 18:04:36 +0100 | [diff] [blame] | 3556 | * |
| 3557 | * Since this is common to all placement strategies, this lives here. |
| 3558 | * |
| 3559 | * [ this allows ->select_task() to simply return task_cpu(p) and |
| 3560 | * not worry about this generic constraint ] |
| 3561 | */ |
Paul Burton | 7af443e | 2018-05-26 08:46:47 -0700 | [diff] [blame] | 3562 | if (unlikely(!is_cpu_allowed(p, cpu))) |
Peter Zijlstra | 5da9a0f | 2009-12-16 18:04:38 +0100 | [diff] [blame] | 3563 | cpu = select_fallback_rq(task_cpu(p), p); |
Peter Zijlstra | e291200 | 2009-12-16 18:04:36 +0100 | [diff] [blame] | 3564 | |
| 3565 | return cpu; |
Peter Zijlstra | 970b13b | 2009-11-25 13:31:39 +0100 | [diff] [blame] | 3566 | } |
Mike Galbraith | 09a40af | 2010-04-15 07:29:59 +0200 | [diff] [blame] | 3567 | |
Nicolas Pitre | f5832c1 | 2017-05-29 17:02:57 -0400 | [diff] [blame] | 3568 | void sched_set_stop_task(int cpu, struct task_struct *stop) |
| 3569 | { |
Peter Zijlstra | ded467d | 2020-10-01 16:13:01 +0200 | [diff] [blame] | 3570 | static struct lock_class_key stop_pi_lock; |
Nicolas Pitre | f5832c1 | 2017-05-29 17:02:57 -0400 | [diff] [blame] | 3571 | struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; |
| 3572 | struct task_struct *old_stop = cpu_rq(cpu)->stop; |
| 3573 | |
| 3574 | if (stop) { |
| 3575 | /* |
| 3576 | * Make it appear like a SCHED_FIFO task, its something |
| 3577 | * userspace knows about and won't get confused about. |
| 3578 | * |
| 3579 | * Also, it will make PI more or less work without too |
| 3580 | * much confusion -- but then, stop work should not |
| 3581 | * rely on PI working anyway. |
| 3582 | */ |
| 3583 | sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); |
| 3584 | |
| 3585 | stop->sched_class = &stop_sched_class; |
Peter Zijlstra | ded467d | 2020-10-01 16:13:01 +0200 | [diff] [blame] | 3586 | |
| 3587 | /* |
| 3588 | * The PI code calls rt_mutex_setprio() with ->pi_lock held to |
| 3589 | * adjust the effective priority of a task. As a result, |
| 3590 | * rt_mutex_setprio() can trigger (RT) balancing operations, |
| 3591 | * which can then trigger wakeups of the stop thread to push |
| 3592 | * around the current task. |
| 3593 | * |
| 3594 | * The stop task itself will never be part of the PI-chain, it |
| 3595 | * never blocks, therefore that ->pi_lock recursion is safe. |
| 3596 | * Tell lockdep about this by placing the stop->pi_lock in its |
| 3597 | * own class. |
| 3598 | */ |
| 3599 | lockdep_set_class(&stop->pi_lock, &stop_pi_lock); |
Nicolas Pitre | f5832c1 | 2017-05-29 17:02:57 -0400 | [diff] [blame] | 3600 | } |
| 3601 | |
| 3602 | cpu_rq(cpu)->stop = stop; |
| 3603 | |
| 3604 | if (old_stop) { |
| 3605 | /* |
| 3606 | * Reset it back to a normal scheduling class so that |
| 3607 | * it can die in pieces. |
| 3608 | */ |
| 3609 | old_stop->sched_class = &rt_sched_class; |
| 3610 | } |
| 3611 | } |
| 3612 | |
Thomas Gleixner | 74d862b6 | 2020-11-18 20:48:42 +0100 | [diff] [blame] | 3613 | #else /* CONFIG_SMP */ |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 3614 | |
| 3615 | static inline int __set_cpus_allowed_ptr(struct task_struct *p, |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 3616 | struct affinity_context *ctx) |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 3617 | { |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 3618 | return set_cpus_allowed_ptr(p, ctx->new_mask); |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 3619 | } |
| 3620 | |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 3621 | static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } |
| 3622 | |
Thomas Gleixner | 3015ef4 | 2020-08-26 14:08:10 +0200 | [diff] [blame] | 3623 | static inline bool rq_has_pinned_tasks(struct rq *rq) |
| 3624 | { |
| 3625 | return false; |
| 3626 | } |
| 3627 | |
Waiman Long | 9a5418b | 2022-12-30 23:11:20 -0500 | [diff] [blame] | 3628 | static inline cpumask_t *alloc_user_cpus_ptr(int node) |
| 3629 | { |
| 3630 | return NULL; |
| 3631 | } |
| 3632 | |
Thomas Gleixner | 74d862b6 | 2020-11-18 20:48:42 +0100 | [diff] [blame] | 3633 | #endif /* !CONFIG_SMP */ |
Peter Zijlstra | 970b13b | 2009-11-25 13:31:39 +0100 | [diff] [blame] | 3634 | |
Peter Zijlstra | d7c01d2 | 2011-04-05 17:23:43 +0200 | [diff] [blame] | 3635 | static void |
Peter Zijlstra | b84cb5d | 2011-04-05 17:23:55 +0200 | [diff] [blame] | 3636 | ttwu_stat(struct task_struct *p, int cpu, int wake_flags) |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3637 | { |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3638 | struct rq *rq; |
| 3639 | |
| 3640 | if (!schedstat_enabled()) |
| 3641 | return; |
| 3642 | |
| 3643 | rq = this_rq(); |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3644 | |
Peter Zijlstra | d7c01d2 | 2011-04-05 17:23:43 +0200 | [diff] [blame] | 3645 | #ifdef CONFIG_SMP |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3646 | if (cpu == rq->cpu) { |
Peter Zijlstra | b85c8b7 | 2018-01-16 20:51:06 +0100 | [diff] [blame] | 3647 | __schedstat_inc(rq->ttwu_local); |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 3648 | __schedstat_inc(p->stats.nr_wakeups_local); |
Peter Zijlstra | d7c01d2 | 2011-04-05 17:23:43 +0200 | [diff] [blame] | 3649 | } else { |
| 3650 | struct sched_domain *sd; |
| 3651 | |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 3652 | __schedstat_inc(p->stats.nr_wakeups_remote); |
Peter Zijlstra | 057f3fa | 2011-04-18 11:24:34 +0200 | [diff] [blame] | 3653 | rcu_read_lock(); |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3654 | for_each_domain(rq->cpu, sd) { |
Peter Zijlstra | d7c01d2 | 2011-04-05 17:23:43 +0200 | [diff] [blame] | 3655 | if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { |
Peter Zijlstra | b85c8b7 | 2018-01-16 20:51:06 +0100 | [diff] [blame] | 3656 | __schedstat_inc(sd->ttwu_wake_remote); |
Peter Zijlstra | d7c01d2 | 2011-04-05 17:23:43 +0200 | [diff] [blame] | 3657 | break; |
| 3658 | } |
| 3659 | } |
Peter Zijlstra | 057f3fa | 2011-04-18 11:24:34 +0200 | [diff] [blame] | 3660 | rcu_read_unlock(); |
Peter Zijlstra | d7c01d2 | 2011-04-05 17:23:43 +0200 | [diff] [blame] | 3661 | } |
Peter Zijlstra | f339b9d | 2011-05-31 10:49:20 +0200 | [diff] [blame] | 3662 | |
| 3663 | if (wake_flags & WF_MIGRATED) |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 3664 | __schedstat_inc(p->stats.nr_wakeups_migrate); |
Peter Zijlstra | d7c01d2 | 2011-04-05 17:23:43 +0200 | [diff] [blame] | 3665 | #endif /* CONFIG_SMP */ |
| 3666 | |
Peter Zijlstra | b85c8b7 | 2018-01-16 20:51:06 +0100 | [diff] [blame] | 3667 | __schedstat_inc(rq->ttwu_count); |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 3668 | __schedstat_inc(p->stats.nr_wakeups); |
Peter Zijlstra | d7c01d2 | 2011-04-05 17:23:43 +0200 | [diff] [blame] | 3669 | |
| 3670 | if (wake_flags & WF_SYNC) |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 3671 | __schedstat_inc(p->stats.nr_wakeups_sync); |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3672 | } |
| 3673 | |
Peter Zijlstra | 23f41ee | 2011-04-05 17:23:56 +0200 | [diff] [blame] | 3674 | /* |
| 3675 | * Mark the task runnable and perform wakeup-preemption. |
| 3676 | */ |
Peter Zijlstra | e7904a2 | 2015-08-01 19:25:08 +0200 | [diff] [blame] | 3677 | static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags, |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3678 | struct rq_flags *rf) |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3679 | { |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3680 | check_preempt_curr(rq, p, wake_flags); |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 3681 | WRITE_ONCE(p->__state, TASK_RUNNING); |
Peter Zijlstra | fbd705a | 2015-06-09 11:13:36 +0200 | [diff] [blame] | 3682 | trace_sched_wakeup(p); |
| 3683 | |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3684 | #ifdef CONFIG_SMP |
Peter Zijlstra | 4c9a4bc | 2015-06-11 14:46:39 +0200 | [diff] [blame] | 3685 | if (p->sched_class->task_woken) { |
| 3686 | /* |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 3687 | * Our task @p is fully woken up and running; so it's safe to |
Peter Zijlstra | cbce1a6 | 2015-06-11 14:46:54 +0200 | [diff] [blame] | 3688 | * drop the rq->lock, hereafter rq is only used for statistics. |
Peter Zijlstra | 4c9a4bc | 2015-06-11 14:46:39 +0200 | [diff] [blame] | 3689 | */ |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3690 | rq_unpin_lock(rq, rf); |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3691 | p->sched_class->task_woken(rq, p); |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3692 | rq_repin_lock(rq, rf); |
Peter Zijlstra | 4c9a4bc | 2015-06-11 14:46:39 +0200 | [diff] [blame] | 3693 | } |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3694 | |
Steven Rostedt | e69c634 | 2010-12-06 17:10:31 -0500 | [diff] [blame] | 3695 | if (rq->idle_stamp) { |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 3696 | u64 delta = rq_clock(rq) - rq->idle_stamp; |
Jason Low | 9bd721c | 2013-09-13 11:26:52 -0700 | [diff] [blame] | 3697 | u64 max = 2*rq->max_idle_balance_cost; |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3698 | |
Jason Low | abfafa5 | 2013-09-13 11:26:51 -0700 | [diff] [blame] | 3699 | update_avg(&rq->avg_idle, delta); |
| 3700 | |
| 3701 | if (rq->avg_idle > max) |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3702 | rq->avg_idle = max; |
Jason Low | abfafa5 | 2013-09-13 11:26:51 -0700 | [diff] [blame] | 3703 | |
Peter Zijlstra | 94aafc3 | 2021-06-15 12:16:11 +0100 | [diff] [blame] | 3704 | rq->wake_stamp = jiffies; |
| 3705 | rq->wake_avg_idle = rq->avg_idle / 2; |
| 3706 | |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3707 | rq->idle_stamp = 0; |
| 3708 | } |
| 3709 | #endif |
| 3710 | } |
| 3711 | |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3712 | static void |
Peter Zijlstra | e7904a2 | 2015-08-01 19:25:08 +0200 | [diff] [blame] | 3713 | ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3714 | struct rq_flags *rf) |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3715 | { |
Peter Zijlstra | 77558e4 | 2017-02-21 14:36:23 +0100 | [diff] [blame] | 3716 | int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; |
Peter Zijlstra | b5179ac | 2016-05-11 16:10:34 +0200 | [diff] [blame] | 3717 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 3718 | lockdep_assert_rq_held(rq); |
Peter Zijlstra | cbce1a6 | 2015-06-11 14:46:54 +0200 | [diff] [blame] | 3719 | |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3720 | if (p->sched_contributes_to_load) |
| 3721 | rq->nr_uninterruptible--; |
Peter Zijlstra | b5179ac | 2016-05-11 16:10:34 +0200 | [diff] [blame] | 3722 | |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 3723 | #ifdef CONFIG_SMP |
Peter Zijlstra | b5179ac | 2016-05-11 16:10:34 +0200 | [diff] [blame] | 3724 | if (wake_flags & WF_MIGRATED) |
Peter Zijlstra | 59efa0b | 2016-05-10 18:24:37 +0200 | [diff] [blame] | 3725 | en_flags |= ENQUEUE_MIGRATED; |
Peter Zijlstra | ec618b8 | 2020-09-24 13:50:42 +0200 | [diff] [blame] | 3726 | else |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3727 | #endif |
Peter Zijlstra | ec618b8 | 2020-09-24 13:50:42 +0200 | [diff] [blame] | 3728 | if (p->in_iowait) { |
| 3729 | delayacct_blkio_end(p); |
| 3730 | atomic_dec(&task_rq(p)->nr_iowait); |
| 3731 | } |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3732 | |
Peter Zijlstra | 1b174a2 | 2019-04-09 09:53:13 +0200 | [diff] [blame] | 3733 | activate_task(rq, p, en_flags); |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3734 | ttwu_do_wakeup(rq, p, wake_flags, rf); |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3735 | } |
| 3736 | |
| 3737 | /* |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 3738 | * Consider @p being inside a wait loop: |
| 3739 | * |
| 3740 | * for (;;) { |
| 3741 | * set_current_state(TASK_UNINTERRUPTIBLE); |
| 3742 | * |
| 3743 | * if (CONDITION) |
| 3744 | * break; |
| 3745 | * |
| 3746 | * schedule(); |
| 3747 | * } |
| 3748 | * __set_current_state(TASK_RUNNING); |
| 3749 | * |
| 3750 | * between set_current_state() and schedule(). In this case @p is still |
| 3751 | * runnable, so all that needs doing is change p->state back to TASK_RUNNING in |
| 3752 | * an atomic manner. |
| 3753 | * |
| 3754 | * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq |
| 3755 | * then schedule() must still happen and p->state can be changed to |
| 3756 | * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we |
| 3757 | * need to do a full wakeup with enqueue. |
| 3758 | * |
| 3759 | * Returns: %true when the wakeup is done, |
| 3760 | * %false otherwise. |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3761 | */ |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 3762 | static int ttwu_runnable(struct task_struct *p, int wake_flags) |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3763 | { |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 3764 | struct rq_flags rf; |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3765 | struct rq *rq; |
| 3766 | int ret = 0; |
| 3767 | |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 3768 | rq = __task_rq_lock(p, &rf); |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 3769 | if (task_on_rq_queued(p)) { |
Frederic Weisbecker | 1ad4ec0 | 2013-04-12 01:51:00 +0200 | [diff] [blame] | 3770 | /* check_preempt_curr() may use rq clock */ |
| 3771 | update_rq_clock(rq); |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3772 | ttwu_do_wakeup(rq, p, wake_flags, &rf); |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3773 | ret = 1; |
| 3774 | } |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 3775 | __task_rq_unlock(rq, &rf); |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3776 | |
| 3777 | return ret; |
| 3778 | } |
| 3779 | |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3780 | #ifdef CONFIG_SMP |
Peter Zijlstra | a1488664 | 2020-05-26 18:11:04 +0200 | [diff] [blame] | 3781 | void sched_ttwu_pending(void *arg) |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3782 | { |
Peter Zijlstra | a1488664 | 2020-05-26 18:11:04 +0200 | [diff] [blame] | 3783 | struct llist_node *llist = arg; |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3784 | struct rq *rq = this_rq(); |
Byungchul Park | 7321584 | 2017-05-12 09:39:44 +0900 | [diff] [blame] | 3785 | struct task_struct *p, *t; |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3786 | struct rq_flags rf; |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3787 | |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 3788 | if (!llist) |
| 3789 | return; |
| 3790 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 3791 | rq_lock_irqsave(rq, &rf); |
Peter Zijlstra | 77558e4 | 2017-02-21 14:36:23 +0100 | [diff] [blame] | 3792 | update_rq_clock(rq); |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3793 | |
Peter Zijlstra | 8c4890d | 2020-06-22 12:01:25 +0200 | [diff] [blame] | 3794 | llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { |
Peter Zijlstra | b6e13e8 | 2020-06-22 12:01:23 +0200 | [diff] [blame] | 3795 | if (WARN_ON_ONCE(p->on_cpu)) |
| 3796 | smp_cond_load_acquire(&p->on_cpu, !VAL); |
| 3797 | |
| 3798 | if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) |
| 3799 | set_task_cpu(p, cpu_of(rq)); |
| 3800 | |
Byungchul Park | 7321584 | 2017-05-12 09:39:44 +0900 | [diff] [blame] | 3801 | ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); |
Peter Zijlstra | b6e13e8 | 2020-06-22 12:01:23 +0200 | [diff] [blame] | 3802 | } |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3803 | |
Tianchen Ding | d6962c4 | 2022-11-04 10:36:01 +0800 | [diff] [blame] | 3804 | /* |
| 3805 | * Must be after enqueueing at least once task such that |
| 3806 | * idle_cpu() does not observe a false-negative -- if it does, |
| 3807 | * it is possible for select_idle_siblings() to stack a number |
| 3808 | * of tasks on this CPU during that window. |
| 3809 | * |
| 3810 | * It is ok to clear ttwu_pending when another task pending. |
| 3811 | * We will receive IPI after local irq enabled and then enqueue it. |
| 3812 | * Since now nr_running > 0, idle_cpu() will always get correct result. |
| 3813 | */ |
| 3814 | WRITE_ONCE(rq->ttwu_pending, 0); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 3815 | rq_unlock_irqrestore(rq, &rf); |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3816 | } |
| 3817 | |
Peter Zijlstra | b2a02fc | 2020-05-26 18:11:01 +0200 | [diff] [blame] | 3818 | void send_call_function_single_ipi(int cpu) |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3819 | { |
Peter Zijlstra | b2a02fc | 2020-05-26 18:11:01 +0200 | [diff] [blame] | 3820 | struct rq *rq = cpu_rq(cpu); |
Peter Zijlstra | f27dde8 | 2013-08-14 14:55:31 +0200 | [diff] [blame] | 3821 | |
Peter Zijlstra | b2a02fc | 2020-05-26 18:11:01 +0200 | [diff] [blame] | 3822 | if (!set_nr_if_polling(rq->idle)) |
| 3823 | arch_send_call_function_single_ipi(cpu); |
| 3824 | else |
| 3825 | trace_sched_wake_idle_without_ipi(cpu); |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3826 | } |
| 3827 | |
Mel Gorman | 2ebb177 | 2020-05-24 21:29:56 +0100 | [diff] [blame] | 3828 | /* |
| 3829 | * Queue a task on the target CPUs wake_list and wake the CPU via IPI if |
| 3830 | * necessary. The wakee CPU on receipt of the IPI will queue the task |
| 3831 | * via sched_ttwu_wakeup() for activation so the wakee incurs the cost |
| 3832 | * of the wakeup instead of the waker. |
| 3833 | */ |
| 3834 | static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3835 | { |
Peter Zijlstra | e3baac47 | 2014-06-04 10:31:18 -0700 | [diff] [blame] | 3836 | struct rq *rq = cpu_rq(cpu); |
| 3837 | |
Peter Zijlstra | b7e7ade | 2016-05-23 11:19:07 +0200 | [diff] [blame] | 3838 | p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); |
| 3839 | |
Peter Zijlstra | 126c209 | 2020-05-26 18:11:03 +0200 | [diff] [blame] | 3840 | WRITE_ONCE(rq->ttwu_pending, 1); |
Peter Zijlstra | 8c4890d | 2020-06-22 12:01:25 +0200 | [diff] [blame] | 3841 | __smp_call_single_queue(cpu, &p->wake_entry.llist); |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3842 | } |
Peter Zijlstra | d6aa8f8 | 2011-05-26 14:21:33 +0200 | [diff] [blame] | 3843 | |
Chuansheng Liu | f6be8af | 2014-09-04 15:17:53 +0800 | [diff] [blame] | 3844 | void wake_up_if_idle(int cpu) |
| 3845 | { |
| 3846 | struct rq *rq = cpu_rq(cpu); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 3847 | struct rq_flags rf; |
Chuansheng Liu | f6be8af | 2014-09-04 15:17:53 +0800 | [diff] [blame] | 3848 | |
Andy Lutomirski | fd7de1e8 | 2014-11-29 08:13:51 -0800 | [diff] [blame] | 3849 | rcu_read_lock(); |
| 3850 | |
| 3851 | if (!is_idle_task(rcu_dereference(rq->curr))) |
| 3852 | goto out; |
Chuansheng Liu | f6be8af | 2014-09-04 15:17:53 +0800 | [diff] [blame] | 3853 | |
Peter Zijlstra | 8850cb6 | 2021-09-21 22:16:02 +0200 | [diff] [blame] | 3854 | rq_lock_irqsave(rq, &rf); |
| 3855 | if (is_idle_task(rq->curr)) |
| 3856 | resched_curr(rq); |
| 3857 | /* Else CPU is not idle, do nothing here: */ |
| 3858 | rq_unlock_irqrestore(rq, &rf); |
Andy Lutomirski | fd7de1e8 | 2014-11-29 08:13:51 -0800 | [diff] [blame] | 3859 | |
| 3860 | out: |
| 3861 | rcu_read_unlock(); |
Chuansheng Liu | f6be8af | 2014-09-04 15:17:53 +0800 | [diff] [blame] | 3862 | } |
| 3863 | |
Peter Zijlstra | 39be350 | 2012-01-26 12:44:34 +0100 | [diff] [blame] | 3864 | bool cpus_share_cache(int this_cpu, int that_cpu) |
Peter Zijlstra | 518cd62 | 2011-12-07 15:07:31 +0100 | [diff] [blame] | 3865 | { |
Vincent Donnefort | 42dc938 | 2021-11-04 17:51:20 +0000 | [diff] [blame] | 3866 | if (this_cpu == that_cpu) |
| 3867 | return true; |
| 3868 | |
Peter Zijlstra | 518cd62 | 2011-12-07 15:07:31 +0100 | [diff] [blame] | 3869 | return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); |
| 3870 | } |
Peter Zijlstra | c6e7bd7 | 2020-05-24 21:29:55 +0100 | [diff] [blame] | 3871 | |
Mel Gorman | 751d4cb | 2022-08-04 10:21:19 +0100 | [diff] [blame] | 3872 | static inline bool ttwu_queue_cond(struct task_struct *p, int cpu) |
Peter Zijlstra | c6e7bd7 | 2020-05-24 21:29:55 +0100 | [diff] [blame] | 3873 | { |
Mel Gorman | 2ebb177 | 2020-05-24 21:29:56 +0100 | [diff] [blame] | 3874 | /* |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 3875 | * Do not complicate things with the async wake_list while the CPU is |
| 3876 | * in hotplug state. |
| 3877 | */ |
| 3878 | if (!cpu_active(cpu)) |
| 3879 | return false; |
| 3880 | |
Mel Gorman | 751d4cb | 2022-08-04 10:21:19 +0100 | [diff] [blame] | 3881 | /* Ensure the task will still be allowed to run on the CPU. */ |
| 3882 | if (!cpumask_test_cpu(cpu, p->cpus_ptr)) |
| 3883 | return false; |
| 3884 | |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 3885 | /* |
Mel Gorman | 2ebb177 | 2020-05-24 21:29:56 +0100 | [diff] [blame] | 3886 | * If the CPU does not share cache, then queue the task on the |
| 3887 | * remote rqs wakelist to avoid accessing remote data. |
| 3888 | */ |
| 3889 | if (!cpus_share_cache(smp_processor_id(), cpu)) |
| 3890 | return true; |
| 3891 | |
Tianchen Ding | f3dd3f6 | 2022-06-09 07:34:12 +0800 | [diff] [blame] | 3892 | if (cpu == smp_processor_id()) |
| 3893 | return false; |
| 3894 | |
Mel Gorman | 2ebb177 | 2020-05-24 21:29:56 +0100 | [diff] [blame] | 3895 | /* |
Tianchen Ding | f3dd3f6 | 2022-06-09 07:34:12 +0800 | [diff] [blame] | 3896 | * If the wakee cpu is idle, or the task is descheduling and the |
| 3897 | * only running task on the CPU, then use the wakelist to offload |
| 3898 | * the task activation to the idle (or soon-to-be-idle) CPU as |
| 3899 | * the current CPU is likely busy. nr_running is checked to |
| 3900 | * avoid unnecessary task stacking. |
Tianchen Ding | 2815610 | 2022-06-09 07:34:11 +0800 | [diff] [blame] | 3901 | * |
| 3902 | * Note that we can only get here with (wakee) p->on_rq=0, |
| 3903 | * p->on_cpu can be whatever, we've done the dequeue, so |
| 3904 | * the wakee has been accounted out of ->nr_running. |
Mel Gorman | 2ebb177 | 2020-05-24 21:29:56 +0100 | [diff] [blame] | 3905 | */ |
Tianchen Ding | f3dd3f6 | 2022-06-09 07:34:12 +0800 | [diff] [blame] | 3906 | if (!cpu_rq(cpu)->nr_running) |
Mel Gorman | 2ebb177 | 2020-05-24 21:29:56 +0100 | [diff] [blame] | 3907 | return true; |
| 3908 | |
| 3909 | return false; |
| 3910 | } |
| 3911 | |
| 3912 | static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) |
| 3913 | { |
Mel Gorman | 751d4cb | 2022-08-04 10:21:19 +0100 | [diff] [blame] | 3914 | if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) { |
Peter Zijlstra | c6e7bd7 | 2020-05-24 21:29:55 +0100 | [diff] [blame] | 3915 | sched_clock_cpu(cpu); /* Sync clocks across CPUs */ |
Mel Gorman | 2ebb177 | 2020-05-24 21:29:56 +0100 | [diff] [blame] | 3916 | __ttwu_queue_wakelist(p, cpu, wake_flags); |
Peter Zijlstra | c6e7bd7 | 2020-05-24 21:29:55 +0100 | [diff] [blame] | 3917 | return true; |
| 3918 | } |
| 3919 | |
| 3920 | return false; |
| 3921 | } |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 3922 | |
| 3923 | #else /* !CONFIG_SMP */ |
| 3924 | |
| 3925 | static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) |
| 3926 | { |
| 3927 | return false; |
| 3928 | } |
| 3929 | |
Peter Zijlstra | d6aa8f8 | 2011-05-26 14:21:33 +0200 | [diff] [blame] | 3930 | #endif /* CONFIG_SMP */ |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3931 | |
Peter Zijlstra | b5179ac | 2016-05-11 16:10:34 +0200 | [diff] [blame] | 3932 | static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3933 | { |
| 3934 | struct rq *rq = cpu_rq(cpu); |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3935 | struct rq_flags rf; |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 3936 | |
Mel Gorman | 2ebb177 | 2020-05-24 21:29:56 +0100 | [diff] [blame] | 3937 | if (ttwu_queue_wakelist(p, cpu, wake_flags)) |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3938 | return; |
Peter Zijlstra | 317f394 | 2011-04-05 17:23:58 +0200 | [diff] [blame] | 3939 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 3940 | rq_lock(rq, &rf); |
Peter Zijlstra | 77558e4 | 2017-02-21 14:36:23 +0100 | [diff] [blame] | 3941 | update_rq_clock(rq); |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 3942 | ttwu_do_activate(rq, p, wake_flags, &rf); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 3943 | rq_unlock(rq, &rf); |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 3944 | } |
| 3945 | |
Peter Zijlstra | 8643cda | 2015-11-17 19:01:11 +0100 | [diff] [blame] | 3946 | /* |
Thomas Gleixner | 43295d7 | 2021-08-15 23:27:40 +0200 | [diff] [blame] | 3947 | * Invoked from try_to_wake_up() to check whether the task can be woken up. |
| 3948 | * |
| 3949 | * The caller holds p::pi_lock if p != current or has preemption |
| 3950 | * disabled when p == current. |
Thomas Gleixner | 5f220be | 2021-08-15 23:27:44 +0200 | [diff] [blame] | 3951 | * |
| 3952 | * The rules of PREEMPT_RT saved_state: |
| 3953 | * |
| 3954 | * The related locking code always holds p::pi_lock when updating |
| 3955 | * p::saved_state, which means the code is fully serialized in both cases. |
| 3956 | * |
| 3957 | * The lock wait and lock wakeups happen via TASK_RTLOCK_WAIT. No other |
| 3958 | * bits set. This allows to distinguish all wakeup scenarios. |
Thomas Gleixner | 43295d7 | 2021-08-15 23:27:40 +0200 | [diff] [blame] | 3959 | */ |
| 3960 | static __always_inline |
| 3961 | bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) |
| 3962 | { |
Thomas Gleixner | 5f220be | 2021-08-15 23:27:44 +0200 | [diff] [blame] | 3963 | if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { |
| 3964 | WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) && |
| 3965 | state != TASK_RTLOCK_WAIT); |
| 3966 | } |
| 3967 | |
Thomas Gleixner | 43295d7 | 2021-08-15 23:27:40 +0200 | [diff] [blame] | 3968 | if (READ_ONCE(p->__state) & state) { |
| 3969 | *success = 1; |
| 3970 | return true; |
| 3971 | } |
Thomas Gleixner | 5f220be | 2021-08-15 23:27:44 +0200 | [diff] [blame] | 3972 | |
| 3973 | #ifdef CONFIG_PREEMPT_RT |
| 3974 | /* |
| 3975 | * Saved state preserves the task state across blocking on |
| 3976 | * an RT lock. If the state matches, set p::saved_state to |
| 3977 | * TASK_RUNNING, but do not wake the task because it waits |
| 3978 | * for a lock wakeup. Also indicate success because from |
| 3979 | * the regular waker's point of view this has succeeded. |
| 3980 | * |
| 3981 | * After acquiring the lock the task will restore p::__state |
| 3982 | * from p::saved_state which ensures that the regular |
| 3983 | * wakeup is not lost. The restore will also set |
| 3984 | * p::saved_state to TASK_RUNNING so any further tests will |
| 3985 | * not result in false positives vs. @success |
| 3986 | */ |
| 3987 | if (p->saved_state & state) { |
| 3988 | p->saved_state = TASK_RUNNING; |
| 3989 | *success = 1; |
| 3990 | } |
| 3991 | #endif |
Thomas Gleixner | 43295d7 | 2021-08-15 23:27:40 +0200 | [diff] [blame] | 3992 | return false; |
| 3993 | } |
| 3994 | |
| 3995 | /* |
Peter Zijlstra | 8643cda | 2015-11-17 19:01:11 +0100 | [diff] [blame] | 3996 | * Notes on Program-Order guarantees on SMP systems. |
| 3997 | * |
| 3998 | * MIGRATION |
| 3999 | * |
| 4000 | * The basic program-order guarantee on SMP systems is that when a task [t] |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 4001 | * migrates, all its activity on its old CPU [c0] happens-before any subsequent |
| 4002 | * execution on its new CPU [c1]. |
Peter Zijlstra | 8643cda | 2015-11-17 19:01:11 +0100 | [diff] [blame] | 4003 | * |
| 4004 | * For migration (of runnable tasks) this is provided by the following means: |
| 4005 | * |
| 4006 | * A) UNLOCK of the rq(c0)->lock scheduling out task t |
| 4007 | * B) migration for t is required to synchronize *both* rq(c0)->lock and |
| 4008 | * rq(c1)->lock (if not at the same time, then in that order). |
| 4009 | * C) LOCK of the rq(c1)->lock scheduling in task |
| 4010 | * |
Andrea Parri | 7696f99 | 2018-07-16 11:06:03 -0700 | [diff] [blame] | 4011 | * Release/acquire chaining guarantees that B happens after A and C after B. |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 4012 | * Note: the CPU doing B need not be c0 or c1 |
Peter Zijlstra | 8643cda | 2015-11-17 19:01:11 +0100 | [diff] [blame] | 4013 | * |
| 4014 | * Example: |
| 4015 | * |
| 4016 | * CPU0 CPU1 CPU2 |
| 4017 | * |
| 4018 | * LOCK rq(0)->lock |
| 4019 | * sched-out X |
| 4020 | * sched-in Y |
| 4021 | * UNLOCK rq(0)->lock |
| 4022 | * |
| 4023 | * LOCK rq(0)->lock // orders against CPU0 |
| 4024 | * dequeue X |
| 4025 | * UNLOCK rq(0)->lock |
| 4026 | * |
| 4027 | * LOCK rq(1)->lock |
| 4028 | * enqueue X |
| 4029 | * UNLOCK rq(1)->lock |
| 4030 | * |
| 4031 | * LOCK rq(1)->lock // orders against CPU2 |
| 4032 | * sched-out Z |
| 4033 | * sched-in X |
| 4034 | * UNLOCK rq(1)->lock |
| 4035 | * |
| 4036 | * |
| 4037 | * BLOCKING -- aka. SLEEP + WAKEUP |
| 4038 | * |
| 4039 | * For blocking we (obviously) need to provide the same guarantee as for |
| 4040 | * migration. However the means are completely different as there is no lock |
| 4041 | * chain to provide order. Instead we do: |
| 4042 | * |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4043 | * 1) smp_store_release(X->on_cpu, 0) -- finish_task() |
| 4044 | * 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up() |
Peter Zijlstra | 8643cda | 2015-11-17 19:01:11 +0100 | [diff] [blame] | 4045 | * |
| 4046 | * Example: |
| 4047 | * |
| 4048 | * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) |
| 4049 | * |
| 4050 | * LOCK rq(0)->lock LOCK X->pi_lock |
| 4051 | * dequeue X |
| 4052 | * sched-out X |
| 4053 | * smp_store_release(X->on_cpu, 0); |
| 4054 | * |
Peter Zijlstra | 1f03e8d | 2016-04-04 10:57:12 +0200 | [diff] [blame] | 4055 | * smp_cond_load_acquire(&X->on_cpu, !VAL); |
Peter Zijlstra | 8643cda | 2015-11-17 19:01:11 +0100 | [diff] [blame] | 4056 | * X->state = WAKING |
| 4057 | * set_task_cpu(X,2) |
| 4058 | * |
| 4059 | * LOCK rq(2)->lock |
| 4060 | * enqueue X |
| 4061 | * X->state = RUNNING |
| 4062 | * UNLOCK rq(2)->lock |
| 4063 | * |
| 4064 | * LOCK rq(2)->lock // orders against CPU1 |
| 4065 | * sched-out Z |
| 4066 | * sched-in X |
| 4067 | * UNLOCK rq(2)->lock |
| 4068 | * |
| 4069 | * UNLOCK X->pi_lock |
| 4070 | * UNLOCK rq(0)->lock |
| 4071 | * |
| 4072 | * |
Andrea Parri | 7696f99 | 2018-07-16 11:06:03 -0700 | [diff] [blame] | 4073 | * However, for wakeups there is a second guarantee we must provide, namely we |
| 4074 | * must ensure that CONDITION=1 done by the caller can not be reordered with |
| 4075 | * accesses to the task state; see try_to_wake_up() and set_current_state(). |
Peter Zijlstra | 8643cda | 2015-11-17 19:01:11 +0100 | [diff] [blame] | 4076 | */ |
| 4077 | |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 4078 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4079 | * try_to_wake_up - wake up a thread |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 4080 | * @p: the thread to be awakened |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4081 | * @state: the mask of task states that can be woken |
Tejun Heo | 9ed3811 | 2009-12-03 15:08:03 +0900 | [diff] [blame] | 4082 | * @wake_flags: wake modifier flags (WF_*) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4083 | * |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4084 | * Conceptually does: |
| 4085 | * |
| 4086 | * If (@state & @p->state) @p->state = TASK_RUNNING. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4087 | * |
Peter Zijlstra | a225023 | 2016-10-19 15:45:27 +0200 | [diff] [blame] | 4088 | * If the task was not queued/runnable, also place it back on a runqueue. |
| 4089 | * |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4090 | * This function is atomic against schedule() which would dequeue the task. |
Peter Zijlstra | a225023 | 2016-10-19 15:45:27 +0200 | [diff] [blame] | 4091 | * |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4092 | * It issues a full memory barrier before accessing @p->state, see the comment |
| 4093 | * with set_current_state(). |
| 4094 | * |
| 4095 | * Uses p->pi_lock to serialize against concurrent wake-ups. |
| 4096 | * |
| 4097 | * Relies on p->pi_lock stabilizing: |
| 4098 | * - p->sched_class |
| 4099 | * - p->cpus_ptr |
| 4100 | * - p->sched_task_group |
| 4101 | * in order to do migration, see its use of select_task_rq()/set_task_cpu(). |
| 4102 | * |
| 4103 | * Tries really hard to only take one task_rq(p)->lock for performance. |
| 4104 | * Takes rq->lock in: |
| 4105 | * - ttwu_runnable() -- old rq, unavoidable, see comment there; |
| 4106 | * - ttwu_queue() -- new rq, for enqueue of the task; |
| 4107 | * - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us. |
| 4108 | * |
| 4109 | * As a consequence we race really badly with just about everything. See the |
| 4110 | * many memory barriers and their comments for details. |
Andrea Parri | 7696f99 | 2018-07-16 11:06:03 -0700 | [diff] [blame] | 4111 | * |
Peter Zijlstra | a225023 | 2016-10-19 15:45:27 +0200 | [diff] [blame] | 4112 | * Return: %true if @p->state changes (an actual wakeup was done), |
| 4113 | * %false otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4114 | */ |
Peter Zijlstra | e4a52bc | 2011-04-05 17:23:54 +0200 | [diff] [blame] | 4115 | static int |
| 4116 | try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4117 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4118 | unsigned long flags; |
Peter Zijlstra | c05fbaf | 2011-04-05 17:23:57 +0200 | [diff] [blame] | 4119 | int cpu, success = 0; |
Peter Zijlstra | 2398f2c | 2008-06-27 13:41:35 +0200 | [diff] [blame] | 4120 | |
Peter Zijlstra | e3d8548 | 2019-07-10 12:57:36 +0200 | [diff] [blame] | 4121 | preempt_disable(); |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4122 | if (p == current) { |
| 4123 | /* |
| 4124 | * We're waking current, this means 'p->on_rq' and 'task_cpu(p) |
| 4125 | * == smp_processor_id()'. Together this means we can special |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4126 | * case the whole 'p->on_rq && ttwu_runnable()' case below |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4127 | * without taking any locks. |
| 4128 | * |
| 4129 | * In particular: |
| 4130 | * - we rely on Program-Order guarantees for all the ordering, |
| 4131 | * - we're serialized against set_special_state() by virtue of |
| 4132 | * it disabling IRQs (this allows not taking ->pi_lock). |
| 4133 | */ |
Thomas Gleixner | 43295d7 | 2021-08-15 23:27:40 +0200 | [diff] [blame] | 4134 | if (!ttwu_state_match(p, state, &success)) |
Peter Zijlstra | e3d8548 | 2019-07-10 12:57:36 +0200 | [diff] [blame] | 4135 | goto out; |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4136 | |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4137 | trace_sched_waking(p); |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 4138 | WRITE_ONCE(p->__state, TASK_RUNNING); |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4139 | trace_sched_wakeup(p); |
| 4140 | goto out; |
| 4141 | } |
| 4142 | |
Oleg Nesterov | e0acd0a | 2013-08-12 18:14:00 +0200 | [diff] [blame] | 4143 | /* |
| 4144 | * If we are going to wake up a thread waiting for CONDITION we |
| 4145 | * need to ensure that CONDITION=1 done by the caller can not be |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4146 | * reordered with p->state check below. This pairs with smp_store_mb() |
| 4147 | * in set_current_state() that the waiting thread does. |
Oleg Nesterov | e0acd0a | 2013-08-12 18:14:00 +0200 | [diff] [blame] | 4148 | */ |
Peter Zijlstra | 013fdb8 | 2011-04-05 17:23:45 +0200 | [diff] [blame] | 4149 | raw_spin_lock_irqsave(&p->pi_lock, flags); |
Peter Zijlstra | d89e588c | 2016-09-05 11:37:53 +0200 | [diff] [blame] | 4150 | smp_mb__after_spinlock(); |
Thomas Gleixner | 43295d7 | 2021-08-15 23:27:40 +0200 | [diff] [blame] | 4151 | if (!ttwu_state_match(p, state, &success)) |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4152 | goto unlock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4153 | |
Peter Zijlstra | fbd705a | 2015-06-09 11:13:36 +0200 | [diff] [blame] | 4154 | trace_sched_waking(p); |
| 4155 | |
Balbir Singh | 135e8c9 | 2016-09-05 13:16:40 +1000 | [diff] [blame] | 4156 | /* |
| 4157 | * Ensure we load p->on_rq _after_ p->state, otherwise it would |
| 4158 | * be possible to, falsely, observe p->on_rq == 0 and get stuck |
| 4159 | * in smp_cond_load_acquire() below. |
| 4160 | * |
Andrea Parri | 3d85b27 | 2018-07-16 11:06:02 -0700 | [diff] [blame] | 4161 | * sched_ttwu_pending() try_to_wake_up() |
| 4162 | * STORE p->on_rq = 1 LOAD p->state |
| 4163 | * UNLOCK rq->lock |
| 4164 | * |
| 4165 | * __schedule() (switch to task 'p') |
| 4166 | * LOCK rq->lock smp_rmb(); |
| 4167 | * smp_mb__after_spinlock(); |
| 4168 | * UNLOCK rq->lock |
Balbir Singh | 135e8c9 | 2016-09-05 13:16:40 +1000 | [diff] [blame] | 4169 | * |
| 4170 | * [task p] |
Andrea Parri | 3d85b27 | 2018-07-16 11:06:02 -0700 | [diff] [blame] | 4171 | * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq |
Balbir Singh | 135e8c9 | 2016-09-05 13:16:40 +1000 | [diff] [blame] | 4172 | * |
Andrea Parri | 3d85b27 | 2018-07-16 11:06:02 -0700 | [diff] [blame] | 4173 | * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in |
| 4174 | * __schedule(). See the comment for smp_mb__after_spinlock(). |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4175 | * |
| 4176 | * A similar smb_rmb() lives in try_invoke_on_locked_down_task(). |
Balbir Singh | 135e8c9 | 2016-09-05 13:16:40 +1000 | [diff] [blame] | 4177 | */ |
| 4178 | smp_rmb(); |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4179 | if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4180 | goto unlock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4181 | |
| 4182 | #ifdef CONFIG_SMP |
Peter Zijlstra | e9c8431 | 2009-09-15 14:43:03 +0200 | [diff] [blame] | 4183 | /* |
Peter Zijlstra | ecf7d01 | 2015-10-07 14:14:13 +0200 | [diff] [blame] | 4184 | * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be |
| 4185 | * possible to, falsely, observe p->on_cpu == 0. |
| 4186 | * |
| 4187 | * One must be running (->on_cpu == 1) in order to remove oneself |
| 4188 | * from the runqueue. |
| 4189 | * |
Andrea Parri | 3d85b27 | 2018-07-16 11:06:02 -0700 | [diff] [blame] | 4190 | * __schedule() (switch to task 'p') try_to_wake_up() |
| 4191 | * STORE p->on_cpu = 1 LOAD p->on_rq |
| 4192 | * UNLOCK rq->lock |
Peter Zijlstra | ecf7d01 | 2015-10-07 14:14:13 +0200 | [diff] [blame] | 4193 | * |
Andrea Parri | 3d85b27 | 2018-07-16 11:06:02 -0700 | [diff] [blame] | 4194 | * __schedule() (put 'p' to sleep) |
| 4195 | * LOCK rq->lock smp_rmb(); |
| 4196 | * smp_mb__after_spinlock(); |
| 4197 | * STORE p->on_rq = 0 LOAD p->on_cpu |
| 4198 | * |
| 4199 | * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in |
| 4200 | * __schedule(). See the comment for smp_mb__after_spinlock(). |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 4201 | * |
| 4202 | * Form a control-dep-acquire with p->on_rq == 0 above, to ensure |
| 4203 | * schedule()'s deactivate_task() has 'happened' and p will no longer |
| 4204 | * care about it's own p->state. See the comment in __schedule(). |
Peter Zijlstra | ecf7d01 | 2015-10-07 14:14:13 +0200 | [diff] [blame] | 4205 | */ |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 4206 | smp_acquire__after_ctrl_dep(); |
| 4207 | |
| 4208 | /* |
| 4209 | * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq |
| 4210 | * == 0), which means we need to do an enqueue, change p->state to |
| 4211 | * TASK_WAKING such that we can unlock p->pi_lock before doing the |
| 4212 | * enqueue, such as ttwu_queue_wakelist(). |
| 4213 | */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 4214 | WRITE_ONCE(p->__state, TASK_WAKING); |
Peter Zijlstra | ecf7d01 | 2015-10-07 14:14:13 +0200 | [diff] [blame] | 4215 | |
| 4216 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 4217 | * If the owning (remote) CPU is still in the middle of schedule() with |
Peter Zijlstra | c6e7bd7 | 2020-05-24 21:29:55 +0100 | [diff] [blame] | 4218 | * this task as prev, considering queueing p on the remote CPUs wake_list |
| 4219 | * which potentially sends an IPI instead of spinning on p->on_cpu to |
| 4220 | * let the waker make forward progress. This is safe because IRQs are |
| 4221 | * disabled and the IPI will deliver after on_cpu is cleared. |
Peter Zijlstra | b6e13e8 | 2020-06-22 12:01:23 +0200 | [diff] [blame] | 4222 | * |
| 4223 | * Ensure we load task_cpu(p) after p->on_cpu: |
| 4224 | * |
| 4225 | * set_task_cpu(p, cpu); |
| 4226 | * STORE p->cpu = @cpu |
| 4227 | * __schedule() (switch to task 'p') |
| 4228 | * LOCK rq->lock |
| 4229 | * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) |
| 4230 | * STORE p->on_cpu = 1 LOAD p->cpu |
| 4231 | * |
| 4232 | * to ensure we observe the correct CPU on which the task is currently |
| 4233 | * scheduling. |
Peter Zijlstra | c6e7bd7 | 2020-05-24 21:29:55 +0100 | [diff] [blame] | 4234 | */ |
Peter Zijlstra | b6e13e8 | 2020-06-22 12:01:23 +0200 | [diff] [blame] | 4235 | if (smp_load_acquire(&p->on_cpu) && |
Tianchen Ding | f3dd3f6 | 2022-06-09 07:34:12 +0800 | [diff] [blame] | 4236 | ttwu_queue_wakelist(p, task_cpu(p), wake_flags)) |
Peter Zijlstra | c6e7bd7 | 2020-05-24 21:29:55 +0100 | [diff] [blame] | 4237 | goto unlock; |
| 4238 | |
| 4239 | /* |
| 4240 | * If the owning (remote) CPU is still in the middle of schedule() with |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 4241 | * this task as prev, wait until it's done referencing the task. |
Peter Zijlstra | b75a225 | 2015-10-06 14:36:17 +0200 | [diff] [blame] | 4242 | * |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 4243 | * Pairs with the smp_store_release() in finish_task(). |
Peter Zijlstra | b75a225 | 2015-10-06 14:36:17 +0200 | [diff] [blame] | 4244 | * |
| 4245 | * This ensures that tasks getting woken will be fully ordered against |
| 4246 | * their previous state and preserve Program Order. |
Peter Zijlstra | e4a52bc | 2011-04-05 17:23:54 +0200 | [diff] [blame] | 4247 | */ |
Peter Zijlstra | 1f03e8d | 2016-04-04 10:57:12 +0200 | [diff] [blame] | 4248 | smp_cond_load_acquire(&p->on_cpu, !VAL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4249 | |
Valentin Schneider | 3aef1551 | 2020-11-02 18:45:13 +0000 | [diff] [blame] | 4250 | cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); |
Peter Zijlstra | f339b9d | 2011-05-31 10:49:20 +0200 | [diff] [blame] | 4251 | if (task_cpu(p) != cpu) { |
Peter Zijlstra | ec618b8 | 2020-09-24 13:50:42 +0200 | [diff] [blame] | 4252 | if (p->in_iowait) { |
| 4253 | delayacct_blkio_end(p); |
| 4254 | atomic_dec(&task_rq(p)->nr_iowait); |
| 4255 | } |
| 4256 | |
Peter Zijlstra | f339b9d | 2011-05-31 10:49:20 +0200 | [diff] [blame] | 4257 | wake_flags |= WF_MIGRATED; |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 4258 | psi_ttwu_dequeue(p); |
Mike Galbraith | 055a008 | 2009-11-12 11:07:44 +0100 | [diff] [blame] | 4259 | set_task_cpu(p, cpu); |
Peter Zijlstra | f339b9d | 2011-05-31 10:49:20 +0200 | [diff] [blame] | 4260 | } |
Peter Zijlstra | b6e13e8 | 2020-06-22 12:01:23 +0200 | [diff] [blame] | 4261 | #else |
| 4262 | cpu = task_cpu(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4263 | #endif /* CONFIG_SMP */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4264 | |
Peter Zijlstra | b5179ac | 2016-05-11 16:10:34 +0200 | [diff] [blame] | 4265 | ttwu_queue(p, cpu, wake_flags); |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4266 | unlock: |
Peter Zijlstra | 013fdb8 | 2011-04-05 17:23:45 +0200 | [diff] [blame] | 4267 | raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
Peter Zijlstra | aacedf2 | 2019-06-07 15:39:49 +0200 | [diff] [blame] | 4268 | out: |
| 4269 | if (success) |
Peter Zijlstra | b6e13e8 | 2020-06-22 12:01:23 +0200 | [diff] [blame] | 4270 | ttwu_stat(p, task_cpu(p), wake_flags); |
Peter Zijlstra | e3d8548 | 2019-07-10 12:57:36 +0200 | [diff] [blame] | 4271 | preempt_enable(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4272 | |
| 4273 | return success; |
| 4274 | } |
| 4275 | |
Peter Zijlstra | 91dabf3 | 2022-10-26 13:43:00 +0200 | [diff] [blame] | 4276 | static bool __task_needs_rq_lock(struct task_struct *p) |
| 4277 | { |
| 4278 | unsigned int state = READ_ONCE(p->__state); |
| 4279 | |
| 4280 | /* |
| 4281 | * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when |
| 4282 | * the task is blocked. Make sure to check @state since ttwu() can drop |
| 4283 | * locks at the end, see ttwu_queue_wakelist(). |
| 4284 | */ |
| 4285 | if (state == TASK_RUNNING || state == TASK_WAKING) |
| 4286 | return true; |
| 4287 | |
| 4288 | /* |
| 4289 | * Ensure we load p->on_rq after p->__state, otherwise it would be |
| 4290 | * possible to, falsely, observe p->on_rq == 0. |
| 4291 | * |
| 4292 | * See try_to_wake_up() for a longer comment. |
| 4293 | */ |
| 4294 | smp_rmb(); |
| 4295 | if (p->on_rq) |
| 4296 | return true; |
| 4297 | |
| 4298 | #ifdef CONFIG_SMP |
| 4299 | /* |
| 4300 | * Ensure the task has finished __schedule() and will not be referenced |
| 4301 | * anymore. Again, see try_to_wake_up() for a longer comment. |
| 4302 | */ |
| 4303 | smp_rmb(); |
| 4304 | smp_cond_load_acquire(&p->on_cpu, !VAL); |
| 4305 | #endif |
| 4306 | |
| 4307 | return false; |
| 4308 | } |
| 4309 | |
David Howells | 50fa610 | 2009-04-28 15:01:38 +0100 | [diff] [blame] | 4310 | /** |
Peter Zijlstra | 9b3c4ab | 2021-09-21 21:54:32 +0200 | [diff] [blame] | 4311 | * task_call_func - Invoke a function on task in fixed state |
Peter Zijlstra | 1b7af29 | 2020-08-29 10:22:24 -0700 | [diff] [blame] | 4312 | * @p: Process for which the function is to be invoked, can be @current. |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4313 | * @func: Function to invoke. |
| 4314 | * @arg: Argument to function. |
| 4315 | * |
Peter Zijlstra | f6ac18f | 2021-09-22 10:14:15 +0200 | [diff] [blame] | 4316 | * Fix the task in it's current state by avoiding wakeups and or rq operations |
| 4317 | * and call @func(@arg) on it. This function can use ->on_rq and task_curr() |
| 4318 | * to work out what the state is, if required. Given that @func can be invoked |
| 4319 | * with a runqueue lock held, it had better be quite lightweight. |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4320 | * |
| 4321 | * Returns: |
Peter Zijlstra | f6ac18f | 2021-09-22 10:14:15 +0200 | [diff] [blame] | 4322 | * Whatever @func returns |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4323 | */ |
Peter Zijlstra | 9b3c4ab | 2021-09-21 21:54:32 +0200 | [diff] [blame] | 4324 | int task_call_func(struct task_struct *p, task_call_f func, void *arg) |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4325 | { |
Peter Zijlstra | f6ac18f | 2021-09-22 10:14:15 +0200 | [diff] [blame] | 4326 | struct rq *rq = NULL; |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4327 | struct rq_flags rf; |
Peter Zijlstra | 9b3c4ab | 2021-09-21 21:54:32 +0200 | [diff] [blame] | 4328 | int ret; |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4329 | |
Peter Zijlstra | 1b7af29 | 2020-08-29 10:22:24 -0700 | [diff] [blame] | 4330 | raw_spin_lock_irqsave(&p->pi_lock, rf.flags); |
Peter Zijlstra | f6ac18f | 2021-09-22 10:14:15 +0200 | [diff] [blame] | 4331 | |
Peter Zijlstra | 91dabf3 | 2022-10-26 13:43:00 +0200 | [diff] [blame] | 4332 | if (__task_needs_rq_lock(p)) |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4333 | rq = __task_rq_lock(p, &rf); |
Peter Zijlstra | f6ac18f | 2021-09-22 10:14:15 +0200 | [diff] [blame] | 4334 | |
| 4335 | /* |
| 4336 | * At this point the task is pinned; either: |
| 4337 | * - blocked and we're holding off wakeups (pi->lock) |
| 4338 | * - woken, and we're holding off enqueue (rq->lock) |
| 4339 | * - queued, and we're holding off schedule (rq->lock) |
| 4340 | * - running, and we're holding off de-schedule (rq->lock) |
| 4341 | * |
| 4342 | * The called function (@func) can use: task_curr(), p->on_rq and |
| 4343 | * p->__state to differentiate between these states. |
| 4344 | */ |
| 4345 | ret = func(p, arg); |
| 4346 | |
| 4347 | if (rq) |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4348 | rq_unlock(rq, &rf); |
Peter Zijlstra | f6ac18f | 2021-09-22 10:14:15 +0200 | [diff] [blame] | 4349 | |
Peter Zijlstra | 1b7af29 | 2020-08-29 10:22:24 -0700 | [diff] [blame] | 4350 | raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); |
Paul E. McKenney | 2beaf32 | 2020-03-11 14:23:21 -0700 | [diff] [blame] | 4351 | return ret; |
| 4352 | } |
| 4353 | |
| 4354 | /** |
Paul E. McKenney | e386b67 | 2022-06-02 17:30:01 -0700 | [diff] [blame] | 4355 | * cpu_curr_snapshot - Return a snapshot of the currently running task |
| 4356 | * @cpu: The CPU on which to snapshot the task. |
| 4357 | * |
| 4358 | * Returns the task_struct pointer of the task "currently" running on |
| 4359 | * the specified CPU. If the same task is running on that CPU throughout, |
| 4360 | * the return value will be a pointer to that task's task_struct structure. |
| 4361 | * If the CPU did any context switches even vaguely concurrently with the |
| 4362 | * execution of this function, the return value will be a pointer to the |
| 4363 | * task_struct structure of a randomly chosen task that was running on |
| 4364 | * that CPU somewhere around the time that this function was executing. |
| 4365 | * |
| 4366 | * If the specified CPU was offline, the return value is whatever it |
| 4367 | * is, perhaps a pointer to the task_struct structure of that CPU's idle |
| 4368 | * task, but there is no guarantee. Callers wishing a useful return |
| 4369 | * value must take some action to ensure that the specified CPU remains |
| 4370 | * online throughout. |
| 4371 | * |
| 4372 | * This function executes full memory barriers before and after fetching |
| 4373 | * the pointer, which permits the caller to confine this function's fetch |
| 4374 | * with respect to the caller's accesses to other shared variables. |
| 4375 | */ |
| 4376 | struct task_struct *cpu_curr_snapshot(int cpu) |
| 4377 | { |
| 4378 | struct task_struct *t; |
| 4379 | |
| 4380 | smp_mb(); /* Pairing determined by caller's synchronization design. */ |
| 4381 | t = rcu_dereference(cpu_curr(cpu)); |
| 4382 | smp_mb(); /* Pairing determined by caller's synchronization design. */ |
| 4383 | return t; |
| 4384 | } |
| 4385 | |
| 4386 | /** |
David Howells | 50fa610 | 2009-04-28 15:01:38 +0100 | [diff] [blame] | 4387 | * wake_up_process - Wake up a specific process |
| 4388 | * @p: The process to be woken up. |
| 4389 | * |
| 4390 | * Attempt to wake up the nominated process and move it to the set of runnable |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 4391 | * processes. |
| 4392 | * |
| 4393 | * Return: 1 if the process was woken up, 0 if it was already running. |
David Howells | 50fa610 | 2009-04-28 15:01:38 +0100 | [diff] [blame] | 4394 | * |
Andrea Parri | 7696f99 | 2018-07-16 11:06:03 -0700 | [diff] [blame] | 4395 | * This function executes a full memory barrier before accessing the task state. |
David Howells | 50fa610 | 2009-04-28 15:01:38 +0100 | [diff] [blame] | 4396 | */ |
Harvey Harrison | 7ad5b3a | 2008-02-08 04:19:53 -0800 | [diff] [blame] | 4397 | int wake_up_process(struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4398 | { |
Oleg Nesterov | 9067ac85 | 2013-01-21 20:48:17 +0100 | [diff] [blame] | 4399 | return try_to_wake_up(p, TASK_NORMAL, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4400 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4401 | EXPORT_SYMBOL(wake_up_process); |
| 4402 | |
Harvey Harrison | 7ad5b3a | 2008-02-08 04:19:53 -0800 | [diff] [blame] | 4403 | int wake_up_state(struct task_struct *p, unsigned int state) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4404 | { |
| 4405 | return try_to_wake_up(p, state, 0); |
| 4406 | } |
| 4407 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4408 | /* |
| 4409 | * Perform scheduler related setup for a newly forked process p. |
| 4410 | * p is forked by current. |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 4411 | * |
| 4412 | * __sched_fork() is basic setup used by init_idle() too: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4413 | */ |
Rik van Riel | 5e1576e | 2013-10-07 11:29:26 +0100 | [diff] [blame] | 4414 | static void __sched_fork(unsigned long clone_flags, struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4415 | { |
Peter Zijlstra | fd2f441 | 2011-04-05 17:23:44 +0200 | [diff] [blame] | 4416 | p->on_rq = 0; |
| 4417 | |
| 4418 | p->se.on_rq = 0; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 4419 | p->se.exec_start = 0; |
| 4420 | p->se.sum_exec_runtime = 0; |
Ingo Molnar | f6cf891 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 4421 | p->se.prev_sum_exec_runtime = 0; |
Ingo Molnar | 6c594c2 | 2008-12-14 12:34:15 +0100 | [diff] [blame] | 4422 | p->se.nr_migrations = 0; |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 4423 | p->se.vruntime = 0; |
Peter Zijlstra | fd2f441 | 2011-04-05 17:23:44 +0200 | [diff] [blame] | 4424 | INIT_LIST_HEAD(&p->se.group_node); |
Ingo Molnar | 6cfb0d5 | 2007-08-02 17:41:40 +0200 | [diff] [blame] | 4425 | |
Byungchul Park | ad936d8 | 2015-10-24 01:16:19 +0900 | [diff] [blame] | 4426 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 4427 | p->se.cfs_rq = NULL; |
| 4428 | #endif |
| 4429 | |
Ingo Molnar | 6cfb0d5 | 2007-08-02 17:41:40 +0200 | [diff] [blame] | 4430 | #ifdef CONFIG_SCHEDSTATS |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 4431 | /* Even if schedstat is disabled, there should not be garbage */ |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 4432 | memset(&p->stats, 0, sizeof(p->stats)); |
Ingo Molnar | 6cfb0d5 | 2007-08-02 17:41:40 +0200 | [diff] [blame] | 4433 | #endif |
Nick Piggin | 476d139 | 2005-06-25 14:57:29 -0700 | [diff] [blame] | 4434 | |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 4435 | RB_CLEAR_NODE(&p->dl.rb_node); |
Peter Zijlstra | 40767b0 | 2015-01-28 15:08:03 +0100 | [diff] [blame] | 4436 | init_dl_task_timer(&p->dl); |
Luca Abeni | 209a0cb | 2017-05-18 22:13:29 +0200 | [diff] [blame] | 4437 | init_dl_inactive_task_timer(&p->dl); |
Juri Lelli | a5e7be3 | 2014-09-19 10:22:39 +0100 | [diff] [blame] | 4438 | __dl_clear_params(p); |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 4439 | |
Peter Zijlstra | fa71706 | 2008-01-25 21:08:27 +0100 | [diff] [blame] | 4440 | INIT_LIST_HEAD(&p->rt.run_list); |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 4441 | p->rt.timeout = 0; |
| 4442 | p->rt.time_slice = sched_rr_timeslice; |
| 4443 | p->rt.on_rq = 0; |
| 4444 | p->rt.on_list = 0; |
Nick Piggin | 476d139 | 2005-06-25 14:57:29 -0700 | [diff] [blame] | 4445 | |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4446 | #ifdef CONFIG_PREEMPT_NOTIFIERS |
| 4447 | INIT_HLIST_HEAD(&p->preempt_notifiers); |
| 4448 | #endif |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 4449 | |
Mel Gorman | 5e1f0f0 | 2019-03-05 15:45:41 -0800 | [diff] [blame] | 4450 | #ifdef CONFIG_COMPACTION |
| 4451 | p->capture_control = NULL; |
| 4452 | #endif |
Mel Gorman | 1378447 | 2018-05-04 16:41:09 +0100 | [diff] [blame] | 4453 | init_numa_balancing(clone_flags, p); |
Peter Zijlstra | a1488664 | 2020-05-26 18:11:04 +0200 | [diff] [blame] | 4454 | #ifdef CONFIG_SMP |
Peter Zijlstra | 8c4890d | 2020-06-22 12:01:25 +0200 | [diff] [blame] | 4455 | p->wake_entry.u_flags = CSD_TYPE_TTWU; |
Peter Zijlstra | 6d337ea | 2020-09-18 17:24:31 +0200 | [diff] [blame] | 4456 | p->migration_pending = NULL; |
Peter Zijlstra | a1488664 | 2020-05-26 18:11:04 +0200 | [diff] [blame] | 4457 | #endif |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 4458 | } |
| 4459 | |
Srikar Dronamraju | 2a59572 | 2015-08-11 21:54:21 +0530 | [diff] [blame] | 4460 | DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); |
| 4461 | |
Mel Gorman | 1a687c2 | 2012-11-22 11:16:36 +0000 | [diff] [blame] | 4462 | #ifdef CONFIG_NUMA_BALANCING |
Mel Gorman | 3105b86 | 2012-11-23 11:23:49 +0000 | [diff] [blame] | 4463 | |
Huang Ying | c574bbe | 2022-03-22 14:46:23 -0700 | [diff] [blame] | 4464 | int sysctl_numa_balancing_mode; |
| 4465 | |
| 4466 | static void __set_numabalancing_state(bool enabled) |
Mel Gorman | 3105b86 | 2012-11-23 11:23:49 +0000 | [diff] [blame] | 4467 | { |
Srikar Dronamraju | 2a59572 | 2015-08-11 21:54:21 +0530 | [diff] [blame] | 4468 | if (enabled) |
| 4469 | static_branch_enable(&sched_numa_balancing); |
| 4470 | else |
| 4471 | static_branch_disable(&sched_numa_balancing); |
Mel Gorman | 3105b86 | 2012-11-23 11:23:49 +0000 | [diff] [blame] | 4472 | } |
Andi Kleen | 54a43d5 | 2014-01-23 15:53:13 -0800 | [diff] [blame] | 4473 | |
Huang Ying | c574bbe | 2022-03-22 14:46:23 -0700 | [diff] [blame] | 4474 | void set_numabalancing_state(bool enabled) |
| 4475 | { |
| 4476 | if (enabled) |
| 4477 | sysctl_numa_balancing_mode = NUMA_BALANCING_NORMAL; |
| 4478 | else |
| 4479 | sysctl_numa_balancing_mode = NUMA_BALANCING_DISABLED; |
| 4480 | __set_numabalancing_state(enabled); |
| 4481 | } |
| 4482 | |
Andi Kleen | 54a43d5 | 2014-01-23 15:53:13 -0800 | [diff] [blame] | 4483 | #ifdef CONFIG_PROC_SYSCTL |
Huang Ying | c959924b | 2022-07-13 16:39:53 +0800 | [diff] [blame] | 4484 | static void reset_memory_tiering(void) |
| 4485 | { |
| 4486 | struct pglist_data *pgdat; |
| 4487 | |
| 4488 | for_each_online_pgdat(pgdat) { |
| 4489 | pgdat->nbp_threshold = 0; |
| 4490 | pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); |
| 4491 | pgdat->nbp_th_start = jiffies_to_msecs(jiffies); |
| 4492 | } |
| 4493 | } |
| 4494 | |
Kefeng Wang | 0dff89c | 2022-09-08 20:07:14 +0800 | [diff] [blame] | 4495 | static int sysctl_numa_balancing(struct ctl_table *table, int write, |
Christoph Hellwig | 3292739 | 2020-04-24 08:43:38 +0200 | [diff] [blame] | 4496 | void *buffer, size_t *lenp, loff_t *ppos) |
Andi Kleen | 54a43d5 | 2014-01-23 15:53:13 -0800 | [diff] [blame] | 4497 | { |
| 4498 | struct ctl_table t; |
| 4499 | int err; |
Huang Ying | c574bbe | 2022-03-22 14:46:23 -0700 | [diff] [blame] | 4500 | int state = sysctl_numa_balancing_mode; |
Andi Kleen | 54a43d5 | 2014-01-23 15:53:13 -0800 | [diff] [blame] | 4501 | |
| 4502 | if (write && !capable(CAP_SYS_ADMIN)) |
| 4503 | return -EPERM; |
| 4504 | |
| 4505 | t = *table; |
| 4506 | t.data = &state; |
| 4507 | err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); |
| 4508 | if (err < 0) |
| 4509 | return err; |
Huang Ying | c574bbe | 2022-03-22 14:46:23 -0700 | [diff] [blame] | 4510 | if (write) { |
Huang Ying | c959924b | 2022-07-13 16:39:53 +0800 | [diff] [blame] | 4511 | if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && |
| 4512 | (state & NUMA_BALANCING_MEMORY_TIERING)) |
| 4513 | reset_memory_tiering(); |
Huang Ying | c574bbe | 2022-03-22 14:46:23 -0700 | [diff] [blame] | 4514 | sysctl_numa_balancing_mode = state; |
| 4515 | __set_numabalancing_state(state); |
| 4516 | } |
Andi Kleen | 54a43d5 | 2014-01-23 15:53:13 -0800 | [diff] [blame] | 4517 | return err; |
| 4518 | } |
| 4519 | #endif |
| 4520 | #endif |
Mel Gorman | 1a687c2 | 2012-11-22 11:16:36 +0000 | [diff] [blame] | 4521 | |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 4522 | #ifdef CONFIG_SCHEDSTATS |
Josh Poimboeuf | 4698f88 | 2016-06-07 14:43:16 -0500 | [diff] [blame] | 4523 | |
| 4524 | DEFINE_STATIC_KEY_FALSE(sched_schedstats); |
Josh Poimboeuf | 4698f88 | 2016-06-07 14:43:16 -0500 | [diff] [blame] | 4525 | |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 4526 | static void set_schedstats(bool enabled) |
| 4527 | { |
| 4528 | if (enabled) |
| 4529 | static_branch_enable(&sched_schedstats); |
| 4530 | else |
| 4531 | static_branch_disable(&sched_schedstats); |
| 4532 | } |
| 4533 | |
| 4534 | void force_schedstat_enabled(void) |
| 4535 | { |
| 4536 | if (!schedstat_enabled()) { |
| 4537 | pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); |
| 4538 | static_branch_enable(&sched_schedstats); |
| 4539 | } |
| 4540 | } |
| 4541 | |
| 4542 | static int __init setup_schedstats(char *str) |
| 4543 | { |
| 4544 | int ret = 0; |
| 4545 | if (!str) |
| 4546 | goto out; |
| 4547 | |
| 4548 | if (!strcmp(str, "enable")) { |
Eric Dumazet | 1faa491 | 2021-06-02 04:21:08 -0700 | [diff] [blame] | 4549 | set_schedstats(true); |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 4550 | ret = 1; |
| 4551 | } else if (!strcmp(str, "disable")) { |
Eric Dumazet | 1faa491 | 2021-06-02 04:21:08 -0700 | [diff] [blame] | 4552 | set_schedstats(false); |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 4553 | ret = 1; |
| 4554 | } |
| 4555 | out: |
| 4556 | if (!ret) |
| 4557 | pr_warn("Unable to parse schedstats=\n"); |
| 4558 | |
| 4559 | return ret; |
| 4560 | } |
| 4561 | __setup("schedstats=", setup_schedstats); |
| 4562 | |
| 4563 | #ifdef CONFIG_PROC_SYSCTL |
Zhen Ni | f5ef06d | 2022-02-15 19:45:58 +0800 | [diff] [blame] | 4564 | static int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, |
Christoph Hellwig | 3292739 | 2020-04-24 08:43:38 +0200 | [diff] [blame] | 4565 | size_t *lenp, loff_t *ppos) |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 4566 | { |
| 4567 | struct ctl_table t; |
| 4568 | int err; |
| 4569 | int state = static_branch_likely(&sched_schedstats); |
| 4570 | |
| 4571 | if (write && !capable(CAP_SYS_ADMIN)) |
| 4572 | return -EPERM; |
| 4573 | |
| 4574 | t = *table; |
| 4575 | t.data = &state; |
| 4576 | err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); |
| 4577 | if (err < 0) |
| 4578 | return err; |
| 4579 | if (write) |
| 4580 | set_schedstats(state); |
| 4581 | return err; |
| 4582 | } |
Josh Poimboeuf | 4698f88 | 2016-06-07 14:43:16 -0500 | [diff] [blame] | 4583 | #endif /* CONFIG_PROC_SYSCTL */ |
Josh Poimboeuf | 4698f88 | 2016-06-07 14:43:16 -0500 | [diff] [blame] | 4584 | #endif /* CONFIG_SCHEDSTATS */ |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 4585 | |
Zhen Ni | 3267e01 | 2022-02-15 19:46:02 +0800 | [diff] [blame] | 4586 | #ifdef CONFIG_SYSCTL |
| 4587 | static struct ctl_table sched_core_sysctls[] = { |
| 4588 | #ifdef CONFIG_SCHEDSTATS |
Zhen Ni | f5ef06d | 2022-02-15 19:45:58 +0800 | [diff] [blame] | 4589 | { |
| 4590 | .procname = "sched_schedstats", |
| 4591 | .data = NULL, |
| 4592 | .maxlen = sizeof(unsigned int), |
| 4593 | .mode = 0644, |
| 4594 | .proc_handler = sysctl_schedstats, |
| 4595 | .extra1 = SYSCTL_ZERO, |
| 4596 | .extra2 = SYSCTL_ONE, |
| 4597 | }, |
Zhen Ni | 3267e01 | 2022-02-15 19:46:02 +0800 | [diff] [blame] | 4598 | #endif /* CONFIG_SCHEDSTATS */ |
| 4599 | #ifdef CONFIG_UCLAMP_TASK |
| 4600 | { |
| 4601 | .procname = "sched_util_clamp_min", |
| 4602 | .data = &sysctl_sched_uclamp_util_min, |
| 4603 | .maxlen = sizeof(unsigned int), |
| 4604 | .mode = 0644, |
| 4605 | .proc_handler = sysctl_sched_uclamp_handler, |
| 4606 | }, |
| 4607 | { |
| 4608 | .procname = "sched_util_clamp_max", |
| 4609 | .data = &sysctl_sched_uclamp_util_max, |
| 4610 | .maxlen = sizeof(unsigned int), |
| 4611 | .mode = 0644, |
| 4612 | .proc_handler = sysctl_sched_uclamp_handler, |
| 4613 | }, |
| 4614 | { |
| 4615 | .procname = "sched_util_clamp_min_rt_default", |
| 4616 | .data = &sysctl_sched_uclamp_util_min_rt_default, |
| 4617 | .maxlen = sizeof(unsigned int), |
| 4618 | .mode = 0644, |
| 4619 | .proc_handler = sysctl_sched_uclamp_handler, |
| 4620 | }, |
| 4621 | #endif /* CONFIG_UCLAMP_TASK */ |
Kefeng Wang | 0dff89c | 2022-09-08 20:07:14 +0800 | [diff] [blame] | 4622 | #ifdef CONFIG_NUMA_BALANCING |
| 4623 | { |
| 4624 | .procname = "numa_balancing", |
| 4625 | .data = NULL, /* filled in by handler */ |
| 4626 | .maxlen = sizeof(unsigned int), |
| 4627 | .mode = 0644, |
| 4628 | .proc_handler = sysctl_numa_balancing, |
| 4629 | .extra1 = SYSCTL_ZERO, |
| 4630 | .extra2 = SYSCTL_FOUR, |
| 4631 | }, |
| 4632 | #endif /* CONFIG_NUMA_BALANCING */ |
Zhen Ni | f5ef06d | 2022-02-15 19:45:58 +0800 | [diff] [blame] | 4633 | {} |
| 4634 | }; |
Zhen Ni | 3267e01 | 2022-02-15 19:46:02 +0800 | [diff] [blame] | 4635 | static int __init sched_core_sysctl_init(void) |
Zhen Ni | f5ef06d | 2022-02-15 19:45:58 +0800 | [diff] [blame] | 4636 | { |
Zhen Ni | 3267e01 | 2022-02-15 19:46:02 +0800 | [diff] [blame] | 4637 | register_sysctl_init("kernel", sched_core_sysctls); |
Zhen Ni | f5ef06d | 2022-02-15 19:45:58 +0800 | [diff] [blame] | 4638 | return 0; |
| 4639 | } |
Zhen Ni | 3267e01 | 2022-02-15 19:46:02 +0800 | [diff] [blame] | 4640 | late_initcall(sched_core_sysctl_init); |
| 4641 | #endif /* CONFIG_SYSCTL */ |
Mike Galbraith | b9dc29e | 2009-06-17 10:46:01 +0200 | [diff] [blame] | 4642 | |
Lennart Poettering | ca94c44 | 2009-06-15 17:17:47 +0200 | [diff] [blame] | 4643 | /* |
Hiroshi Shimamoto | 2ddbf95 | 2007-10-15 17:00:11 +0200 | [diff] [blame] | 4644 | * fork()/clone()-time setup: |
| 4645 | */ |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 4646 | int sched_fork(unsigned long clone_flags, struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4647 | { |
Rik van Riel | 5e1576e | 2013-10-07 11:29:26 +0100 | [diff] [blame] | 4648 | __sched_fork(clone_flags, p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4649 | /* |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 4650 | * We mark the process as NEW here. This guarantees that |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4651 | * nobody will actually run it, and a signal or other external |
| 4652 | * event cannot wake it up and insert it on the runqueue either. |
| 4653 | */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 4654 | p->__state = TASK_NEW; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4655 | |
| 4656 | /* |
| 4657 | * Make sure we do not leak PI boosting priority to the child. |
| 4658 | */ |
| 4659 | p->prio = current->normal_prio; |
| 4660 | |
Patrick Bellasi | e8f1417 | 2019-06-21 09:42:05 +0100 | [diff] [blame] | 4661 | uclamp_fork(p); |
| 4662 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4663 | /* |
| 4664 | * Revert to default priority/policy on fork if requested. |
| 4665 | */ |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 4666 | if (unlikely(p->sched_reset_on_fork)) { |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 4667 | if (task_has_dl_policy(p) || task_has_rt_policy(p)) { |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 4668 | p->policy = SCHED_NORMAL; |
| 4669 | p->static_prio = NICE_TO_PRIO(0); |
| 4670 | p->rt_priority = 0; |
| 4671 | } else if (PRIO_TO_NICE(p->static_prio) < 0) |
| 4672 | p->static_prio = NICE_TO_PRIO(0); |
| 4673 | |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 4674 | p->prio = p->normal_prio = p->static_prio; |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 4675 | set_load_weight(p, false); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 4676 | |
| 4677 | /* |
| 4678 | * We don't need the reset flag anymore after the fork. It has |
| 4679 | * fulfilled its duty: |
Ingo Molnar | 02e4bac2 | 2007-10-15 17:00:11 +0200 | [diff] [blame] | 4680 | */ |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 4681 | p->sched_reset_on_fork = 0; |
| 4682 | } |
| 4683 | |
Sebastian Andrzej Siewior | af0fffd | 2018-07-06 15:06:15 +0200 | [diff] [blame] | 4684 | if (dl_prio(p->prio)) |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 4685 | return -EAGAIN; |
Sebastian Andrzej Siewior | af0fffd | 2018-07-06 15:06:15 +0200 | [diff] [blame] | 4686 | else if (rt_prio(p->prio)) |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 4687 | p->sched_class = &rt_sched_class; |
Sebastian Andrzej Siewior | af0fffd | 2018-07-06 15:06:15 +0200 | [diff] [blame] | 4688 | else |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 4689 | p->sched_class = &fair_sched_class; |
| 4690 | |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 4691 | init_entity_runnable_average(&p->se); |
Peter Zijlstra | cd29fe6 | 2009-11-27 17:32:46 +0100 | [diff] [blame] | 4692 | |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 4693 | |
Naveen N. Rao | f6db834 | 2015-06-25 23:53:37 +0530 | [diff] [blame] | 4694 | #ifdef CONFIG_SCHED_INFO |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 4695 | if (likely(sched_info_on())) |
Chandra Seetharaman | 52f17b6 | 2006-07-14 00:24:38 -0700 | [diff] [blame] | 4696 | memset(&p->sched_info, 0, sizeof(p->sched_info)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4697 | #endif |
Peter Zijlstra | 3ca7a44 | 2011-04-05 17:23:40 +0200 | [diff] [blame] | 4698 | #if defined(CONFIG_SMP) |
| 4699 | p->on_cpu = 0; |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 4700 | #endif |
Peter Zijlstra | 0102874 | 2013-08-14 14:55:46 +0200 | [diff] [blame] | 4701 | init_task_preempt_count(p); |
Dario Faggioli | 806c09a | 2010-11-30 19:51:33 +0100 | [diff] [blame] | 4702 | #ifdef CONFIG_SMP |
Gregory Haskins | 917b627 | 2008-12-29 09:39:53 -0500 | [diff] [blame] | 4703 | plist_node_init(&p->pushable_tasks, MAX_PRIO); |
Juri Lelli | 1baca4c | 2013-11-07 14:43:38 +0100 | [diff] [blame] | 4704 | RB_CLEAR_NODE(&p->pushable_dl_tasks); |
Dario Faggioli | 806c09a | 2010-11-30 19:51:33 +0100 | [diff] [blame] | 4705 | #endif |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 4706 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4707 | } |
| 4708 | |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 4709 | void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 4710 | { |
Zhang Qiao | 4ef0c5c | 2021-09-15 14:40:30 +0800 | [diff] [blame] | 4711 | unsigned long flags; |
Zhang Qiao | 4ef0c5c | 2021-09-15 14:40:30 +0800 | [diff] [blame] | 4712 | |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 4713 | /* |
| 4714 | * Because we're not yet on the pid-hash, p->pi_lock isn't strictly |
| 4715 | * required yet, but lockdep gets upset if rules are violated. |
| 4716 | */ |
Zhang Qiao | 4ef0c5c | 2021-09-15 14:40:30 +0800 | [diff] [blame] | 4717 | raw_spin_lock_irqsave(&p->pi_lock, flags); |
| 4718 | #ifdef CONFIG_CGROUP_SCHED |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 4719 | if (1) { |
| 4720 | struct task_group *tg; |
| 4721 | tg = container_of(kargs->cset->subsys[cpu_cgrp_id], |
| 4722 | struct task_group, css); |
| 4723 | tg = autogroup_task_group(p, tg); |
| 4724 | p->sched_task_group = tg; |
| 4725 | } |
Zhang Qiao | 4ef0c5c | 2021-09-15 14:40:30 +0800 | [diff] [blame] | 4726 | #endif |
| 4727 | rseq_migrate(p); |
| 4728 | /* |
| 4729 | * We're setting the CPU for the first time, we don't migrate, |
| 4730 | * so use __set_task_cpu(). |
| 4731 | */ |
| 4732 | __set_task_cpu(p, smp_processor_id()); |
| 4733 | if (p->sched_class->task_fork) |
| 4734 | p->sched_class->task_fork(p); |
| 4735 | raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 4736 | } |
Zhang Qiao | 4ef0c5c | 2021-09-15 14:40:30 +0800 | [diff] [blame] | 4737 | |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 4738 | void sched_post_fork(struct task_struct *p) |
| 4739 | { |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 4740 | uclamp_post_fork(p); |
| 4741 | } |
| 4742 | |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 4743 | unsigned long to_ratio(u64 period, u64 runtime) |
| 4744 | { |
| 4745 | if (runtime == RUNTIME_INF) |
Luca Abeni | c52f14d | 2017-05-18 22:13:31 +0200 | [diff] [blame] | 4746 | return BW_UNIT; |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 4747 | |
| 4748 | /* |
| 4749 | * Doing this here saves a lot of checks in all |
| 4750 | * the calling paths, and returning zero seems |
| 4751 | * safe for them anyway. |
| 4752 | */ |
| 4753 | if (period == 0) |
| 4754 | return 0; |
| 4755 | |
Luca Abeni | c52f14d | 2017-05-18 22:13:31 +0200 | [diff] [blame] | 4756 | return div64_u64(runtime << BW_SHIFT, period); |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 4757 | } |
| 4758 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4759 | /* |
| 4760 | * wake_up_new_task - wake up a newly created task for the first time. |
| 4761 | * |
| 4762 | * This function will do some initial scheduler statistics housekeeping |
| 4763 | * that must be done for every newly created context, then puts the task |
| 4764 | * on the runqueue and wakes it. |
| 4765 | */ |
Samir Bellabes | 3e51e3e | 2011-05-11 18:18:05 +0200 | [diff] [blame] | 4766 | void wake_up_new_task(struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4767 | { |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 4768 | struct rq_flags rf; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 4769 | struct rq *rq; |
Peter Zijlstra | fabf318 | 2010-01-21 21:04:57 +0100 | [diff] [blame] | 4770 | |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 4771 | raw_spin_lock_irqsave(&p->pi_lock, rf.flags); |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 4772 | WRITE_ONCE(p->__state, TASK_RUNNING); |
Peter Zijlstra | fabf318 | 2010-01-21 21:04:57 +0100 | [diff] [blame] | 4773 | #ifdef CONFIG_SMP |
| 4774 | /* |
| 4775 | * Fork balancing, do it here and not earlier because: |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 4776 | * - cpus_ptr can change in the fork path |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 4777 | * - any previously selected CPU might disappear through hotplug |
Peter Zijlstra | e210bff | 2016-06-16 18:51:48 +0200 | [diff] [blame] | 4778 | * |
| 4779 | * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, |
| 4780 | * as we're not fully set-up yet. |
Peter Zijlstra | fabf318 | 2010-01-21 21:04:57 +0100 | [diff] [blame] | 4781 | */ |
Mel Gorman | 32e839d | 2018-01-30 10:45:55 +0000 | [diff] [blame] | 4782 | p->recent_used_cpu = task_cpu(p); |
Mathieu Desnoyers | ce3614d | 2020-07-06 16:49:10 -0400 | [diff] [blame] | 4783 | rseq_migrate(p); |
Valentin Schneider | 3aef1551 | 2020-11-02 18:45:13 +0000 | [diff] [blame] | 4784 | __set_task_cpu(p, select_task_rq(p, task_cpu(p), WF_FORK)); |
Peter Zijlstra | fabf318 | 2010-01-21 21:04:57 +0100 | [diff] [blame] | 4785 | #endif |
Peter Zijlstra | b7fa30c | 2016-06-09 15:07:50 +0200 | [diff] [blame] | 4786 | rq = __task_rq_lock(p, &rf); |
Peter Zijlstra | 4126bad | 2016-10-03 16:20:59 +0200 | [diff] [blame] | 4787 | update_rq_clock(rq); |
Dietmar Eggemann | d0fe0b9 | 2019-01-22 16:25:01 +0000 | [diff] [blame] | 4788 | post_init_entity_util_avg(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4789 | |
Peter Zijlstra | 7a57f32 | 2017-02-21 14:47:02 +0100 | [diff] [blame] | 4790 | activate_task(rq, p, ENQUEUE_NOCLOCK); |
Peter Zijlstra | fbd705a | 2015-06-09 11:13:36 +0200 | [diff] [blame] | 4791 | trace_sched_wakeup_new(p); |
Peter Zijlstra | a7558e0 | 2009-09-14 20:02:34 +0200 | [diff] [blame] | 4792 | check_preempt_curr(rq, p, WF_FORK); |
Steven Rostedt | 9a897c5 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 4793 | #ifdef CONFIG_SMP |
Peter Zijlstra | 0aaafaa | 2015-10-23 11:50:08 +0200 | [diff] [blame] | 4794 | if (p->sched_class->task_woken) { |
| 4795 | /* |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 4796 | * Nothing relies on rq->lock after this, so it's fine to |
Peter Zijlstra | 0aaafaa | 2015-10-23 11:50:08 +0200 | [diff] [blame] | 4797 | * drop it. |
| 4798 | */ |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 4799 | rq_unpin_lock(rq, &rf); |
Peter Zijlstra | efbbd05 | 2009-12-16 18:04:40 +0100 | [diff] [blame] | 4800 | p->sched_class->task_woken(rq, p); |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 4801 | rq_repin_lock(rq, &rf); |
Peter Zijlstra | 0aaafaa | 2015-10-23 11:50:08 +0200 | [diff] [blame] | 4802 | } |
Steven Rostedt | 9a897c5 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 4803 | #endif |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 4804 | task_rq_unlock(rq, p, &rf); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4805 | } |
| 4806 | |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4807 | #ifdef CONFIG_PREEMPT_NOTIFIERS |
| 4808 | |
Davidlohr Bueso | b720342 | 2018-03-26 14:09:26 -0700 | [diff] [blame] | 4809 | static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4810 | |
Peter Zijlstra | 2ecd9d2 | 2015-07-03 18:53:58 +0200 | [diff] [blame] | 4811 | void preempt_notifier_inc(void) |
| 4812 | { |
Davidlohr Bueso | b720342 | 2018-03-26 14:09:26 -0700 | [diff] [blame] | 4813 | static_branch_inc(&preempt_notifier_key); |
Peter Zijlstra | 2ecd9d2 | 2015-07-03 18:53:58 +0200 | [diff] [blame] | 4814 | } |
| 4815 | EXPORT_SYMBOL_GPL(preempt_notifier_inc); |
| 4816 | |
| 4817 | void preempt_notifier_dec(void) |
| 4818 | { |
Davidlohr Bueso | b720342 | 2018-03-26 14:09:26 -0700 | [diff] [blame] | 4819 | static_branch_dec(&preempt_notifier_key); |
Peter Zijlstra | 2ecd9d2 | 2015-07-03 18:53:58 +0200 | [diff] [blame] | 4820 | } |
| 4821 | EXPORT_SYMBOL_GPL(preempt_notifier_dec); |
| 4822 | |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4823 | /** |
Luis Henriques | 80dd99b | 2009-03-16 19:58:09 +0000 | [diff] [blame] | 4824 | * preempt_notifier_register - tell me when current is being preempted & rescheduled |
Randy Dunlap | 421cee2 | 2007-07-31 00:37:50 -0700 | [diff] [blame] | 4825 | * @notifier: notifier struct to register |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4826 | */ |
| 4827 | void preempt_notifier_register(struct preempt_notifier *notifier) |
| 4828 | { |
Davidlohr Bueso | b720342 | 2018-03-26 14:09:26 -0700 | [diff] [blame] | 4829 | if (!static_branch_unlikely(&preempt_notifier_key)) |
Peter Zijlstra | 2ecd9d2 | 2015-07-03 18:53:58 +0200 | [diff] [blame] | 4830 | WARN(1, "registering preempt_notifier while notifiers disabled\n"); |
| 4831 | |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4832 | hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); |
| 4833 | } |
| 4834 | EXPORT_SYMBOL_GPL(preempt_notifier_register); |
| 4835 | |
| 4836 | /** |
| 4837 | * preempt_notifier_unregister - no longer interested in preemption notifications |
Randy Dunlap | 421cee2 | 2007-07-31 00:37:50 -0700 | [diff] [blame] | 4838 | * @notifier: notifier struct to unregister |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4839 | * |
Mathieu Desnoyers | d84525a | 2015-05-17 12:53:10 -0400 | [diff] [blame] | 4840 | * This is *not* safe to call from within a preemption notifier. |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4841 | */ |
| 4842 | void preempt_notifier_unregister(struct preempt_notifier *notifier) |
| 4843 | { |
| 4844 | hlist_del(¬ifier->link); |
| 4845 | } |
| 4846 | EXPORT_SYMBOL_GPL(preempt_notifier_unregister); |
| 4847 | |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4848 | static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4849 | { |
| 4850 | struct preempt_notifier *notifier; |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4851 | |
Sasha Levin | b67bfe0 | 2013-02-27 17:06:00 -0800 | [diff] [blame] | 4852 | hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4853 | notifier->ops->sched_in(notifier, raw_smp_processor_id()); |
| 4854 | } |
| 4855 | |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4856 | static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) |
| 4857 | { |
Davidlohr Bueso | b720342 | 2018-03-26 14:09:26 -0700 | [diff] [blame] | 4858 | if (static_branch_unlikely(&preempt_notifier_key)) |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4859 | __fire_sched_in_preempt_notifiers(curr); |
| 4860 | } |
| 4861 | |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4862 | static void |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4863 | __fire_sched_out_preempt_notifiers(struct task_struct *curr, |
| 4864 | struct task_struct *next) |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4865 | { |
| 4866 | struct preempt_notifier *notifier; |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4867 | |
Sasha Levin | b67bfe0 | 2013-02-27 17:06:00 -0800 | [diff] [blame] | 4868 | hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4869 | notifier->ops->sched_out(notifier, next); |
| 4870 | } |
| 4871 | |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4872 | static __always_inline void |
| 4873 | fire_sched_out_preempt_notifiers(struct task_struct *curr, |
| 4874 | struct task_struct *next) |
| 4875 | { |
Davidlohr Bueso | b720342 | 2018-03-26 14:09:26 -0700 | [diff] [blame] | 4876 | if (static_branch_unlikely(&preempt_notifier_key)) |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4877 | __fire_sched_out_preempt_notifiers(curr, next); |
| 4878 | } |
| 4879 | |
Dhaval Giani | 6d6bc0a | 2008-05-30 14:23:45 +0200 | [diff] [blame] | 4880 | #else /* !CONFIG_PREEMPT_NOTIFIERS */ |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4881 | |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4882 | static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4883 | { |
| 4884 | } |
| 4885 | |
Peter Zijlstra | 1cde293 | 2015-06-08 16:00:30 +0200 | [diff] [blame] | 4886 | static inline void |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4887 | fire_sched_out_preempt_notifiers(struct task_struct *curr, |
| 4888 | struct task_struct *next) |
| 4889 | { |
| 4890 | } |
| 4891 | |
Dhaval Giani | 6d6bc0a | 2008-05-30 14:23:45 +0200 | [diff] [blame] | 4892 | #endif /* CONFIG_PREEMPT_NOTIFIERS */ |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 4893 | |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 4894 | static inline void prepare_task(struct task_struct *next) |
| 4895 | { |
| 4896 | #ifdef CONFIG_SMP |
| 4897 | /* |
| 4898 | * Claim the task as running, we do this before switching to it |
| 4899 | * such that any running task will have this set. |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4900 | * |
Tianchen Ding | f3dd3f6 | 2022-06-09 07:34:12 +0800 | [diff] [blame] | 4901 | * See the smp_load_acquire(&p->on_cpu) case in ttwu() and |
| 4902 | * its ordering comment. |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 4903 | */ |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4904 | WRITE_ONCE(next->on_cpu, 1); |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 4905 | #endif |
| 4906 | } |
| 4907 | |
| 4908 | static inline void finish_task(struct task_struct *prev) |
| 4909 | { |
| 4910 | #ifdef CONFIG_SMP |
| 4911 | /* |
Peter Zijlstra | 58877d3 | 2020-07-02 14:52:11 +0200 | [diff] [blame] | 4912 | * This must be the very last reference to @prev from this CPU. After |
| 4913 | * p->on_cpu is cleared, the task can be moved to a different CPU. We |
| 4914 | * must ensure this doesn't happen until the switch is completely |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 4915 | * finished. |
| 4916 | * |
| 4917 | * In particular, the load of prev->state in finish_task_switch() must |
| 4918 | * happen before this. |
| 4919 | * |
| 4920 | * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). |
| 4921 | */ |
| 4922 | smp_store_release(&prev->on_cpu, 0); |
| 4923 | #endif |
| 4924 | } |
| 4925 | |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4926 | #ifdef CONFIG_SMP |
| 4927 | |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 4928 | static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4929 | { |
| 4930 | void (*func)(struct rq *rq); |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 4931 | struct balance_callback *next; |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4932 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 4933 | lockdep_assert_rq_held(rq); |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4934 | |
| 4935 | while (head) { |
| 4936 | func = (void (*)(struct rq *))head->func; |
| 4937 | next = head->next; |
| 4938 | head->next = NULL; |
| 4939 | head = next; |
| 4940 | |
| 4941 | func(rq); |
| 4942 | } |
| 4943 | } |
| 4944 | |
Peter Zijlstra | ae79270 | 2020-12-10 17:14:08 +0100 | [diff] [blame] | 4945 | static void balance_push(struct rq *rq); |
| 4946 | |
Peter Zijlstra | 04193d5 | 2022-06-07 22:41:55 +0200 | [diff] [blame] | 4947 | /* |
| 4948 | * balance_push_callback is a right abuse of the callback interface and plays |
| 4949 | * by significantly different rules. |
| 4950 | * |
| 4951 | * Where the normal balance_callback's purpose is to be ran in the same context |
| 4952 | * that queued it (only later, when it's safe to drop rq->lock again), |
| 4953 | * balance_push_callback is specifically targeted at __schedule(). |
| 4954 | * |
| 4955 | * This abuse is tolerated because it places all the unlikely/odd cases behind |
| 4956 | * a single test, namely: rq->balance_callback == NULL. |
| 4957 | */ |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 4958 | struct balance_callback balance_push_callback = { |
Peter Zijlstra | ae79270 | 2020-12-10 17:14:08 +0100 | [diff] [blame] | 4959 | .next = NULL, |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 4960 | .func = balance_push, |
Peter Zijlstra | ae79270 | 2020-12-10 17:14:08 +0100 | [diff] [blame] | 4961 | }; |
| 4962 | |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 4963 | static inline struct balance_callback * |
Peter Zijlstra | 04193d5 | 2022-06-07 22:41:55 +0200 | [diff] [blame] | 4964 | __splice_balance_callbacks(struct rq *rq, bool split) |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4965 | { |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 4966 | struct balance_callback *head = rq->balance_callback; |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4967 | |
Peter Zijlstra | 04193d5 | 2022-06-07 22:41:55 +0200 | [diff] [blame] | 4968 | if (likely(!head)) |
| 4969 | return NULL; |
| 4970 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 4971 | lockdep_assert_rq_held(rq); |
Peter Zijlstra | 04193d5 | 2022-06-07 22:41:55 +0200 | [diff] [blame] | 4972 | /* |
| 4973 | * Must not take balance_push_callback off the list when |
| 4974 | * splice_balance_callbacks() and balance_callbacks() are not |
| 4975 | * in the same rq->lock section. |
| 4976 | * |
| 4977 | * In that case it would be possible for __schedule() to interleave |
| 4978 | * and observe the list empty. |
| 4979 | */ |
| 4980 | if (split && head == &balance_push_callback) |
| 4981 | head = NULL; |
| 4982 | else |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4983 | rq->balance_callback = NULL; |
| 4984 | |
| 4985 | return head; |
| 4986 | } |
| 4987 | |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 4988 | static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) |
Peter Zijlstra | 04193d5 | 2022-06-07 22:41:55 +0200 | [diff] [blame] | 4989 | { |
| 4990 | return __splice_balance_callbacks(rq, true); |
| 4991 | } |
| 4992 | |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4993 | static void __balance_callbacks(struct rq *rq) |
| 4994 | { |
Peter Zijlstra | 04193d5 | 2022-06-07 22:41:55 +0200 | [diff] [blame] | 4995 | do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4996 | } |
| 4997 | |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 4998 | static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 4999 | { |
| 5000 | unsigned long flags; |
| 5001 | |
| 5002 | if (unlikely(head)) { |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 5003 | raw_spin_rq_lock_irqsave(rq, flags); |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 5004 | do_balance_callbacks(rq, head); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 5005 | raw_spin_rq_unlock_irqrestore(rq, flags); |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 5006 | } |
| 5007 | } |
| 5008 | |
| 5009 | #else |
| 5010 | |
| 5011 | static inline void __balance_callbacks(struct rq *rq) |
| 5012 | { |
| 5013 | } |
| 5014 | |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 5015 | static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 5016 | { |
| 5017 | return NULL; |
| 5018 | } |
| 5019 | |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 5020 | static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 5021 | { |
| 5022 | } |
| 5023 | |
| 5024 | #endif |
| 5025 | |
Peter Zijlstra | 269d599 | 2018-02-06 17:52:13 +0100 | [diff] [blame] | 5026 | static inline void |
| 5027 | prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5028 | { |
Peter Zijlstra | 269d599 | 2018-02-06 17:52:13 +0100 | [diff] [blame] | 5029 | /* |
| 5030 | * Since the runqueue lock will be released by the next |
| 5031 | * task (which is an invalid locking op but in the case |
| 5032 | * of the scheduler it's an obvious special-case), so we |
| 5033 | * do an early lockdep release here: |
| 5034 | */ |
| 5035 | rq_unpin_lock(rq, rf); |
Peter Zijlstra | 9ef7e7e | 2021-03-03 16:45:41 +0100 | [diff] [blame] | 5036 | spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5037 | #ifdef CONFIG_DEBUG_SPINLOCK |
| 5038 | /* this is a valid case when another task releases the spinlock */ |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 5039 | rq_lockp(rq)->owner = next; |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5040 | #endif |
Peter Zijlstra | 269d599 | 2018-02-06 17:52:13 +0100 | [diff] [blame] | 5041 | } |
| 5042 | |
| 5043 | static inline void finish_lock_switch(struct rq *rq) |
| 5044 | { |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5045 | /* |
| 5046 | * If we are tracking spinlock dependencies then we have to |
| 5047 | * fix up the runqueue lock - which gets 'carried over' from |
| 5048 | * prev into current: |
| 5049 | */ |
Peter Zijlstra | 9ef7e7e | 2021-03-03 16:45:41 +0100 | [diff] [blame] | 5050 | spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); |
Peter Zijlstra | ae79270 | 2020-12-10 17:14:08 +0100 | [diff] [blame] | 5051 | __balance_callbacks(rq); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 5052 | raw_spin_rq_unlock_irq(rq); |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5053 | } |
| 5054 | |
Ingo Molnar | 325ea10 | 2018-03-03 12:20:47 +0100 | [diff] [blame] | 5055 | /* |
| 5056 | * NOP if the arch has not defined these: |
| 5057 | */ |
| 5058 | |
| 5059 | #ifndef prepare_arch_switch |
| 5060 | # define prepare_arch_switch(next) do { } while (0) |
| 5061 | #endif |
| 5062 | |
| 5063 | #ifndef finish_arch_post_lock_switch |
| 5064 | # define finish_arch_post_lock_switch() do { } while (0) |
| 5065 | #endif |
| 5066 | |
Thomas Gleixner | 5fbda3e | 2020-11-18 20:48:43 +0100 | [diff] [blame] | 5067 | static inline void kmap_local_sched_out(void) |
| 5068 | { |
| 5069 | #ifdef CONFIG_KMAP_LOCAL |
| 5070 | if (unlikely(current->kmap_ctrl.idx)) |
| 5071 | __kmap_local_sched_out(); |
| 5072 | #endif |
| 5073 | } |
| 5074 | |
| 5075 | static inline void kmap_local_sched_in(void) |
| 5076 | { |
| 5077 | #ifdef CONFIG_KMAP_LOCAL |
| 5078 | if (unlikely(current->kmap_ctrl.idx)) |
| 5079 | __kmap_local_sched_in(); |
| 5080 | #endif |
| 5081 | } |
| 5082 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5083 | /** |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 5084 | * prepare_task_switch - prepare to switch tasks |
| 5085 | * @rq: the runqueue preparing to switch |
Randy Dunlap | 421cee2 | 2007-07-31 00:37:50 -0700 | [diff] [blame] | 5086 | * @prev: the current task that is being switched out |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 5087 | * @next: the task we are going to switch to. |
| 5088 | * |
| 5089 | * This is called with the rq lock held and interrupts off. It must |
| 5090 | * be paired with a subsequent finish_task_switch after the context |
| 5091 | * switch. |
| 5092 | * |
| 5093 | * prepare_task_switch sets up locking and calls architecture specific |
| 5094 | * hooks. |
| 5095 | */ |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 5096 | static inline void |
| 5097 | prepare_task_switch(struct rq *rq, struct task_struct *prev, |
| 5098 | struct task_struct *next) |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 5099 | { |
Mark Rutland | 0ed557a | 2018-06-14 15:27:41 -0700 | [diff] [blame] | 5100 | kcov_prepare_switch(prev); |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 5101 | sched_info_switch(rq, prev, next); |
Peter Zijlstra | fe4b04f | 2011-02-02 13:19:09 +0100 | [diff] [blame] | 5102 | perf_event_task_sched_out(prev, next); |
Mathieu Desnoyers | d7822b1 | 2018-06-02 08:43:54 -0400 | [diff] [blame] | 5103 | rseq_preempt(prev); |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 5104 | fire_sched_out_preempt_notifiers(prev, next); |
Thomas Gleixner | 5fbda3e | 2020-11-18 20:48:43 +0100 | [diff] [blame] | 5105 | kmap_local_sched_out(); |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5106 | prepare_task(next); |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 5107 | prepare_arch_switch(next); |
| 5108 | } |
| 5109 | |
| 5110 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5111 | * finish_task_switch - clean up after a task-switch |
| 5112 | * @prev: the thread we just switched away from. |
| 5113 | * |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 5114 | * finish_task_switch must be called after the context switch, paired |
| 5115 | * with a prepare_task_switch call before the context switch. |
| 5116 | * finish_task_switch will reconcile locking set up by prepare_task_switch, |
| 5117 | * and do any other architecture-specific cleanup actions. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5118 | * |
| 5119 | * Note that we may have delayed dropping an mm in context_switch(). If |
Ingo Molnar | 41a2d6c | 2007-12-05 15:46:09 +0100 | [diff] [blame] | 5120 | * so, we finish that here outside of the runqueue lock. (Doing it |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5121 | * with the lock held can cause deadlocks; see schedule() for |
| 5122 | * details.) |
Oleg Nesterov | dfa50b6 | 2014-10-09 21:32:32 +0200 | [diff] [blame] | 5123 | * |
| 5124 | * The context switch have flipped the stack from under us and restored the |
| 5125 | * local variables which were saved when this task called schedule() in the |
| 5126 | * past. prev == current is still correct but we need to recalculate this_rq |
| 5127 | * because prev may have moved to another CPU. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5128 | */ |
Oleg Nesterov | dfa50b6 | 2014-10-09 21:32:32 +0200 | [diff] [blame] | 5129 | static struct rq *finish_task_switch(struct task_struct *prev) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5130 | __releases(rq->lock) |
| 5131 | { |
Oleg Nesterov | dfa50b6 | 2014-10-09 21:32:32 +0200 | [diff] [blame] | 5132 | struct rq *rq = this_rq(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5133 | struct mm_struct *mm = rq->prev_mm; |
Valentin Schneider | fa2c325 | 2022-01-20 16:25:19 +0000 | [diff] [blame] | 5134 | unsigned int prev_state; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5135 | |
Peter Zijlstra | 609ca06 | 2015-09-28 17:52:18 +0200 | [diff] [blame] | 5136 | /* |
| 5137 | * The previous task will have left us with a preempt_count of 2 |
| 5138 | * because it left us after: |
| 5139 | * |
| 5140 | * schedule() |
| 5141 | * preempt_disable(); // 1 |
| 5142 | * __schedule() |
| 5143 | * raw_spin_lock_irq(&rq->lock) // 2 |
| 5144 | * |
| 5145 | * Also, see FORK_PREEMPT_COUNT. |
| 5146 | */ |
Peter Zijlstra | e2bf1c4b | 2015-09-29 12:18:46 +0200 | [diff] [blame] | 5147 | if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, |
| 5148 | "corrupted preempt_count: %s/%d/0x%x\n", |
| 5149 | current->comm, current->pid, preempt_count())) |
| 5150 | preempt_count_set(FORK_PREEMPT_COUNT); |
Peter Zijlstra | 609ca06 | 2015-09-28 17:52:18 +0200 | [diff] [blame] | 5151 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5152 | rq->prev_mm = NULL; |
| 5153 | |
| 5154 | /* |
| 5155 | * A task struct has one reference for the use as "current". |
Oleg Nesterov | c394cc9 | 2006-09-29 02:01:11 -0700 | [diff] [blame] | 5156 | * If a task dies, then it sets TASK_DEAD in tsk->state and calls |
Oleg Nesterov | 55a101f | 2006-09-29 02:01:10 -0700 | [diff] [blame] | 5157 | * schedule one last time. The schedule call will never return, and |
| 5158 | * the scheduled task must drop that reference. |
Peter Zijlstra | 95913d9 | 2015-09-29 14:45:09 +0200 | [diff] [blame] | 5159 | * |
| 5160 | * We must observe prev->state before clearing prev->on_cpu (in |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5161 | * finish_task), otherwise a concurrent wakeup can get prev |
Peter Zijlstra | 95913d9 | 2015-09-29 14:45:09 +0200 | [diff] [blame] | 5162 | * running on another CPU and we could rave with its RUNNING -> DEAD |
| 5163 | * transition, resulting in a double drop. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5164 | */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 5165 | prev_state = READ_ONCE(prev->__state); |
Frederic Weisbecker | bf9fae9 | 2012-09-08 15:23:11 +0200 | [diff] [blame] | 5166 | vtime_task_switch(prev); |
Stephane Eranian | a8d757e | 2011-08-25 15:58:03 +0200 | [diff] [blame] | 5167 | perf_event_task_sched_in(prev, current); |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5168 | finish_task(prev); |
Peter Zijlstra | 0fdcccf | 2021-05-13 01:29:23 +0200 | [diff] [blame] | 5169 | tick_nohz_task_switch(); |
rodrigosiqueira | 31cb1bc | 2017-12-15 12:06:03 -0200 | [diff] [blame] | 5170 | finish_lock_switch(rq); |
Catalin Marinas | 01f23e1 | 2011-11-27 21:43:10 +0000 | [diff] [blame] | 5171 | finish_arch_post_lock_switch(); |
Mark Rutland | 0ed557a | 2018-06-14 15:27:41 -0700 | [diff] [blame] | 5172 | kcov_finish_switch(current); |
Thomas Gleixner | 5fbda3e | 2020-11-18 20:48:43 +0100 | [diff] [blame] | 5173 | /* |
| 5174 | * kmap_local_sched_out() is invoked with rq::lock held and |
| 5175 | * interrupts disabled. There is no requirement for that, but the |
| 5176 | * sched out code does not have an interrupt enabled section. |
| 5177 | * Restoring the maps on sched in does not require interrupts being |
| 5178 | * disabled either. |
| 5179 | */ |
| 5180 | kmap_local_sched_in(); |
Steven Rostedt | e8fa136 | 2008-01-25 21:08:05 +0100 | [diff] [blame] | 5181 | |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 5182 | fire_sched_in_preempt_notifiers(current); |
Mathieu Desnoyers | 306e060 | 2018-01-29 15:20:12 -0500 | [diff] [blame] | 5183 | /* |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 5184 | * When switching through a kernel thread, the loop in |
| 5185 | * membarrier_{private,global}_expedited() may have observed that |
| 5186 | * kernel thread and not issued an IPI. It is therefore possible to |
| 5187 | * schedule between user->kernel->user threads without passing though |
| 5188 | * switch_mm(). Membarrier requires a barrier after storing to |
| 5189 | * rq->curr, before returning to userspace, so provide them here: |
| 5190 | * |
| 5191 | * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly |
| 5192 | * provided by mmdrop(), |
| 5193 | * - a sync_core for SYNC_CORE. |
Mathieu Desnoyers | 306e060 | 2018-01-29 15:20:12 -0500 | [diff] [blame] | 5194 | */ |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 5195 | if (mm) { |
| 5196 | membarrier_mm_sync_core_before_usermode(mm); |
Thomas Gleixner | 8d491de | 2021-09-28 14:24:32 +0200 | [diff] [blame] | 5197 | mmdrop_sched(mm); |
Mathieu Desnoyers | 70216e1 | 2018-01-29 15:20:17 -0500 | [diff] [blame] | 5198 | } |
Peter Zijlstra | 1cef115 | 2018-06-07 11:45:49 +0200 | [diff] [blame] | 5199 | if (unlikely(prev_state == TASK_DEAD)) { |
| 5200 | if (prev->sched_class->task_dead) |
| 5201 | prev->sched_class->task_dead(prev); |
Dario Faggioli | e6c390f | 2013-11-07 14:43:35 +0100 | [diff] [blame] | 5202 | |
Peter Zijlstra | 1cef115 | 2018-06-07 11:45:49 +0200 | [diff] [blame] | 5203 | /* Task is done with its stack. */ |
| 5204 | put_task_stack(prev); |
Andy Lutomirski | 68f24b08 | 2016-09-15 22:45:48 -0700 | [diff] [blame] | 5205 | |
Eric W. Biederman | 0ff7b2cf | 2019-09-14 07:33:58 -0500 | [diff] [blame] | 5206 | put_task_struct_rcu_user(prev); |
bibo mao | c6fd91f | 2006-03-26 01:38:20 -0800 | [diff] [blame] | 5207 | } |
Frederic Weisbecker | 99e5ada | 2013-04-20 17:11:50 +0200 | [diff] [blame] | 5208 | |
Oleg Nesterov | dfa50b6 | 2014-10-09 21:32:32 +0200 | [diff] [blame] | 5209 | return rq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5210 | } |
| 5211 | |
| 5212 | /** |
| 5213 | * schedule_tail - first thing a freshly forked thread must call. |
| 5214 | * @prev: the thread we just switched away from. |
| 5215 | */ |
Andi Kleen | 722a9f9 | 2014-05-02 00:44:38 +0200 | [diff] [blame] | 5216 | asmlinkage __visible void schedule_tail(struct task_struct *prev) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5217 | __releases(rq->lock) |
| 5218 | { |
Peter Zijlstra | 609ca06 | 2015-09-28 17:52:18 +0200 | [diff] [blame] | 5219 | /* |
| 5220 | * New tasks start with FORK_PREEMPT_COUNT, see there and |
| 5221 | * finish_task_switch() for details. |
| 5222 | * |
| 5223 | * finish_task_switch() will drop rq->lock() and lower preempt_count |
| 5224 | * and the preempt_enable() will end up enabling preemption (on |
| 5225 | * PREEMPT_COUNT kernels). |
| 5226 | */ |
| 5227 | |
Edmundo Carmona Antoranz | 13c2235 | 2021-03-06 15:07:39 -0600 | [diff] [blame] | 5228 | finish_task_switch(prev); |
Oleg Nesterov | 1a43a14 | 2014-10-08 21:36:44 +0200 | [diff] [blame] | 5229 | preempt_enable(); |
Steven Rostedt | da19ab5 | 2009-07-29 00:21:22 -0400 | [diff] [blame] | 5230 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5231 | if (current->set_child_tid) |
Pavel Emelyanov | b488893 | 2007-10-18 23:40:14 -0700 | [diff] [blame] | 5232 | put_user(task_pid_vnr(current), current->set_child_tid); |
Eric W. Biederman | 088fe47 | 2018-07-23 17:26:49 -0500 | [diff] [blame] | 5233 | |
| 5234 | calculate_sigpending(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5235 | } |
| 5236 | |
| 5237 | /* |
Oleg Nesterov | dfa50b6 | 2014-10-09 21:32:32 +0200 | [diff] [blame] | 5238 | * context_switch - switch to the new MM and the new thread's register state. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5239 | */ |
Josh Poimboeuf | 0493694 | 2016-02-28 22:22:39 -0600 | [diff] [blame] | 5240 | static __always_inline struct rq * |
Ingo Molnar | 70b97a7 | 2006-07-03 00:25:42 -0700 | [diff] [blame] | 5241 | context_switch(struct rq *rq, struct task_struct *prev, |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 5242 | struct task_struct *next, struct rq_flags *rf) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5243 | { |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 5244 | prepare_task_switch(rq, prev, next); |
Peter Zijlstra | fe4b04f | 2011-02-02 13:19:09 +0100 | [diff] [blame] | 5245 | |
Zachary Amsden | 9226d12 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 5246 | /* |
| 5247 | * For paravirt, this is coupled with an exit in switch_to to |
| 5248 | * combine the page table reload and the switch backend into |
| 5249 | * one hypercall. |
| 5250 | */ |
Jeremy Fitzhardinge | 224101e | 2009-02-18 11:18:57 -0800 | [diff] [blame] | 5251 | arch_start_context_switch(prev); |
Zachary Amsden | 9226d12 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 5252 | |
Mathieu Desnoyers | 306e060 | 2018-01-29 15:20:12 -0500 | [diff] [blame] | 5253 | /* |
Peter Zijlstra | 139d025 | 2019-07-29 16:05:15 +0200 | [diff] [blame] | 5254 | * kernel -> kernel lazy + transfer active |
| 5255 | * user -> kernel lazy + mmgrab() active |
| 5256 | * |
| 5257 | * kernel -> user switch + mmdrop() active |
| 5258 | * user -> user switch |
Mathieu Desnoyers | 306e060 | 2018-01-29 15:20:12 -0500 | [diff] [blame] | 5259 | */ |
Peter Zijlstra | 139d025 | 2019-07-29 16:05:15 +0200 | [diff] [blame] | 5260 | if (!next->mm) { // to kernel |
| 5261 | enter_lazy_tlb(prev->active_mm, next); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5262 | |
Peter Zijlstra | 139d025 | 2019-07-29 16:05:15 +0200 | [diff] [blame] | 5263 | next->active_mm = prev->active_mm; |
| 5264 | if (prev->mm) // from user |
| 5265 | mmgrab(prev->active_mm); |
| 5266 | else |
| 5267 | prev->active_mm = NULL; |
| 5268 | } else { // to user |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 5269 | membarrier_switch_mm(rq, prev->active_mm, next->mm); |
Peter Zijlstra | 139d025 | 2019-07-29 16:05:15 +0200 | [diff] [blame] | 5270 | /* |
| 5271 | * sys_membarrier() requires an smp_mb() between setting |
Mathieu Desnoyers | 227a4aa | 2019-09-19 13:37:02 -0400 | [diff] [blame] | 5272 | * rq->curr / membarrier_switch_mm() and returning to userspace. |
Peter Zijlstra | 139d025 | 2019-07-29 16:05:15 +0200 | [diff] [blame] | 5273 | * |
| 5274 | * The below provides this either through switch_mm(), or in |
| 5275 | * case 'prev->active_mm == next->mm' through |
| 5276 | * finish_task_switch()'s mmdrop(). |
| 5277 | */ |
Peter Zijlstra | 139d025 | 2019-07-29 16:05:15 +0200 | [diff] [blame] | 5278 | switch_mm_irqs_off(prev->active_mm, next->mm, next); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 5279 | lru_gen_use_mm(next->mm); |
Peter Zijlstra | 139d025 | 2019-07-29 16:05:15 +0200 | [diff] [blame] | 5280 | |
| 5281 | if (!prev->mm) { // from kernel |
| 5282 | /* will mmdrop() in finish_task_switch(). */ |
| 5283 | rq->prev_mm = prev->active_mm; |
| 5284 | prev->active_mm = NULL; |
| 5285 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5286 | } |
Matt Fleming | 92509b7 | 2016-09-21 14:38:11 +0100 | [diff] [blame] | 5287 | |
Matt Fleming | cb42c9a | 2016-09-21 14:38:13 +0100 | [diff] [blame] | 5288 | rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); |
Matt Fleming | 92509b7 | 2016-09-21 14:38:11 +0100 | [diff] [blame] | 5289 | |
Peter Zijlstra | 269d599 | 2018-02-06 17:52:13 +0100 | [diff] [blame] | 5290 | prepare_lock_switch(rq, next, rf); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5291 | |
| 5292 | /* Here we just switch the register state and the stack. */ |
| 5293 | switch_to(prev, next, prev); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5294 | barrier(); |
Oleg Nesterov | dfa50b6 | 2014-10-09 21:32:32 +0200 | [diff] [blame] | 5295 | |
| 5296 | return finish_task_switch(prev); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5297 | } |
| 5298 | |
| 5299 | /* |
Sha Zhengju | 1c3e826 | 2013-02-20 17:14:38 +0800 | [diff] [blame] | 5300 | * nr_running and nr_context_switches: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5301 | * |
| 5302 | * externally visible scheduler statistics: current number of runnable |
Sha Zhengju | 1c3e826 | 2013-02-20 17:14:38 +0800 | [diff] [blame] | 5303 | * threads, total number of context switches performed since bootup. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5304 | */ |
Alexey Dobriyan | 01aee8f | 2021-04-22 23:02:25 +0300 | [diff] [blame] | 5305 | unsigned int nr_running(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5306 | { |
Alexey Dobriyan | 01aee8f | 2021-04-22 23:02:25 +0300 | [diff] [blame] | 5307 | unsigned int i, sum = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5308 | |
| 5309 | for_each_online_cpu(i) |
| 5310 | sum += cpu_rq(i)->nr_running; |
| 5311 | |
| 5312 | return sum; |
| 5313 | } |
| 5314 | |
Tim Chen | 2ee507c | 2014-07-31 10:29:48 -0700 | [diff] [blame] | 5315 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 5316 | * Check if only the current task is running on the CPU. |
Dominik Dingel | 00cc1633 | 2015-09-18 11:27:45 +0200 | [diff] [blame] | 5317 | * |
| 5318 | * Caution: this function does not check that the caller has disabled |
| 5319 | * preemption, thus the result might have a time-of-check-to-time-of-use |
| 5320 | * race. The caller is responsible to use it correctly, for example: |
| 5321 | * |
Ingo Molnar | dfcb245 | 2018-12-03 10:05:56 +0100 | [diff] [blame] | 5322 | * - from a non-preemptible section (of course) |
Dominik Dingel | 00cc1633 | 2015-09-18 11:27:45 +0200 | [diff] [blame] | 5323 | * |
| 5324 | * - from a thread that is bound to a single CPU |
| 5325 | * |
| 5326 | * - in a loop with very short iterations (e.g. a polling loop) |
Tim Chen | 2ee507c | 2014-07-31 10:29:48 -0700 | [diff] [blame] | 5327 | */ |
| 5328 | bool single_task_running(void) |
| 5329 | { |
Dominik Dingel | 00cc1633 | 2015-09-18 11:27:45 +0200 | [diff] [blame] | 5330 | return raw_rq()->nr_running == 1; |
Tim Chen | 2ee507c | 2014-07-31 10:29:48 -0700 | [diff] [blame] | 5331 | } |
| 5332 | EXPORT_SYMBOL(single_task_running); |
| 5333 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5334 | unsigned long long nr_context_switches(void) |
| 5335 | { |
Steven Rostedt | cc94abf | 2006-06-27 02:54:31 -0700 | [diff] [blame] | 5336 | int i; |
| 5337 | unsigned long long sum = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5338 | |
KAMEZAWA Hiroyuki | 0a94502 | 2006-03-28 01:56:37 -0800 | [diff] [blame] | 5339 | for_each_possible_cpu(i) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5340 | sum += cpu_rq(i)->nr_switches; |
| 5341 | |
| 5342 | return sum; |
| 5343 | } |
| 5344 | |
Tejun Heo | e33a9bb | 2016-12-07 15:48:41 -0500 | [diff] [blame] | 5345 | /* |
Daniel Lezcano | 145d952 | 2018-10-04 14:04:02 +0200 | [diff] [blame] | 5346 | * Consumers of these two interfaces, like for example the cpuidle menu |
| 5347 | * governor, are using nonsensical data. Preferring shallow idle state selection |
| 5348 | * for a CPU that has IO-wait which might not even end up running the task when |
| 5349 | * it does become runnable. |
| 5350 | */ |
| 5351 | |
Alexey Dobriyan | 8fc2858 | 2021-04-22 23:02:27 +0300 | [diff] [blame] | 5352 | unsigned int nr_iowait_cpu(int cpu) |
Daniel Lezcano | 145d952 | 2018-10-04 14:04:02 +0200 | [diff] [blame] | 5353 | { |
| 5354 | return atomic_read(&cpu_rq(cpu)->nr_iowait); |
| 5355 | } |
| 5356 | |
Daniel Lezcano | 145d952 | 2018-10-04 14:04:02 +0200 | [diff] [blame] | 5357 | /* |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 5358 | * IO-wait accounting, and how it's mostly bollocks (on SMP). |
Tejun Heo | e33a9bb | 2016-12-07 15:48:41 -0500 | [diff] [blame] | 5359 | * |
| 5360 | * The idea behind IO-wait account is to account the idle time that we could |
| 5361 | * have spend running if it were not for IO. That is, if we were to improve the |
| 5362 | * storage performance, we'd have a proportional reduction in IO-wait time. |
| 5363 | * |
| 5364 | * This all works nicely on UP, where, when a task blocks on IO, we account |
| 5365 | * idle time as IO-wait, because if the storage were faster, it could've been |
| 5366 | * running and we'd not be idle. |
| 5367 | * |
| 5368 | * This has been extended to SMP, by doing the same for each CPU. This however |
| 5369 | * is broken. |
| 5370 | * |
| 5371 | * Imagine for instance the case where two tasks block on one CPU, only the one |
| 5372 | * CPU will have IO-wait accounted, while the other has regular idle. Even |
| 5373 | * though, if the storage were faster, both could've ran at the same time, |
| 5374 | * utilising both CPUs. |
| 5375 | * |
| 5376 | * This means, that when looking globally, the current IO-wait accounting on |
| 5377 | * SMP is a lower bound, by reason of under accounting. |
| 5378 | * |
| 5379 | * Worse, since the numbers are provided per CPU, they are sometimes |
| 5380 | * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly |
| 5381 | * associated with any one particular CPU, it can wake to another CPU than it |
| 5382 | * blocked on. This means the per CPU IO-wait number is meaningless. |
| 5383 | * |
| 5384 | * Task CPU affinities can make all that even more 'interesting'. |
| 5385 | */ |
| 5386 | |
Alexey Dobriyan | 9745516 | 2021-04-22 23:02:26 +0300 | [diff] [blame] | 5387 | unsigned int nr_iowait(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5388 | { |
Alexey Dobriyan | 9745516 | 2021-04-22 23:02:26 +0300 | [diff] [blame] | 5389 | unsigned int i, sum = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5390 | |
KAMEZAWA Hiroyuki | 0a94502 | 2006-03-28 01:56:37 -0800 | [diff] [blame] | 5391 | for_each_possible_cpu(i) |
Daniel Lezcano | 145d952 | 2018-10-04 14:04:02 +0200 | [diff] [blame] | 5392 | sum += nr_iowait_cpu(i); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5393 | |
| 5394 | return sum; |
| 5395 | } |
| 5396 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5397 | #ifdef CONFIG_SMP |
| 5398 | |
Ingo Molnar | 48f24c4 | 2006-07-03 00:25:40 -0700 | [diff] [blame] | 5399 | /* |
Peter Zijlstra | 3802290 | 2009-12-16 18:04:37 +0100 | [diff] [blame] | 5400 | * sched_exec - execve() is a valuable balancing opportunity, because at |
| 5401 | * this point the task has the smallest effective memory and cache footprint. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5402 | */ |
Peter Zijlstra | 3802290 | 2009-12-16 18:04:37 +0100 | [diff] [blame] | 5403 | void sched_exec(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5404 | { |
Peter Zijlstra | 3802290 | 2009-12-16 18:04:37 +0100 | [diff] [blame] | 5405 | struct task_struct *p = current; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5406 | unsigned long flags; |
Peter Zijlstra | 0017d73 | 2010-03-24 18:34:10 +0100 | [diff] [blame] | 5407 | int dest_cpu; |
Peter Zijlstra | 3802290 | 2009-12-16 18:04:37 +0100 | [diff] [blame] | 5408 | |
Peter Zijlstra | 8f42ced | 2011-04-05 17:23:53 +0200 | [diff] [blame] | 5409 | raw_spin_lock_irqsave(&p->pi_lock, flags); |
Valentin Schneider | 3aef1551 | 2020-11-02 18:45:13 +0000 | [diff] [blame] | 5410 | dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC); |
Peter Zijlstra | 0017d73 | 2010-03-24 18:34:10 +0100 | [diff] [blame] | 5411 | if (dest_cpu == smp_processor_id()) |
| 5412 | goto unlock; |
Peter Zijlstra | 3802290 | 2009-12-16 18:04:37 +0100 | [diff] [blame] | 5413 | |
Peter Zijlstra | 8f42ced | 2011-04-05 17:23:53 +0200 | [diff] [blame] | 5414 | if (likely(cpu_active(dest_cpu))) { |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 5415 | struct migration_arg arg = { p, dest_cpu }; |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 5416 | |
Peter Zijlstra | 8f42ced | 2011-04-05 17:23:53 +0200 | [diff] [blame] | 5417 | raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
| 5418 | stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5419 | return; |
| 5420 | } |
Peter Zijlstra | 0017d73 | 2010-03-24 18:34:10 +0100 | [diff] [blame] | 5421 | unlock: |
Peter Zijlstra | 8f42ced | 2011-04-05 17:23:53 +0200 | [diff] [blame] | 5422 | raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5423 | } |
| 5424 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5425 | #endif |
| 5426 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5427 | DEFINE_PER_CPU(struct kernel_stat, kstat); |
Glauber Costa | 3292beb | 2011-11-28 14:45:17 -0200 | [diff] [blame] | 5428 | DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5429 | |
| 5430 | EXPORT_PER_CPU_SYMBOL(kstat); |
Glauber Costa | 3292beb | 2011-11-28 14:45:17 -0200 | [diff] [blame] | 5431 | EXPORT_PER_CPU_SYMBOL(kernel_cpustat); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5432 | |
| 5433 | /* |
Giovanni Gherdovich | 6075620 | 2016-08-05 10:21:56 +0200 | [diff] [blame] | 5434 | * The function fair_sched_class.update_curr accesses the struct curr |
| 5435 | * and its field curr->exec_start; when called from task_sched_runtime(), |
| 5436 | * we observe a high rate of cache misses in practice. |
| 5437 | * Prefetching this data results in improved performance. |
| 5438 | */ |
| 5439 | static inline void prefetch_curr_exec_start(struct task_struct *p) |
| 5440 | { |
| 5441 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 5442 | struct sched_entity *curr = (&p->se)->cfs_rq->curr; |
| 5443 | #else |
| 5444 | struct sched_entity *curr = (&task_rq(p)->cfs)->curr; |
| 5445 | #endif |
| 5446 | prefetch(curr); |
| 5447 | prefetch(&curr->exec_start); |
| 5448 | } |
| 5449 | |
| 5450 | /* |
Hidetoshi Seto | c5f8d99 | 2009-03-31 16:56:03 +0900 | [diff] [blame] | 5451 | * Return accounted runtime for the task. |
| 5452 | * In case the task is currently running, return the runtime plus current's |
| 5453 | * pending runtime that have not been accounted yet. |
| 5454 | */ |
| 5455 | unsigned long long task_sched_runtime(struct task_struct *p) |
| 5456 | { |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 5457 | struct rq_flags rf; |
Hidetoshi Seto | c5f8d99 | 2009-03-31 16:56:03 +0900 | [diff] [blame] | 5458 | struct rq *rq; |
Stanislaw Gruszka | 6e99891 | 2014-11-12 16:58:44 +0100 | [diff] [blame] | 5459 | u64 ns; |
Ingo Molnar | 48f24c4 | 2006-07-03 00:25:40 -0700 | [diff] [blame] | 5460 | |
Peter Zijlstra | 911b289 | 2013-11-11 18:21:56 +0100 | [diff] [blame] | 5461 | #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) |
| 5462 | /* |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 5463 | * 64-bit doesn't need locks to atomically read a 64-bit value. |
Peter Zijlstra | 911b289 | 2013-11-11 18:21:56 +0100 | [diff] [blame] | 5464 | * So we have a optimization chance when the task's delta_exec is 0. |
| 5465 | * Reading ->on_cpu is racy, but this is ok. |
| 5466 | * |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 5467 | * If we race with it leaving CPU, we'll take a lock. So we're correct. |
| 5468 | * If we race with it entering CPU, unaccounted time is 0. This is |
Peter Zijlstra | 911b289 | 2013-11-11 18:21:56 +0100 | [diff] [blame] | 5469 | * indistinguishable from the read occurring a few cycles earlier. |
Mike Galbraith | 4036ac1 | 2014-06-24 07:49:40 +0200 | [diff] [blame] | 5470 | * If we see ->on_cpu without ->on_rq, the task is leaving, and has |
| 5471 | * been accounted, so we're correct here as well. |
Peter Zijlstra | 911b289 | 2013-11-11 18:21:56 +0100 | [diff] [blame] | 5472 | */ |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 5473 | if (!p->on_cpu || !task_on_rq_queued(p)) |
Peter Zijlstra | 911b289 | 2013-11-11 18:21:56 +0100 | [diff] [blame] | 5474 | return p->se.sum_exec_runtime; |
| 5475 | #endif |
| 5476 | |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 5477 | rq = task_rq_lock(p, &rf); |
Stanislaw Gruszka | 6e99891 | 2014-11-12 16:58:44 +0100 | [diff] [blame] | 5478 | /* |
| 5479 | * Must be ->curr _and_ ->on_rq. If dequeued, we would |
| 5480 | * project cycles that may never be accounted to this |
| 5481 | * thread, breaking clock_gettime(). |
| 5482 | */ |
| 5483 | if (task_current(rq, p) && task_on_rq_queued(p)) { |
Giovanni Gherdovich | 6075620 | 2016-08-05 10:21:56 +0200 | [diff] [blame] | 5484 | prefetch_curr_exec_start(p); |
Stanislaw Gruszka | 6e99891 | 2014-11-12 16:58:44 +0100 | [diff] [blame] | 5485 | update_rq_clock(rq); |
| 5486 | p->sched_class->update_curr(rq); |
| 5487 | } |
| 5488 | ns = p->se.sum_exec_runtime; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 5489 | task_rq_unlock(rq, p, &rf); |
Hidetoshi Seto | c5f8d99 | 2009-03-31 16:56:03 +0900 | [diff] [blame] | 5490 | |
| 5491 | return ns; |
| 5492 | } |
| 5493 | |
Paul Turner | c006fac | 2021-04-16 14:29:36 -0700 | [diff] [blame] | 5494 | #ifdef CONFIG_SCHED_DEBUG |
| 5495 | static u64 cpu_resched_latency(struct rq *rq) |
| 5496 | { |
| 5497 | int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms); |
| 5498 | u64 resched_latency, now = rq_clock(rq); |
| 5499 | static bool warned_once; |
| 5500 | |
| 5501 | if (sysctl_resched_latency_warn_once && warned_once) |
| 5502 | return 0; |
| 5503 | |
| 5504 | if (!need_resched() || !latency_warn_ms) |
| 5505 | return 0; |
| 5506 | |
| 5507 | if (system_state == SYSTEM_BOOTING) |
| 5508 | return 0; |
| 5509 | |
| 5510 | if (!rq->last_seen_need_resched_ns) { |
| 5511 | rq->last_seen_need_resched_ns = now; |
| 5512 | rq->ticks_without_resched = 0; |
| 5513 | return 0; |
| 5514 | } |
| 5515 | |
| 5516 | rq->ticks_without_resched++; |
| 5517 | resched_latency = now - rq->last_seen_need_resched_ns; |
| 5518 | if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC) |
| 5519 | return 0; |
| 5520 | |
| 5521 | warned_once = true; |
| 5522 | |
| 5523 | return resched_latency; |
| 5524 | } |
| 5525 | |
| 5526 | static int __init setup_resched_latency_warn_ms(char *str) |
| 5527 | { |
| 5528 | long val; |
| 5529 | |
| 5530 | if ((kstrtol(str, 0, &val))) { |
| 5531 | pr_warn("Unable to set resched_latency_warn_ms\n"); |
| 5532 | return 1; |
| 5533 | } |
| 5534 | |
| 5535 | sysctl_resched_latency_warn_ms = val; |
| 5536 | return 1; |
| 5537 | } |
| 5538 | __setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms); |
| 5539 | #else |
| 5540 | static inline u64 cpu_resched_latency(struct rq *rq) { return 0; } |
| 5541 | #endif /* CONFIG_SCHED_DEBUG */ |
| 5542 | |
Balbir Singh | 4904862 | 2008-09-05 18:12:23 +0200 | [diff] [blame] | 5543 | /* |
Christoph Lameter | 7835b98 | 2006-12-10 02:20:22 -0800 | [diff] [blame] | 5544 | * This function gets called by the timer code, with HZ frequency. |
| 5545 | * We call it with interrupts disabled. |
Christoph Lameter | 7835b98 | 2006-12-10 02:20:22 -0800 | [diff] [blame] | 5546 | */ |
| 5547 | void scheduler_tick(void) |
| 5548 | { |
Christoph Lameter | 7835b98 | 2006-12-10 02:20:22 -0800 | [diff] [blame] | 5549 | int cpu = smp_processor_id(); |
| 5550 | struct rq *rq = cpu_rq(cpu); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5551 | struct task_struct *curr = rq->curr; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 5552 | struct rq_flags rf; |
Thara Gopinath | b4eccf5 | 2020-02-21 19:52:10 -0500 | [diff] [blame] | 5553 | unsigned long thermal_pressure; |
Paul Turner | c006fac | 2021-04-16 14:29:36 -0700 | [diff] [blame] | 5554 | u64 resched_latency; |
Peter Zijlstra | 3e51f33 | 2008-05-03 18:29:28 +0200 | [diff] [blame] | 5555 | |
Yair Podemsky | 7fb3ff2 | 2022-11-30 14:51:21 +0200 | [diff] [blame] | 5556 | if (housekeeping_cpu(cpu, HK_TYPE_TICK)) |
| 5557 | arch_scale_freq_tick(); |
| 5558 | |
Peter Zijlstra | 3e51f33 | 2008-05-03 18:29:28 +0200 | [diff] [blame] | 5559 | sched_clock_tick(); |
Christoph Lameter | 7835b98 | 2006-12-10 02:20:22 -0800 | [diff] [blame] | 5560 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 5561 | rq_lock(rq, &rf); |
| 5562 | |
Peter Zijlstra | 3e51f33 | 2008-05-03 18:29:28 +0200 | [diff] [blame] | 5563 | update_rq_clock(rq); |
Thara Gopinath | b4eccf5 | 2020-02-21 19:52:10 -0500 | [diff] [blame] | 5564 | thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); |
Thara Gopinath | 05289b9 | 2020-02-21 19:52:13 -0500 | [diff] [blame] | 5565 | update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); |
Peter Zijlstra | fa85ae2 | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 5566 | curr->sched_class->task_tick(rq, curr, 0); |
Paul Turner | c006fac | 2021-04-16 14:29:36 -0700 | [diff] [blame] | 5567 | if (sched_feat(LATENCY_WARN)) |
| 5568 | resched_latency = cpu_resched_latency(rq); |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5569 | calc_global_load_tick(rq); |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 5570 | sched_core_tick(rq); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 5571 | |
| 5572 | rq_unlock(rq, &rf); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5573 | |
Paul Turner | c006fac | 2021-04-16 14:29:36 -0700 | [diff] [blame] | 5574 | if (sched_feat(LATENCY_WARN) && resched_latency) |
| 5575 | resched_latency_warn(cpu, resched_latency); |
| 5576 | |
Peter Zijlstra | e9d2b06 | 2010-09-17 11:28:50 +0200 | [diff] [blame] | 5577 | perf_event_task_tick(); |
Peter Zijlstra | e220d2d | 2009-05-23 18:28:55 +0200 | [diff] [blame] | 5578 | |
Christoph Lameter | e418e1c | 2006-12-10 02:20:23 -0800 | [diff] [blame] | 5579 | #ifdef CONFIG_SMP |
Suresh Siddha | 6eb57e0 | 2011-10-03 15:09:01 -0700 | [diff] [blame] | 5580 | rq->idle_balance = idle_cpu(cpu); |
Daniel Lezcano | 7caff66 | 2014-01-06 12:34:38 +0100 | [diff] [blame] | 5581 | trigger_load_balance(rq); |
Christoph Lameter | e418e1c | 2006-12-10 02:20:23 -0800 | [diff] [blame] | 5582 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5583 | } |
| 5584 | |
Frederic Weisbecker | 265f22a | 2013-05-03 03:39:05 +0200 | [diff] [blame] | 5585 | #ifdef CONFIG_NO_HZ_FULL |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5586 | |
| 5587 | struct tick_work { |
| 5588 | int cpu; |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5589 | atomic_t state; |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5590 | struct delayed_work work; |
| 5591 | }; |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5592 | /* Values for ->state, see diagram below. */ |
| 5593 | #define TICK_SCHED_REMOTE_OFFLINE 0 |
| 5594 | #define TICK_SCHED_REMOTE_OFFLINING 1 |
| 5595 | #define TICK_SCHED_REMOTE_RUNNING 2 |
| 5596 | |
| 5597 | /* |
| 5598 | * State diagram for ->state: |
| 5599 | * |
| 5600 | * |
| 5601 | * TICK_SCHED_REMOTE_OFFLINE |
| 5602 | * | ^ |
| 5603 | * | | |
| 5604 | * | | sched_tick_remote() |
| 5605 | * | | |
| 5606 | * | | |
| 5607 | * +--TICK_SCHED_REMOTE_OFFLINING |
| 5608 | * | ^ |
| 5609 | * | | |
| 5610 | * sched_tick_start() | | sched_tick_stop() |
| 5611 | * | | |
| 5612 | * V | |
| 5613 | * TICK_SCHED_REMOTE_RUNNING |
| 5614 | * |
| 5615 | * |
| 5616 | * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() |
| 5617 | * and sched_tick_start() are happy to leave the state in RUNNING. |
| 5618 | */ |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5619 | |
| 5620 | static struct tick_work __percpu *tick_work_cpu; |
| 5621 | |
| 5622 | static void sched_tick_remote(struct work_struct *work) |
| 5623 | { |
| 5624 | struct delayed_work *dwork = to_delayed_work(work); |
| 5625 | struct tick_work *twork = container_of(dwork, struct tick_work, work); |
| 5626 | int cpu = twork->cpu; |
| 5627 | struct rq *rq = cpu_rq(cpu); |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5628 | struct task_struct *curr; |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5629 | struct rq_flags rf; |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5630 | u64 delta; |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5631 | int os; |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5632 | |
| 5633 | /* |
| 5634 | * Handle the tick only if it appears the remote CPU is running in full |
| 5635 | * dynticks mode. The check is racy by nature, but missing a tick or |
| 5636 | * having one too much is no big deal because the scheduler tick updates |
| 5637 | * statistics and checks timeslices in a time-independent way, regardless |
| 5638 | * of when exactly it is running. |
| 5639 | */ |
Scott Wood | 488603b | 2020-01-11 04:53:38 -0500 | [diff] [blame] | 5640 | if (!tick_nohz_tick_stopped_cpu(cpu)) |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5641 | goto out_requeue; |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5642 | |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5643 | rq_lock_irq(rq, &rf); |
| 5644 | curr = rq->curr; |
Scott Wood | 488603b | 2020-01-11 04:53:38 -0500 | [diff] [blame] | 5645 | if (cpu_is_offline(cpu)) |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5646 | goto out_unlock; |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5647 | |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5648 | update_rq_clock(rq); |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5649 | |
Scott Wood | 488603b | 2020-01-11 04:53:38 -0500 | [diff] [blame] | 5650 | if (!is_idle_task(curr)) { |
| 5651 | /* |
| 5652 | * Make sure the next tick runs within a reasonable |
| 5653 | * amount of time. |
| 5654 | */ |
| 5655 | delta = rq_clock_task(rq) - curr->se.exec_start; |
| 5656 | WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); |
| 5657 | } |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5658 | curr->sched_class->task_tick(rq, curr, 0); |
| 5659 | |
Peter Zijlstra (Intel) | ebc0f83 | 2020-01-11 04:53:39 -0500 | [diff] [blame] | 5660 | calc_load_nohz_remote(rq); |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5661 | out_unlock: |
| 5662 | rq_unlock_irq(rq, &rf); |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5663 | out_requeue: |
Peter Zijlstra (Intel) | ebc0f83 | 2020-01-11 04:53:39 -0500 | [diff] [blame] | 5664 | |
Frederic Weisbecker | d9c0ffc | 2018-06-28 18:29:41 +0200 | [diff] [blame] | 5665 | /* |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5666 | * Run the remote tick once per second (1Hz). This arbitrary |
| 5667 | * frequency is large enough to avoid overload but short enough |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5668 | * to keep scheduler internal stats reasonably up to date. But |
| 5669 | * first update state to reflect hotplug activity if required. |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5670 | */ |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5671 | os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); |
| 5672 | WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); |
| 5673 | if (os == TICK_SCHED_REMOTE_RUNNING) |
| 5674 | queue_delayed_work(system_unbound_wq, dwork, HZ); |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5675 | } |
| 5676 | |
| 5677 | static void sched_tick_start(int cpu) |
| 5678 | { |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5679 | int os; |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5680 | struct tick_work *twork; |
| 5681 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 5682 | if (housekeeping_cpu(cpu, HK_TYPE_TICK)) |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5683 | return; |
| 5684 | |
| 5685 | WARN_ON_ONCE(!tick_work_cpu); |
| 5686 | |
| 5687 | twork = per_cpu_ptr(tick_work_cpu, cpu); |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5688 | os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); |
| 5689 | WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); |
| 5690 | if (os == TICK_SCHED_REMOTE_OFFLINE) { |
| 5691 | twork->cpu = cpu; |
| 5692 | INIT_DELAYED_WORK(&twork->work, sched_tick_remote); |
| 5693 | queue_delayed_work(system_unbound_wq, &twork->work, HZ); |
| 5694 | } |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5695 | } |
| 5696 | |
| 5697 | #ifdef CONFIG_HOTPLUG_CPU |
| 5698 | static void sched_tick_stop(int cpu) |
| 5699 | { |
| 5700 | struct tick_work *twork; |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5701 | int os; |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5702 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 5703 | if (housekeeping_cpu(cpu, HK_TYPE_TICK)) |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5704 | return; |
| 5705 | |
| 5706 | WARN_ON_ONCE(!tick_work_cpu); |
| 5707 | |
| 5708 | twork = per_cpu_ptr(tick_work_cpu, cpu); |
Paul E. McKenney | b55bd58 | 2019-05-30 05:39:25 -0700 | [diff] [blame] | 5709 | /* There cannot be competing actions, but don't rely on stop-machine. */ |
| 5710 | os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); |
| 5711 | WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); |
| 5712 | /* Don't cancel, as this would mess up the state machine. */ |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5713 | } |
| 5714 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 5715 | |
| 5716 | int __init sched_tick_offload_init(void) |
| 5717 | { |
| 5718 | tick_work_cpu = alloc_percpu(struct tick_work); |
| 5719 | BUG_ON(!tick_work_cpu); |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 5720 | return 0; |
| 5721 | } |
| 5722 | |
| 5723 | #else /* !CONFIG_NO_HZ_FULL */ |
| 5724 | static inline void sched_tick_start(int cpu) { } |
| 5725 | static inline void sched_tick_stop(int cpu) { } |
Frederic Weisbecker | 265f22a | 2013-05-03 03:39:05 +0200 | [diff] [blame] | 5726 | #endif |
| 5727 | |
Thomas Gleixner | c1a280b | 2019-07-26 23:19:37 +0200 | [diff] [blame] | 5728 | #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 5729 | defined(CONFIG_TRACE_PREEMPT_TOGGLE)) |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 5730 | /* |
| 5731 | * If the value passed in is equal to the current preempt count |
| 5732 | * then we just disabled preemption. Start timing the latency. |
| 5733 | */ |
| 5734 | static inline void preempt_latency_start(int val) |
| 5735 | { |
| 5736 | if (preempt_count() == val) { |
| 5737 | unsigned long ip = get_lock_parent_ip(); |
| 5738 | #ifdef CONFIG_DEBUG_PREEMPT |
| 5739 | current->preempt_disable_ip = ip; |
| 5740 | #endif |
| 5741 | trace_preempt_off(CALLER_ADDR0, ip); |
| 5742 | } |
| 5743 | } |
Steven Rostedt | 7e49fcc | 2009-01-22 19:01:40 -0500 | [diff] [blame] | 5744 | |
Masami Hiramatsu | edafe3a | 2014-04-17 17:18:42 +0900 | [diff] [blame] | 5745 | void preempt_count_add(int val) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5746 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 5747 | #ifdef CONFIG_DEBUG_PREEMPT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5748 | /* |
| 5749 | * Underflow? |
| 5750 | */ |
Ingo Molnar | 9a11b49a | 2006-07-03 00:24:33 -0700 | [diff] [blame] | 5751 | if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) |
| 5752 | return; |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 5753 | #endif |
Peter Zijlstra | bdb4380 | 2013-09-10 12:15:23 +0200 | [diff] [blame] | 5754 | __preempt_count_add(val); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 5755 | #ifdef CONFIG_DEBUG_PREEMPT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5756 | /* |
| 5757 | * Spinlock count overflowing soon? |
| 5758 | */ |
Miguel Ojeda Sandonis | 33859f7 | 2006-12-10 02:20:38 -0800 | [diff] [blame] | 5759 | DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= |
| 5760 | PREEMPT_MASK - 10); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 5761 | #endif |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 5762 | preempt_latency_start(val); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5763 | } |
Peter Zijlstra | bdb4380 | 2013-09-10 12:15:23 +0200 | [diff] [blame] | 5764 | EXPORT_SYMBOL(preempt_count_add); |
Masami Hiramatsu | edafe3a | 2014-04-17 17:18:42 +0900 | [diff] [blame] | 5765 | NOKPROBE_SYMBOL(preempt_count_add); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5766 | |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 5767 | /* |
| 5768 | * If the value passed in equals to the current preempt count |
| 5769 | * then we just enabled preemption. Stop timing the latency. |
| 5770 | */ |
| 5771 | static inline void preempt_latency_stop(int val) |
| 5772 | { |
| 5773 | if (preempt_count() == val) |
| 5774 | trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); |
| 5775 | } |
| 5776 | |
Masami Hiramatsu | edafe3a | 2014-04-17 17:18:42 +0900 | [diff] [blame] | 5777 | void preempt_count_sub(int val) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5778 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 5779 | #ifdef CONFIG_DEBUG_PREEMPT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5780 | /* |
| 5781 | * Underflow? |
| 5782 | */ |
Ingo Molnar | 01e3eb8 | 2009-01-12 13:00:50 +0100 | [diff] [blame] | 5783 | if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) |
Ingo Molnar | 9a11b49a | 2006-07-03 00:24:33 -0700 | [diff] [blame] | 5784 | return; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5785 | /* |
| 5786 | * Is the spinlock portion underflowing? |
| 5787 | */ |
Ingo Molnar | 9a11b49a | 2006-07-03 00:24:33 -0700 | [diff] [blame] | 5788 | if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && |
| 5789 | !(preempt_count() & PREEMPT_MASK))) |
| 5790 | return; |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 5791 | #endif |
Ingo Molnar | 9a11b49a | 2006-07-03 00:24:33 -0700 | [diff] [blame] | 5792 | |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 5793 | preempt_latency_stop(val); |
Peter Zijlstra | bdb4380 | 2013-09-10 12:15:23 +0200 | [diff] [blame] | 5794 | __preempt_count_sub(val); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5795 | } |
Peter Zijlstra | bdb4380 | 2013-09-10 12:15:23 +0200 | [diff] [blame] | 5796 | EXPORT_SYMBOL(preempt_count_sub); |
Masami Hiramatsu | edafe3a | 2014-04-17 17:18:42 +0900 | [diff] [blame] | 5797 | NOKPROBE_SYMBOL(preempt_count_sub); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5798 | |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 5799 | #else |
| 5800 | static inline void preempt_latency_start(int val) { } |
| 5801 | static inline void preempt_latency_stop(int val) { } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5802 | #endif |
| 5803 | |
Ingo Molnar | 59ddbcb | 2017-02-03 23:37:48 +0100 | [diff] [blame] | 5804 | static inline unsigned long get_preempt_disable_ip(struct task_struct *p) |
| 5805 | { |
| 5806 | #ifdef CONFIG_DEBUG_PREEMPT |
| 5807 | return p->preempt_disable_ip; |
| 5808 | #else |
| 5809 | return 0; |
| 5810 | #endif |
| 5811 | } |
| 5812 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5813 | /* |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5814 | * Print scheduling while atomic bug: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5815 | */ |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5816 | static noinline void __schedule_bug(struct task_struct *prev) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5817 | { |
Vegard Nossum | d1c6d14 | 2016-07-23 09:46:39 +0200 | [diff] [blame] | 5818 | /* Save this before calling printk(), since that will clobber it */ |
| 5819 | unsigned long preempt_disable_ip = get_preempt_disable_ip(current); |
| 5820 | |
Dave Jones | 664dfa6 | 2011-12-22 16:39:30 -0500 | [diff] [blame] | 5821 | if (oops_in_progress) |
| 5822 | return; |
| 5823 | |
Peter Zijlstra | 3df0fc5 | 2009-12-20 14:23:57 +0100 | [diff] [blame] | 5824 | printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", |
| 5825 | prev->comm, prev->pid, preempt_count()); |
Satyam Sharma | 838225b | 2007-10-24 18:23:50 +0200 | [diff] [blame] | 5826 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5827 | debug_show_held_locks(prev); |
Arjan van de Ven | e21f5b1 | 2008-05-23 09:05:58 -0700 | [diff] [blame] | 5828 | print_modules(); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5829 | if (irqs_disabled()) |
| 5830 | print_irqtrace_events(prev); |
Vegard Nossum | d1c6d14 | 2016-07-23 09:46:39 +0200 | [diff] [blame] | 5831 | if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) |
| 5832 | && in_atomic_preempt_off()) { |
Thomas Gleixner | 8f47b18 | 2014-02-07 20:58:39 +0100 | [diff] [blame] | 5833 | pr_err("Preemption disabled at:"); |
Dmitry Safonov | 2062a4e | 2020-06-08 21:29:56 -0700 | [diff] [blame] | 5834 | print_ip_sym(KERN_ERR, preempt_disable_ip); |
Thomas Gleixner | 8f47b18 | 2014-02-07 20:58:39 +0100 | [diff] [blame] | 5835 | } |
Kees Cook | 79cc1ba | 2022-11-17 15:43:24 -0800 | [diff] [blame] | 5836 | check_panic_on_warn("scheduling while atomic"); |
Daniel Bristot de Oliveira | 748c720 | 2016-06-03 17:10:18 -0300 | [diff] [blame] | 5837 | |
Stephen Boyd | 6135fc1 | 2012-03-28 17:10:47 -0700 | [diff] [blame] | 5838 | dump_stack(); |
Rusty Russell | 373d4d0 | 2013-01-21 17:17:39 +1030 | [diff] [blame] | 5839 | add_taint(TAINT_WARN, LOCKDEP_STILL_OK); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5840 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5841 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5842 | /* |
| 5843 | * Various schedule()-time debugging checks and statistics: |
| 5844 | */ |
Daniel Vetter | 312364f3 | 2019-08-26 22:14:23 +0200 | [diff] [blame] | 5845 | static inline void schedule_debug(struct task_struct *prev, bool preempt) |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5846 | { |
Aaron Tomlin | 0d9e263 | 2014-09-12 14:16:19 +0100 | [diff] [blame] | 5847 | #ifdef CONFIG_SCHED_STACK_END_CHECK |
Jann Horn | 29d6455 | 2016-06-01 11:55:07 +0200 | [diff] [blame] | 5848 | if (task_stack_end_corrupted(prev)) |
| 5849 | panic("corrupted stack end detected inside scheduler\n"); |
Will Deacon | 88485be | 2020-05-15 14:56:05 +0100 | [diff] [blame] | 5850 | |
| 5851 | if (task_scs_end_corrupted(prev)) |
| 5852 | panic("corrupted shadow stack detected inside scheduler\n"); |
Aaron Tomlin | 0d9e263 | 2014-09-12 14:16:19 +0100 | [diff] [blame] | 5853 | #endif |
Peter Zijlstra | b99def8 | 2015-09-28 18:02:03 +0200 | [diff] [blame] | 5854 | |
Daniel Vetter | 312364f3 | 2019-08-26 22:14:23 +0200 | [diff] [blame] | 5855 | #ifdef CONFIG_DEBUG_ATOMIC_SLEEP |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 5856 | if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) { |
Daniel Vetter | 312364f3 | 2019-08-26 22:14:23 +0200 | [diff] [blame] | 5857 | printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", |
| 5858 | prev->comm, prev->pid, prev->non_block_count); |
| 5859 | dump_stack(); |
| 5860 | add_taint(TAINT_WARN, LOCKDEP_STILL_OK); |
| 5861 | } |
| 5862 | #endif |
| 5863 | |
Peter Zijlstra | 1dc0fff | 2015-09-28 17:57:39 +0200 | [diff] [blame] | 5864 | if (unlikely(in_atomic_preempt_off())) { |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5865 | __schedule_bug(prev); |
Peter Zijlstra | 1dc0fff | 2015-09-28 17:57:39 +0200 | [diff] [blame] | 5866 | preempt_count_set(PREEMPT_DISABLED); |
| 5867 | } |
Paul E. McKenney | b3fbab0 | 2011-05-24 08:31:09 -0700 | [diff] [blame] | 5868 | rcu_sleep_check(); |
Frederic Weisbecker | 9f68b5b | 2020-11-17 16:16:35 +0100 | [diff] [blame] | 5869 | SCHED_WARN_ON(ct_state() == CONTEXT_USER); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5870 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5871 | profile_hit(SCHED_PROFILING, __builtin_return_address(0)); |
| 5872 | |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 5873 | schedstat_inc(this_rq()->sched_count); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5874 | } |
| 5875 | |
Chen Yu | 457d1f4 | 2020-04-21 18:50:43 +0800 | [diff] [blame] | 5876 | static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, |
| 5877 | struct rq_flags *rf) |
| 5878 | { |
| 5879 | #ifdef CONFIG_SMP |
| 5880 | const struct sched_class *class; |
| 5881 | /* |
| 5882 | * We must do the balancing pass before put_prev_task(), such |
| 5883 | * that when we release the rq->lock the task is in the same |
| 5884 | * state as before we took rq->lock. |
| 5885 | * |
| 5886 | * We can terminate the balance pass as soon as we know there is |
| 5887 | * a runnable task of @class priority or higher. |
| 5888 | */ |
| 5889 | for_class_range(class, prev->sched_class, &idle_sched_class) { |
| 5890 | if (class->balance(rq, prev, rf)) |
| 5891 | break; |
| 5892 | } |
| 5893 | #endif |
| 5894 | |
| 5895 | put_prev_task(rq, prev); |
| 5896 | } |
| 5897 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5898 | /* |
| 5899 | * Pick up the highest-prio task: |
| 5900 | */ |
| 5901 | static inline struct task_struct * |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5902 | __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5903 | { |
Peter Zijlstra | 49ee576 | 2017-01-19 18:44:08 +0100 | [diff] [blame] | 5904 | const struct sched_class *class; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5905 | struct task_struct *p; |
| 5906 | |
| 5907 | /* |
Peter Zijlstra | 0ba87bb | 2017-03-01 10:51:47 +0100 | [diff] [blame] | 5908 | * Optimization: we know that if all tasks are in the fair class we can |
| 5909 | * call that function directly, but only if the @prev task wasn't of a |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 5910 | * higher scheduling class, because otherwise those lose the |
Peter Zijlstra | 0ba87bb | 2017-03-01 10:51:47 +0100 | [diff] [blame] | 5911 | * opportunity to pull in more work from other CPUs. |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5912 | */ |
Peter Zijlstra | 546a3fe | 2022-05-17 13:46:54 +0200 | [diff] [blame] | 5913 | if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && |
Peter Zijlstra | 0ba87bb | 2017-03-01 10:51:47 +0100 | [diff] [blame] | 5914 | rq->nr_running == rq->cfs.h_nr_running)) { |
| 5915 | |
Peter Zijlstra | 5d7d605 | 2019-11-08 14:15:57 +0100 | [diff] [blame] | 5916 | p = pick_next_task_fair(rq, prev, rf); |
Peter Zijlstra | 6ccdc84 | 2014-04-24 12:00:47 +0200 | [diff] [blame] | 5917 | if (unlikely(p == RETRY_TASK)) |
Peter Zijlstra | 6769243 | 2019-05-29 20:36:44 +0000 | [diff] [blame] | 5918 | goto restart; |
Peter Zijlstra | 6ccdc84 | 2014-04-24 12:00:47 +0200 | [diff] [blame] | 5919 | |
Masahiro Yamada | 1699949 | 2021-05-19 15:37:09 +0900 | [diff] [blame] | 5920 | /* Assume the next prioritized class is idle_sched_class */ |
Peter Zijlstra | 5d7d605 | 2019-11-08 14:15:57 +0100 | [diff] [blame] | 5921 | if (!p) { |
Peter Zijlstra | f488e10 | 2019-11-08 14:15:56 +0100 | [diff] [blame] | 5922 | put_prev_task(rq, prev); |
Peter Zijlstra | 98c2f70 | 2019-11-08 14:15:58 +0100 | [diff] [blame] | 5923 | p = pick_next_task_idle(rq); |
Peter Zijlstra | f488e10 | 2019-11-08 14:15:56 +0100 | [diff] [blame] | 5924 | } |
Peter Zijlstra | 6ccdc84 | 2014-04-24 12:00:47 +0200 | [diff] [blame] | 5925 | |
| 5926 | return p; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5927 | } |
| 5928 | |
Peter Zijlstra | 6769243 | 2019-05-29 20:36:44 +0000 | [diff] [blame] | 5929 | restart: |
Chen Yu | 457d1f4 | 2020-04-21 18:50:43 +0800 | [diff] [blame] | 5930 | put_prev_task_balance(rq, prev, rf); |
Peter Zijlstra | 6769243 | 2019-05-29 20:36:44 +0000 | [diff] [blame] | 5931 | |
Peter Zijlstra | 34f971f | 2010-09-22 13:53:15 +0200 | [diff] [blame] | 5932 | for_each_class(class) { |
Peter Zijlstra | 98c2f70 | 2019-11-08 14:15:58 +0100 | [diff] [blame] | 5933 | p = class->pick_next_task(rq); |
Peter Zijlstra | 6769243 | 2019-05-29 20:36:44 +0000 | [diff] [blame] | 5934 | if (p) |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5935 | return p; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5936 | } |
Peter Zijlstra | 34f971f | 2010-09-22 13:53:15 +0200 | [diff] [blame] | 5937 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 5938 | BUG(); /* The idle class should always have a runnable task. */ |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 5939 | } |
| 5940 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 5941 | #ifdef CONFIG_SCHED_CORE |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5942 | static inline bool is_task_rq_idle(struct task_struct *t) |
| 5943 | { |
| 5944 | return (task_rq(t)->idle == t); |
| 5945 | } |
| 5946 | |
| 5947 | static inline bool cookie_equals(struct task_struct *a, unsigned long cookie) |
| 5948 | { |
| 5949 | return is_task_rq_idle(a) || (a->core_cookie == cookie); |
| 5950 | } |
| 5951 | |
| 5952 | static inline bool cookie_match(struct task_struct *a, struct task_struct *b) |
| 5953 | { |
| 5954 | if (is_task_rq_idle(a) || is_task_rq_idle(b)) |
| 5955 | return true; |
| 5956 | |
| 5957 | return a->core_cookie == b->core_cookie; |
| 5958 | } |
| 5959 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 5960 | static inline struct task_struct *pick_task(struct rq *rq) |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5961 | { |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 5962 | const struct sched_class *class; |
| 5963 | struct task_struct *p; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5964 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 5965 | for_each_class(class) { |
| 5966 | p = class->pick_task(rq); |
| 5967 | if (p) |
| 5968 | return p; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5969 | } |
| 5970 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 5971 | BUG(); /* The idle class should always have a runnable task. */ |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5972 | } |
| 5973 | |
Joel Fernandes (Google) | c6047c2 | 2020-11-17 18:19:39 -0500 | [diff] [blame] | 5974 | extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi); |
| 5975 | |
Peter Zijlstra | 5b6547e | 2022-03-16 22:03:41 +0100 | [diff] [blame] | 5976 | static void queue_core_balance(struct rq *rq); |
| 5977 | |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5978 | static struct task_struct * |
| 5979 | pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) |
| 5980 | { |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 5981 | struct task_struct *next, *p, *max = NULL; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5982 | const struct cpumask *smt_mask; |
Joel Fernandes (Google) | c6047c2 | 2020-11-17 18:19:39 -0500 | [diff] [blame] | 5983 | bool fi_before = false; |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 5984 | bool core_clock_updated = (rq == rq->core); |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 5985 | unsigned long cookie; |
| 5986 | int i, cpu, occ = 0; |
| 5987 | struct rq *rq_i; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5988 | bool need_sync; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 5989 | |
| 5990 | if (!sched_core_enabled(rq)) |
| 5991 | return __pick_next_task(rq, prev, rf); |
| 5992 | |
| 5993 | cpu = cpu_of(rq); |
| 5994 | |
| 5995 | /* Stopper task is switching into idle, no need core-wide selection. */ |
| 5996 | if (cpu_is_offline(cpu)) { |
| 5997 | /* |
| 5998 | * Reset core_pick so that we don't enter the fastpath when |
| 5999 | * coming online. core_pick would already be migrated to |
| 6000 | * another cpu during offline. |
| 6001 | */ |
| 6002 | rq->core_pick = NULL; |
| 6003 | return __pick_next_task(rq, prev, rf); |
| 6004 | } |
| 6005 | |
| 6006 | /* |
| 6007 | * If there were no {en,de}queues since we picked (IOW, the task |
| 6008 | * pointers are all still valid), and we haven't scheduled the last |
| 6009 | * pick yet, do so now. |
| 6010 | * |
| 6011 | * rq->core_pick can be NULL if no selection was made for a CPU because |
| 6012 | * it was either offline or went offline during a sibling's core-wide |
| 6013 | * selection. In this case, do a core-wide selection. |
| 6014 | */ |
| 6015 | if (rq->core->core_pick_seq == rq->core->core_task_seq && |
| 6016 | rq->core->core_pick_seq != rq->core_sched_seq && |
| 6017 | rq->core_pick) { |
| 6018 | WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); |
| 6019 | |
| 6020 | next = rq->core_pick; |
| 6021 | if (next != prev) { |
| 6022 | put_prev_task(rq, prev); |
| 6023 | set_next_task(rq, next); |
| 6024 | } |
| 6025 | |
| 6026 | rq->core_pick = NULL; |
Peter Zijlstra | 5b6547e | 2022-03-16 22:03:41 +0100 | [diff] [blame] | 6027 | goto out; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6028 | } |
| 6029 | |
| 6030 | put_prev_task_balance(rq, prev, rf); |
| 6031 | |
| 6032 | smt_mask = cpu_smt_mask(cpu); |
Joel Fernandes (Google) | 7afbba1 | 2020-11-17 18:19:42 -0500 | [diff] [blame] | 6033 | need_sync = !!rq->core->core_cookie; |
| 6034 | |
| 6035 | /* reset state */ |
| 6036 | rq->core->core_cookie = 0UL; |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 6037 | if (rq->core->core_forceidle_count) { |
| 6038 | if (!core_clock_updated) { |
| 6039 | update_rq_clock(rq->core); |
| 6040 | core_clock_updated = true; |
| 6041 | } |
| 6042 | sched_core_account_forceidle(rq); |
| 6043 | /* reset after accounting force idle */ |
| 6044 | rq->core->core_forceidle_start = 0; |
| 6045 | rq->core->core_forceidle_count = 0; |
| 6046 | rq->core->core_forceidle_occupation = 0; |
Joel Fernandes (Google) | 7afbba1 | 2020-11-17 18:19:42 -0500 | [diff] [blame] | 6047 | need_sync = true; |
| 6048 | fi_before = true; |
Joel Fernandes (Google) | 7afbba1 | 2020-11-17 18:19:42 -0500 | [diff] [blame] | 6049 | } |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6050 | |
| 6051 | /* |
| 6052 | * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq |
| 6053 | * |
| 6054 | * @task_seq guards the task state ({en,de}queues) |
| 6055 | * @pick_seq is the @task_seq we did a selection on |
| 6056 | * @sched_seq is the @pick_seq we scheduled |
| 6057 | * |
| 6058 | * However, preemptions can cause multiple picks on the same task set. |
| 6059 | * 'Fix' this by also increasing @task_seq for every pick. |
| 6060 | */ |
| 6061 | rq->core->core_task_seq++; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6062 | |
Joel Fernandes (Google) | 7afbba1 | 2020-11-17 18:19:42 -0500 | [diff] [blame] | 6063 | /* |
| 6064 | * Optimize for common case where this CPU has no cookies |
| 6065 | * and there are no cookied tasks running on siblings. |
| 6066 | */ |
| 6067 | if (!need_sync) { |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6068 | next = pick_task(rq); |
Joel Fernandes (Google) | 7afbba1 | 2020-11-17 18:19:42 -0500 | [diff] [blame] | 6069 | if (!next->core_cookie) { |
| 6070 | rq->core_pick = NULL; |
Joel Fernandes (Google) | c6047c2 | 2020-11-17 18:19:39 -0500 | [diff] [blame] | 6071 | /* |
| 6072 | * For robustness, update the min_vruntime_fi for |
| 6073 | * unconstrained picks as well. |
| 6074 | */ |
| 6075 | WARN_ON_ONCE(fi_before); |
| 6076 | task_vruntime_update(rq, next, false); |
Peter Zijlstra | 5b6547e | 2022-03-16 22:03:41 +0100 | [diff] [blame] | 6077 | goto out_set_next; |
Joel Fernandes (Google) | 7afbba1 | 2020-11-17 18:19:42 -0500 | [diff] [blame] | 6078 | } |
Vineeth Pillai | 8039e96f | 2020-11-17 18:19:38 -0500 | [diff] [blame] | 6079 | } |
Joel Fernandes (Google) | 7afbba1 | 2020-11-17 18:19:42 -0500 | [diff] [blame] | 6080 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6081 | /* |
| 6082 | * For each thread: do the regular task pick and find the max prio task |
| 6083 | * amongst them. |
| 6084 | * |
| 6085 | * Tie-break prio towards the current CPU |
| 6086 | */ |
| 6087 | for_each_cpu_wrap(i, smt_mask, cpu) { |
| 6088 | rq_i = cpu_rq(i); |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6089 | |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 6090 | /* |
| 6091 | * Current cpu always has its clock updated on entrance to |
| 6092 | * pick_next_task(). If the current cpu is not the core, |
| 6093 | * the core may also have been updated above. |
| 6094 | */ |
| 6095 | if (i != cpu && (rq_i != rq->core || !core_clock_updated)) |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6096 | update_rq_clock(rq_i); |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6097 | |
| 6098 | p = rq_i->core_pick = pick_task(rq_i); |
| 6099 | if (!max || prio_less(max, p, fi_before)) |
| 6100 | max = p; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6101 | } |
| 6102 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6103 | cookie = rq->core->core_cookie = max->core_cookie; |
| 6104 | |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6105 | /* |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6106 | * For each thread: try and find a runnable task that matches @max or |
| 6107 | * force idle. |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6108 | */ |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6109 | for_each_cpu(i, smt_mask) { |
| 6110 | rq_i = cpu_rq(i); |
| 6111 | p = rq_i->core_pick; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6112 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6113 | if (!cookie_equals(p, cookie)) { |
| 6114 | p = NULL; |
| 6115 | if (cookie) |
| 6116 | p = sched_core_find(rq_i, cookie); |
Joel Fernandes (Google) | 7afbba1 | 2020-11-17 18:19:42 -0500 | [diff] [blame] | 6117 | if (!p) |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6118 | p = idle_sched_class.pick_task(rq_i); |
| 6119 | } |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6120 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6121 | rq_i->core_pick = p; |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 6122 | |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6123 | if (p == rq_i->idle) { |
| 6124 | if (rq_i->nr_running) { |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 6125 | rq->core->core_forceidle_count++; |
Joel Fernandes (Google) | c6047c2 | 2020-11-17 18:19:39 -0500 | [diff] [blame] | 6126 | if (!fi_before) |
| 6127 | rq->core->core_forceidle_seq++; |
| 6128 | } |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6129 | } else { |
| 6130 | occ++; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6131 | } |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6132 | } |
| 6133 | |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 6134 | if (schedstat_enabled() && rq->core->core_forceidle_count) { |
Cruz Zhao | b171501 | 2022-01-11 17:55:59 +0800 | [diff] [blame] | 6135 | rq->core->core_forceidle_start = rq_clock(rq->core); |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 6136 | rq->core->core_forceidle_occupation = occ; |
| 6137 | } |
| 6138 | |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6139 | rq->core->core_pick_seq = rq->core->core_task_seq; |
| 6140 | next = rq->core_pick; |
| 6141 | rq->core_sched_seq = rq->core->core_pick_seq; |
| 6142 | |
| 6143 | /* Something should have been selected for current CPU */ |
| 6144 | WARN_ON_ONCE(!next); |
| 6145 | |
| 6146 | /* |
| 6147 | * Reschedule siblings |
| 6148 | * |
| 6149 | * NOTE: L1TF -- at this point we're no longer running the old task and |
| 6150 | * sending an IPI (below) ensures the sibling will no longer be running |
| 6151 | * their task. This ensures there is no inter-sibling overlap between |
| 6152 | * non-matching user state. |
| 6153 | */ |
| 6154 | for_each_cpu(i, smt_mask) { |
Peter Zijlstra | bc9ffef | 2021-08-24 11:05:47 +0200 | [diff] [blame] | 6155 | rq_i = cpu_rq(i); |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6156 | |
| 6157 | /* |
| 6158 | * An online sibling might have gone offline before a task |
| 6159 | * could be picked for it, or it might be offline but later |
| 6160 | * happen to come online, but its too late and nothing was |
| 6161 | * picked for it. That's Ok - it will pick tasks for itself, |
| 6162 | * so ignore it. |
| 6163 | */ |
| 6164 | if (!rq_i->core_pick) |
| 6165 | continue; |
| 6166 | |
Joel Fernandes (Google) | c6047c2 | 2020-11-17 18:19:39 -0500 | [diff] [blame] | 6167 | /* |
| 6168 | * Update for new !FI->FI transitions, or if continuing to be in !FI: |
| 6169 | * fi_before fi update? |
| 6170 | * 0 0 1 |
| 6171 | * 0 1 1 |
| 6172 | * 1 0 1 |
| 6173 | * 1 1 0 |
| 6174 | */ |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 6175 | if (!(fi_before && rq->core->core_forceidle_count)) |
| 6176 | task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6177 | |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 6178 | rq_i->core_pick->core_occupation = occ; |
| 6179 | |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6180 | if (i == cpu) { |
| 6181 | rq_i->core_pick = NULL; |
| 6182 | continue; |
| 6183 | } |
| 6184 | |
| 6185 | /* Did we break L1TF mitigation requirements? */ |
| 6186 | WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick)); |
| 6187 | |
| 6188 | if (rq_i->curr == rq_i->core_pick) { |
| 6189 | rq_i->core_pick = NULL; |
| 6190 | continue; |
| 6191 | } |
| 6192 | |
| 6193 | resched_curr(rq_i); |
| 6194 | } |
| 6195 | |
Peter Zijlstra | 5b6547e | 2022-03-16 22:03:41 +0100 | [diff] [blame] | 6196 | out_set_next: |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6197 | set_next_task(rq, next); |
Peter Zijlstra | 5b6547e | 2022-03-16 22:03:41 +0100 | [diff] [blame] | 6198 | out: |
| 6199 | if (rq->core->core_forceidle_count && next == rq->idle) |
| 6200 | queue_core_balance(rq); |
| 6201 | |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6202 | return next; |
| 6203 | } |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6204 | |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 6205 | static bool try_steal_cookie(int this, int that) |
| 6206 | { |
| 6207 | struct rq *dst = cpu_rq(this), *src = cpu_rq(that); |
| 6208 | struct task_struct *p; |
| 6209 | unsigned long cookie; |
| 6210 | bool success = false; |
| 6211 | |
| 6212 | local_irq_disable(); |
| 6213 | double_rq_lock(dst, src); |
| 6214 | |
| 6215 | cookie = dst->core->core_cookie; |
| 6216 | if (!cookie) |
| 6217 | goto unlock; |
| 6218 | |
| 6219 | if (dst->curr != dst->idle) |
| 6220 | goto unlock; |
| 6221 | |
| 6222 | p = sched_core_find(src, cookie); |
| 6223 | if (p == src->idle) |
| 6224 | goto unlock; |
| 6225 | |
| 6226 | do { |
| 6227 | if (p == src->core_pick || p == src->curr) |
| 6228 | goto next; |
| 6229 | |
Sebastian Andrzej Siewior | 386ef21 | 2022-03-17 15:51:32 +0100 | [diff] [blame] | 6230 | if (!is_cpu_allowed(p, this)) |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 6231 | goto next; |
| 6232 | |
| 6233 | if (p->core_occupation > dst->idle->core_occupation) |
| 6234 | goto next; |
| 6235 | |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 6236 | deactivate_task(src, p, 0); |
| 6237 | set_task_cpu(p, this); |
| 6238 | activate_task(dst, p, 0); |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 6239 | |
| 6240 | resched_curr(dst); |
| 6241 | |
| 6242 | success = true; |
| 6243 | break; |
| 6244 | |
| 6245 | next: |
| 6246 | p = sched_core_next(p, cookie); |
| 6247 | } while (p); |
| 6248 | |
| 6249 | unlock: |
| 6250 | double_rq_unlock(dst, src); |
| 6251 | local_irq_enable(); |
| 6252 | |
| 6253 | return success; |
| 6254 | } |
| 6255 | |
| 6256 | static bool steal_cookie_task(int cpu, struct sched_domain *sd) |
| 6257 | { |
| 6258 | int i; |
| 6259 | |
| 6260 | for_each_cpu_wrap(i, sched_domain_span(sd), cpu) { |
| 6261 | if (i == cpu) |
| 6262 | continue; |
| 6263 | |
| 6264 | if (need_resched()) |
| 6265 | break; |
| 6266 | |
| 6267 | if (try_steal_cookie(cpu, i)) |
| 6268 | return true; |
| 6269 | } |
| 6270 | |
| 6271 | return false; |
| 6272 | } |
| 6273 | |
| 6274 | static void sched_core_balance(struct rq *rq) |
| 6275 | { |
| 6276 | struct sched_domain *sd; |
| 6277 | int cpu = cpu_of(rq); |
| 6278 | |
| 6279 | preempt_disable(); |
| 6280 | rcu_read_lock(); |
| 6281 | raw_spin_rq_unlock_irq(rq); |
| 6282 | for_each_domain(cpu, sd) { |
| 6283 | if (need_resched()) |
| 6284 | break; |
| 6285 | |
| 6286 | if (steal_cookie_task(cpu, sd)) |
| 6287 | break; |
| 6288 | } |
| 6289 | raw_spin_rq_lock_irq(rq); |
| 6290 | rcu_read_unlock(); |
| 6291 | preempt_enable(); |
| 6292 | } |
| 6293 | |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 6294 | static DEFINE_PER_CPU(struct balance_callback, core_balance_head); |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 6295 | |
Peter Zijlstra | 5b6547e | 2022-03-16 22:03:41 +0100 | [diff] [blame] | 6296 | static void queue_core_balance(struct rq *rq) |
Peter Zijlstra | d2dfa17 | 2020-11-17 18:19:43 -0500 | [diff] [blame] | 6297 | { |
| 6298 | if (!sched_core_enabled(rq)) |
| 6299 | return; |
| 6300 | |
| 6301 | if (!rq->core->core_cookie) |
| 6302 | return; |
| 6303 | |
| 6304 | if (!rq->nr_running) /* not forced idle */ |
| 6305 | return; |
| 6306 | |
| 6307 | queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); |
| 6308 | } |
| 6309 | |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6310 | static void sched_core_cpu_starting(unsigned int cpu) |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6311 | { |
| 6312 | const struct cpumask *smt_mask = cpu_smt_mask(cpu); |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6313 | struct rq *rq = cpu_rq(cpu), *core_rq = NULL; |
| 6314 | unsigned long flags; |
| 6315 | int t; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6316 | |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6317 | sched_core_lock(cpu, &flags); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6318 | |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6319 | WARN_ON_ONCE(rq->core != rq); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6320 | |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6321 | /* if we're the first, we'll be our own leader */ |
| 6322 | if (cpumask_weight(smt_mask) == 1) |
| 6323 | goto unlock; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6324 | |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6325 | /* find the leader */ |
| 6326 | for_each_cpu(t, smt_mask) { |
| 6327 | if (t == cpu) |
| 6328 | continue; |
| 6329 | rq = cpu_rq(t); |
| 6330 | if (rq->core == rq) { |
| 6331 | core_rq = rq; |
| 6332 | break; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6333 | } |
| 6334 | } |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6335 | |
| 6336 | if (WARN_ON_ONCE(!core_rq)) /* whoopsie */ |
| 6337 | goto unlock; |
| 6338 | |
| 6339 | /* install and validate core_rq */ |
| 6340 | for_each_cpu(t, smt_mask) { |
| 6341 | rq = cpu_rq(t); |
| 6342 | |
| 6343 | if (t == cpu) |
| 6344 | rq->core = core_rq; |
| 6345 | |
| 6346 | WARN_ON_ONCE(rq->core != core_rq); |
| 6347 | } |
| 6348 | |
| 6349 | unlock: |
| 6350 | sched_core_unlock(cpu, &flags); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6351 | } |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6352 | |
| 6353 | static void sched_core_cpu_deactivate(unsigned int cpu) |
| 6354 | { |
| 6355 | const struct cpumask *smt_mask = cpu_smt_mask(cpu); |
| 6356 | struct rq *rq = cpu_rq(cpu), *core_rq = NULL; |
| 6357 | unsigned long flags; |
| 6358 | int t; |
| 6359 | |
| 6360 | sched_core_lock(cpu, &flags); |
| 6361 | |
| 6362 | /* if we're the last man standing, nothing to do */ |
| 6363 | if (cpumask_weight(smt_mask) == 1) { |
| 6364 | WARN_ON_ONCE(rq->core != rq); |
| 6365 | goto unlock; |
| 6366 | } |
| 6367 | |
| 6368 | /* if we're not the leader, nothing to do */ |
| 6369 | if (rq->core != rq) |
| 6370 | goto unlock; |
| 6371 | |
| 6372 | /* find a new leader */ |
| 6373 | for_each_cpu(t, smt_mask) { |
| 6374 | if (t == cpu) |
| 6375 | continue; |
| 6376 | core_rq = cpu_rq(t); |
| 6377 | break; |
| 6378 | } |
| 6379 | |
| 6380 | if (WARN_ON_ONCE(!core_rq)) /* impossible */ |
| 6381 | goto unlock; |
| 6382 | |
| 6383 | /* copy the shared state to the new leader */ |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 6384 | core_rq->core_task_seq = rq->core_task_seq; |
| 6385 | core_rq->core_pick_seq = rq->core_pick_seq; |
| 6386 | core_rq->core_cookie = rq->core_cookie; |
| 6387 | core_rq->core_forceidle_count = rq->core_forceidle_count; |
| 6388 | core_rq->core_forceidle_seq = rq->core_forceidle_seq; |
| 6389 | core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; |
| 6390 | |
| 6391 | /* |
| 6392 | * Accounting edge for forced idle is handled in pick_next_task(). |
| 6393 | * Don't need another one here, since the hotplug thread shouldn't |
| 6394 | * have a cookie. |
| 6395 | */ |
| 6396 | core_rq->core_forceidle_start = 0; |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6397 | |
| 6398 | /* install new leader */ |
| 6399 | for_each_cpu(t, smt_mask) { |
| 6400 | rq = cpu_rq(t); |
| 6401 | rq->core = core_rq; |
| 6402 | } |
| 6403 | |
| 6404 | unlock: |
| 6405 | sched_core_unlock(cpu, &flags); |
| 6406 | } |
| 6407 | |
| 6408 | static inline void sched_core_cpu_dying(unsigned int cpu) |
| 6409 | { |
| 6410 | struct rq *rq = cpu_rq(cpu); |
| 6411 | |
| 6412 | if (rq->core != rq) |
| 6413 | rq->core = rq; |
| 6414 | } |
| 6415 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6416 | #else /* !CONFIG_SCHED_CORE */ |
| 6417 | |
| 6418 | static inline void sched_core_cpu_starting(unsigned int cpu) {} |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 6419 | static inline void sched_core_cpu_deactivate(unsigned int cpu) {} |
| 6420 | static inline void sched_core_cpu_dying(unsigned int cpu) {} |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6421 | |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 6422 | static struct task_struct * |
| 6423 | pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) |
| 6424 | { |
| 6425 | return __pick_next_task(rq, prev, rf); |
| 6426 | } |
| 6427 | |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 6428 | #endif /* CONFIG_SCHED_CORE */ |
| 6429 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6430 | /* |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6431 | * Constants for the sched_mode argument of __schedule(). |
| 6432 | * |
| 6433 | * The mode argument allows RT enabled kernels to differentiate a |
| 6434 | * preemption from blocking on an 'sleeping' spin/rwlock. Note that |
| 6435 | * SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to |
| 6436 | * optimize the AND operation out and just check for zero. |
| 6437 | */ |
| 6438 | #define SM_NONE 0x0 |
| 6439 | #define SM_PREEMPT 0x1 |
Thomas Gleixner | 6991436 | 2021-08-15 23:27:48 +0200 | [diff] [blame] | 6440 | #define SM_RTLOCK_WAIT 0x2 |
| 6441 | |
| 6442 | #ifndef CONFIG_PREEMPT_RT |
| 6443 | # define SM_MASK_PREEMPT (~0U) |
| 6444 | #else |
| 6445 | # define SM_MASK_PREEMPT SM_PREEMPT |
| 6446 | #endif |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6447 | |
| 6448 | /* |
Thomas Gleixner | c259e01 | 2011-06-22 19:47:00 +0200 | [diff] [blame] | 6449 | * __schedule() is the main scheduler function. |
Pekka Enberg | edde96e | 2012-08-04 11:49:47 +0300 | [diff] [blame] | 6450 | * |
| 6451 | * The main means of driving the scheduler and thus entering this function are: |
| 6452 | * |
| 6453 | * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. |
| 6454 | * |
| 6455 | * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return |
| 6456 | * paths. For example, see arch/x86/entry_64.S. |
| 6457 | * |
| 6458 | * To drive preemption between tasks, the scheduler sets the flag in timer |
| 6459 | * interrupt handler scheduler_tick(). |
| 6460 | * |
| 6461 | * 3. Wakeups don't really cause entry into schedule(). They add a |
| 6462 | * task to the run-queue and that's it. |
| 6463 | * |
| 6464 | * Now, if the new task added to the run-queue preempts the current |
| 6465 | * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets |
| 6466 | * called on the nearest possible occasion: |
| 6467 | * |
Thomas Gleixner | c1a280b | 2019-07-26 23:19:37 +0200 | [diff] [blame] | 6468 | * - If the kernel is preemptible (CONFIG_PREEMPTION=y): |
Pekka Enberg | edde96e | 2012-08-04 11:49:47 +0300 | [diff] [blame] | 6469 | * |
| 6470 | * - in syscall or exception context, at the next outmost |
| 6471 | * preempt_enable(). (this might be as soon as the wake_up()'s |
| 6472 | * spin_unlock()!) |
| 6473 | * |
| 6474 | * - in IRQ context, return from interrupt-handler to |
| 6475 | * preemptible context |
| 6476 | * |
Thomas Gleixner | c1a280b | 2019-07-26 23:19:37 +0200 | [diff] [blame] | 6477 | * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) |
Pekka Enberg | edde96e | 2012-08-04 11:49:47 +0300 | [diff] [blame] | 6478 | * then at the next: |
| 6479 | * |
| 6480 | * - cond_resched() call |
| 6481 | * - explicit schedule() call |
| 6482 | * - return from syscall or exception to user-space |
| 6483 | * - return from interrupt-handler to user-space |
Frederic Weisbecker | bfd9b2b | 2015-01-28 01:24:09 +0100 | [diff] [blame] | 6484 | * |
Frederic Weisbecker | b30f0e3 | 2015-05-12 16:41:49 +0200 | [diff] [blame] | 6485 | * WARNING: must be called with preemption disabled! |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6486 | */ |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6487 | static void __sched notrace __schedule(unsigned int sched_mode) |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6488 | { |
| 6489 | struct task_struct *prev, *next; |
Harvey Harrison | 67ca7bd | 2008-02-15 09:56:36 -0800 | [diff] [blame] | 6490 | unsigned long *switch_count; |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6491 | unsigned long prev_state; |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 6492 | struct rq_flags rf; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6493 | struct rq *rq; |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 6494 | int cpu; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6495 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6496 | cpu = smp_processor_id(); |
| 6497 | rq = cpu_rq(cpu); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6498 | prev = rq->curr; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6499 | |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6500 | schedule_debug(prev, !!sched_mode); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6501 | |
Juri Lelli | e0ee463 | 2021-02-08 08:35:54 +0100 | [diff] [blame] | 6502 | if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) |
Mike Galbraith | f333fdc | 2008-05-12 21:20:55 +0200 | [diff] [blame] | 6503 | hrtick_clear(rq); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 6504 | |
Paul E. McKenney | 46a5d16 | 2015-10-07 09:10:48 -0700 | [diff] [blame] | 6505 | local_irq_disable(); |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6506 | rcu_note_context_switch(!!sched_mode); |
Paul E. McKenney | 46a5d16 | 2015-10-07 09:10:48 -0700 | [diff] [blame] | 6507 | |
Oleg Nesterov | e0acd0a | 2013-08-12 18:14:00 +0200 | [diff] [blame] | 6508 | /* |
| 6509 | * Make sure that signal_pending_state()->signal_pending() below |
| 6510 | * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6511 | * done by the caller to avoid the race with signal_wake_up(): |
Mathieu Desnoyers | 306e060 | 2018-01-29 15:20:12 -0500 | [diff] [blame] | 6512 | * |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6513 | * __set_current_state(@state) signal_wake_up() |
| 6514 | * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) |
| 6515 | * wake_up_state(p, state) |
| 6516 | * LOCK rq->lock LOCK p->pi_state |
| 6517 | * smp_mb__after_spinlock() smp_mb__after_spinlock() |
| 6518 | * if (signal_pending_state()) if (p->state & @state) |
| 6519 | * |
| 6520 | * Also, the membarrier system call requires a full memory barrier |
Mathieu Desnoyers | 306e060 | 2018-01-29 15:20:12 -0500 | [diff] [blame] | 6521 | * after coming from user-space, before storing to rq->curr. |
Oleg Nesterov | e0acd0a | 2013-08-12 18:14:00 +0200 | [diff] [blame] | 6522 | */ |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 6523 | rq_lock(rq, &rf); |
Peter Zijlstra | d89e588c | 2016-09-05 11:37:53 +0200 | [diff] [blame] | 6524 | smp_mb__after_spinlock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6525 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 6526 | /* Promote REQ to ACT */ |
| 6527 | rq->clock_update_flags <<= 1; |
Peter Zijlstra | bce4dc8 | 2017-02-21 14:40:35 +0100 | [diff] [blame] | 6528 | update_rq_clock(rq); |
Peter Zijlstra | 9edfbfe | 2015-01-05 11:18:11 +0100 | [diff] [blame] | 6529 | |
Oleg Nesterov | 246d86b | 2010-05-19 14:57:11 +0200 | [diff] [blame] | 6530 | switch_count = &prev->nivcsw; |
Peter Zijlstra | d136122 | 2020-07-20 17:20:21 +0200 | [diff] [blame] | 6531 | |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6532 | /* |
Peter Zijlstra | d136122 | 2020-07-20 17:20:21 +0200 | [diff] [blame] | 6533 | * We must load prev->state once (task_struct::state is volatile), such |
Eric W. Biederman | 2500ad1 | 2022-04-29 08:43:34 -0500 | [diff] [blame] | 6534 | * that we form a control dependency vs deactivate_task() below. |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6535 | */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 6536 | prev_state = READ_ONCE(prev->__state); |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6537 | if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) { |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6538 | if (signal_pending_state(prev_state, prev)) { |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 6539 | WRITE_ONCE(prev->__state, TASK_RUNNING); |
Tejun Heo | 21aa9af | 2010-06-08 21:40:37 +0200 | [diff] [blame] | 6540 | } else { |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6541 | prev->sched_contributes_to_load = |
| 6542 | (prev_state & TASK_UNINTERRUPTIBLE) && |
| 6543 | !(prev_state & TASK_NOLOAD) && |
Peter Zijlstra | f5d39b0 | 2022-08-22 13:18:22 +0200 | [diff] [blame] | 6544 | !(prev_state & TASK_FROZEN); |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6545 | |
| 6546 | if (prev->sched_contributes_to_load) |
| 6547 | rq->nr_uninterruptible++; |
| 6548 | |
| 6549 | /* |
| 6550 | * __schedule() ttwu() |
Peter Zijlstra | d136122 | 2020-07-20 17:20:21 +0200 | [diff] [blame] | 6551 | * prev_state = prev->state; if (p->on_rq && ...) |
| 6552 | * if (prev_state) goto out; |
| 6553 | * p->on_rq = 0; smp_acquire__after_ctrl_dep(); |
| 6554 | * p->state = TASK_WAKING |
| 6555 | * |
| 6556 | * Where __schedule() and ttwu() have matching control dependencies. |
Peter Zijlstra | dbfb089 | 2020-07-03 12:40:33 +0200 | [diff] [blame] | 6557 | * |
| 6558 | * After this, schedule() must not care about p->state any more. |
| 6559 | */ |
Peter Zijlstra | bce4dc8 | 2017-02-21 14:40:35 +0100 | [diff] [blame] | 6560 | deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); |
Peter Zijlstra | 2acca55 | 2011-04-05 17:23:50 +0200 | [diff] [blame] | 6561 | |
Tejun Heo | e33a9bb | 2016-12-07 15:48:41 -0500 | [diff] [blame] | 6562 | if (prev->in_iowait) { |
| 6563 | atomic_inc(&rq->nr_iowait); |
| 6564 | delayacct_blkio_start(); |
| 6565 | } |
Tejun Heo | 21aa9af | 2010-06-08 21:40:37 +0200 | [diff] [blame] | 6566 | } |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 6567 | switch_count = &prev->nvcsw; |
| 6568 | } |
| 6569 | |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 6570 | next = pick_next_task(rq, prev, &rf); |
Mike Galbraith | f26f9af | 2010-12-08 11:05:42 +0100 | [diff] [blame] | 6571 | clear_tsk_need_resched(prev); |
Peter Zijlstra | f27dde8 | 2013-08-14 14:55:31 +0200 | [diff] [blame] | 6572 | clear_preempt_need_resched(); |
Paul Turner | c006fac | 2021-04-16 14:29:36 -0700 | [diff] [blame] | 6573 | #ifdef CONFIG_SCHED_DEBUG |
| 6574 | rq->last_seen_need_resched_ns = 0; |
| 6575 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6576 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6577 | if (likely(prev != next)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6578 | rq->nr_switches++; |
Eric W. Biederman | 5311a98 | 2019-09-14 07:35:02 -0500 | [diff] [blame] | 6579 | /* |
| 6580 | * RCU users of rcu_dereference(rq->curr) may not see |
| 6581 | * changes to task_struct made by pick_next_task(). |
| 6582 | */ |
| 6583 | RCU_INIT_POINTER(rq->curr, next); |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 6584 | /* |
| 6585 | * The membarrier system call requires each architecture |
| 6586 | * to have a full memory barrier after updating |
Mathieu Desnoyers | 306e060 | 2018-01-29 15:20:12 -0500 | [diff] [blame] | 6587 | * rq->curr, before returning to user-space. |
| 6588 | * |
| 6589 | * Here are the schemes providing that barrier on the |
| 6590 | * various architectures: |
| 6591 | * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC. |
| 6592 | * switch_mm() rely on membarrier_arch_switch_mm() on PowerPC. |
| 6593 | * - finish_lock_switch() for weakly-ordered |
| 6594 | * architectures where spin_unlock is a full barrier, |
| 6595 | * - switch_to() for arm64 (weakly-ordered, spin_unlock |
| 6596 | * is a RELEASE barrier), |
Mathieu Desnoyers | 22e4ebb | 2017-07-28 16:40:40 -0400 | [diff] [blame] | 6597 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6598 | ++*switch_count; |
| 6599 | |
Peter Zijlstra | af44990 | 2020-09-17 10:38:30 +0200 | [diff] [blame] | 6600 | migrate_disable_switch(rq, prev); |
Johannes Weiner | b05e75d | 2020-03-16 15:13:31 -0400 | [diff] [blame] | 6601 | psi_sched_switch(prev, next, !task_on_rq_queued(prev)); |
| 6602 | |
Delyan Kratunov | 9c2136b | 2022-05-11 18:28:36 +0000 | [diff] [blame] | 6603 | trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 6604 | |
| 6605 | /* Also unlocks the rq: */ |
| 6606 | rq = context_switch(rq, prev, next, &rf); |
Peter Zijlstra | cbce1a6 | 2015-06-11 14:46:54 +0200 | [diff] [blame] | 6607 | } else { |
Matt Fleming | cb42c9a | 2016-09-21 14:38:13 +0100 | [diff] [blame] | 6608 | rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6609 | |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 6610 | rq_unpin_lock(rq, &rf); |
| 6611 | __balance_callbacks(rq); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 6612 | raw_spin_rq_unlock_irq(rq); |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 6613 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6614 | } |
Thomas Gleixner | c259e01 | 2011-06-22 19:47:00 +0200 | [diff] [blame] | 6615 | |
Peter Zijlstra | 9af6528 | 2016-09-13 18:37:29 +0200 | [diff] [blame] | 6616 | void __noreturn do_task_dead(void) |
| 6617 | { |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 6618 | /* Causes final put_task_struct in finish_task_switch(): */ |
Peter Zijlstra | b5bf9a9 | 2018-04-30 14:51:01 +0200 | [diff] [blame] | 6619 | set_special_state(TASK_DEAD); |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 6620 | |
| 6621 | /* Tell freezer to ignore us: */ |
| 6622 | current->flags |= PF_NOFREEZE; |
| 6623 | |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6624 | __schedule(SM_NONE); |
Peter Zijlstra | 9af6528 | 2016-09-13 18:37:29 +0200 | [diff] [blame] | 6625 | BUG(); |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 6626 | |
| 6627 | /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ |
Peter Zijlstra | 9af6528 | 2016-09-13 18:37:29 +0200 | [diff] [blame] | 6628 | for (;;) |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 6629 | cpu_relax(); |
Peter Zijlstra | 9af6528 | 2016-09-13 18:37:29 +0200 | [diff] [blame] | 6630 | } |
| 6631 | |
Thomas Gleixner | 9c40cef2 | 2011-06-22 19:47:01 +0200 | [diff] [blame] | 6632 | static inline void sched_submit_work(struct task_struct *tsk) |
| 6633 | { |
Sebastian Andrzej Siewior | c1cecf8 | 2020-08-19 22:00:25 +0200 | [diff] [blame] | 6634 | unsigned int task_flags; |
| 6635 | |
Peter Zijlstra | b03fbd4 | 2021-06-11 10:28:12 +0200 | [diff] [blame] | 6636 | if (task_is_running(tsk)) |
Thomas Gleixner | 9c40cef2 | 2011-06-22 19:47:01 +0200 | [diff] [blame] | 6637 | return; |
Thomas Gleixner | 6d25be5 | 2019-03-13 17:55:48 +0100 | [diff] [blame] | 6638 | |
Sebastian Andrzej Siewior | c1cecf8 | 2020-08-19 22:00:25 +0200 | [diff] [blame] | 6639 | task_flags = tsk->flags; |
Thomas Gleixner | 6d25be5 | 2019-03-13 17:55:48 +0100 | [diff] [blame] | 6640 | /* |
Thomas Gleixner | b945efc | 2021-09-29 11:37:32 +0200 | [diff] [blame] | 6641 | * If a worker goes to sleep, notify and ask workqueue whether it |
| 6642 | * wants to wake up a task to maintain concurrency. |
Thomas Gleixner | 6d25be5 | 2019-03-13 17:55:48 +0100 | [diff] [blame] | 6643 | */ |
Sebastian Andrzej Siewior | c1cecf8 | 2020-08-19 22:00:25 +0200 | [diff] [blame] | 6644 | if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) { |
Sebastian Andrzej Siewior | c1cecf8 | 2020-08-19 22:00:25 +0200 | [diff] [blame] | 6645 | if (task_flags & PF_WQ_WORKER) |
Jens Axboe | 771b53d0 | 2019-10-22 10:25:58 -0600 | [diff] [blame] | 6646 | wq_worker_sleeping(tsk); |
| 6647 | else |
| 6648 | io_wq_worker_sleeping(tsk); |
Thomas Gleixner | 6d25be5 | 2019-03-13 17:55:48 +0100 | [diff] [blame] | 6649 | } |
| 6650 | |
John Keeping | 401e496 | 2022-07-08 17:27:02 +0100 | [diff] [blame] | 6651 | /* |
| 6652 | * spinlock and rwlock must not flush block requests. This will |
| 6653 | * deadlock if the callback attempts to acquire a lock which is |
| 6654 | * already acquired. |
| 6655 | */ |
| 6656 | SCHED_WARN_ON(current->__state & TASK_RTLOCK_WAIT); |
Sebastian Andrzej Siewior | b0fdc01 | 2019-08-16 18:06:26 +0200 | [diff] [blame] | 6657 | |
Thomas Gleixner | 9c40cef2 | 2011-06-22 19:47:01 +0200 | [diff] [blame] | 6658 | /* |
| 6659 | * If we are going to sleep and we have plugged IO queued, |
| 6660 | * make sure to submit it to avoid deadlocks. |
| 6661 | */ |
Christoph Hellwig | aa8dccc | 2022-01-27 08:05:49 +0100 | [diff] [blame] | 6662 | blk_flush_plug(tsk->plug, true); |
Thomas Gleixner | 9c40cef2 | 2011-06-22 19:47:01 +0200 | [diff] [blame] | 6663 | } |
| 6664 | |
Thomas Gleixner | 6d25be5 | 2019-03-13 17:55:48 +0100 | [diff] [blame] | 6665 | static void sched_update_worker(struct task_struct *tsk) |
| 6666 | { |
Jens Axboe | 771b53d0 | 2019-10-22 10:25:58 -0600 | [diff] [blame] | 6667 | if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { |
| 6668 | if (tsk->flags & PF_WQ_WORKER) |
| 6669 | wq_worker_running(tsk); |
| 6670 | else |
| 6671 | io_wq_worker_running(tsk); |
| 6672 | } |
Thomas Gleixner | 6d25be5 | 2019-03-13 17:55:48 +0100 | [diff] [blame] | 6673 | } |
| 6674 | |
Andi Kleen | 722a9f9 | 2014-05-02 00:44:38 +0200 | [diff] [blame] | 6675 | asmlinkage __visible void __sched schedule(void) |
Thomas Gleixner | c259e01 | 2011-06-22 19:47:00 +0200 | [diff] [blame] | 6676 | { |
Thomas Gleixner | 9c40cef2 | 2011-06-22 19:47:01 +0200 | [diff] [blame] | 6677 | struct task_struct *tsk = current; |
| 6678 | |
| 6679 | sched_submit_work(tsk); |
Frederic Weisbecker | bfd9b2b | 2015-01-28 01:24:09 +0100 | [diff] [blame] | 6680 | do { |
Frederic Weisbecker | b30f0e3 | 2015-05-12 16:41:49 +0200 | [diff] [blame] | 6681 | preempt_disable(); |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6682 | __schedule(SM_NONE); |
Frederic Weisbecker | b30f0e3 | 2015-05-12 16:41:49 +0200 | [diff] [blame] | 6683 | sched_preempt_enable_no_resched(); |
Frederic Weisbecker | bfd9b2b | 2015-01-28 01:24:09 +0100 | [diff] [blame] | 6684 | } while (need_resched()); |
Thomas Gleixner | 6d25be5 | 2019-03-13 17:55:48 +0100 | [diff] [blame] | 6685 | sched_update_worker(tsk); |
Thomas Gleixner | c259e01 | 2011-06-22 19:47:00 +0200 | [diff] [blame] | 6686 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6687 | EXPORT_SYMBOL(schedule); |
| 6688 | |
Steven Rostedt (VMware) | 8663eff | 2017-04-14 08:48:09 -0400 | [diff] [blame] | 6689 | /* |
| 6690 | * synchronize_rcu_tasks() makes sure that no task is stuck in preempted |
| 6691 | * state (have scheduled out non-voluntarily) by making sure that all |
| 6692 | * tasks have either left the run queue or have gone into user space. |
| 6693 | * As idle tasks do not do either, they must not ever be preempted |
| 6694 | * (schedule out non-voluntarily). |
| 6695 | * |
| 6696 | * schedule_idle() is similar to schedule_preempt_disable() except that it |
| 6697 | * never enables preemption because it does not call sched_submit_work(). |
| 6698 | */ |
| 6699 | void __sched schedule_idle(void) |
| 6700 | { |
| 6701 | /* |
| 6702 | * As this skips calling sched_submit_work(), which the idle task does |
| 6703 | * regardless because that function is a nop when the task is in a |
| 6704 | * TASK_RUNNING state, make sure this isn't used someplace that the |
| 6705 | * current task can be in any other state. Note, idle is always in the |
| 6706 | * TASK_RUNNING state. |
| 6707 | */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 6708 | WARN_ON_ONCE(current->__state); |
Steven Rostedt (VMware) | 8663eff | 2017-04-14 08:48:09 -0400 | [diff] [blame] | 6709 | do { |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6710 | __schedule(SM_NONE); |
Steven Rostedt (VMware) | 8663eff | 2017-04-14 08:48:09 -0400 | [diff] [blame] | 6711 | } while (need_resched()); |
| 6712 | } |
| 6713 | |
Frederic Weisbecker | 24a9c541 | 2022-06-08 16:40:24 +0200 | [diff] [blame] | 6714 | #if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) |
Andi Kleen | 722a9f9 | 2014-05-02 00:44:38 +0200 | [diff] [blame] | 6715 | asmlinkage __visible void __sched schedule_user(void) |
Frederic Weisbecker | 20ab65e3 | 2012-07-11 20:26:37 +0200 | [diff] [blame] | 6716 | { |
| 6717 | /* |
| 6718 | * If we come here after a random call to set_need_resched(), |
| 6719 | * or we have been woken up remotely but the IPI has not yet arrived, |
| 6720 | * we haven't yet exited the RCU idle mode. Do it here manually until |
| 6721 | * we find a better solution. |
Andy Lutomirski | 7cc78f8 | 2014-12-03 15:37:08 -0800 | [diff] [blame] | 6722 | * |
| 6723 | * NB: There are buggy callers of this function. Ideally we |
Frederic Weisbecker | c467ea7 | 2015-03-04 18:06:33 +0100 | [diff] [blame] | 6724 | * should warn if prev_state != CONTEXT_USER, but that will trigger |
Andy Lutomirski | 7cc78f8 | 2014-12-03 15:37:08 -0800 | [diff] [blame] | 6725 | * too frequently to make sense yet. |
Frederic Weisbecker | 20ab65e3 | 2012-07-11 20:26:37 +0200 | [diff] [blame] | 6726 | */ |
Andy Lutomirski | 7cc78f8 | 2014-12-03 15:37:08 -0800 | [diff] [blame] | 6727 | enum ctx_state prev_state = exception_enter(); |
Frederic Weisbecker | 20ab65e3 | 2012-07-11 20:26:37 +0200 | [diff] [blame] | 6728 | schedule(); |
Andy Lutomirski | 7cc78f8 | 2014-12-03 15:37:08 -0800 | [diff] [blame] | 6729 | exception_exit(prev_state); |
Frederic Weisbecker | 20ab65e3 | 2012-07-11 20:26:37 +0200 | [diff] [blame] | 6730 | } |
| 6731 | #endif |
| 6732 | |
Thomas Gleixner | c5491ea | 2011-03-21 12:09:35 +0100 | [diff] [blame] | 6733 | /** |
| 6734 | * schedule_preempt_disabled - called with preemption disabled |
| 6735 | * |
| 6736 | * Returns with preemption disabled. Note: preempt_count must be 1 |
| 6737 | */ |
| 6738 | void __sched schedule_preempt_disabled(void) |
| 6739 | { |
Thomas Gleixner | ba74c14 | 2011-03-21 13:32:17 +0100 | [diff] [blame] | 6740 | sched_preempt_enable_no_resched(); |
Thomas Gleixner | c5491ea | 2011-03-21 12:09:35 +0100 | [diff] [blame] | 6741 | schedule(); |
| 6742 | preempt_disable(); |
| 6743 | } |
| 6744 | |
Thomas Gleixner | 6991436 | 2021-08-15 23:27:48 +0200 | [diff] [blame] | 6745 | #ifdef CONFIG_PREEMPT_RT |
| 6746 | void __sched notrace schedule_rtlock(void) |
| 6747 | { |
| 6748 | do { |
| 6749 | preempt_disable(); |
| 6750 | __schedule(SM_RTLOCK_WAIT); |
| 6751 | sched_preempt_enable_no_resched(); |
| 6752 | } while (need_resched()); |
| 6753 | } |
| 6754 | NOKPROBE_SYMBOL(schedule_rtlock); |
| 6755 | #endif |
| 6756 | |
Frederic Weisbecker | 06b1f80 | 2015-02-16 19:20:07 +0100 | [diff] [blame] | 6757 | static void __sched notrace preempt_schedule_common(void) |
Frederic Weisbecker | a18b5d0 | 2015-01-22 18:08:04 +0100 | [diff] [blame] | 6758 | { |
| 6759 | do { |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 6760 | /* |
| 6761 | * Because the function tracer can trace preempt_count_sub() |
| 6762 | * and it also uses preempt_enable/disable_notrace(), if |
| 6763 | * NEED_RESCHED is set, the preempt_enable_notrace() called |
| 6764 | * by the function tracer will call this function again and |
| 6765 | * cause infinite recursion. |
| 6766 | * |
| 6767 | * Preemption must be disabled here before the function |
| 6768 | * tracer can trace. Break up preempt_disable() into two |
| 6769 | * calls. One to disable preemption without fear of being |
| 6770 | * traced. The other to still record the preemption latency, |
| 6771 | * which can also be traced by the function tracer. |
| 6772 | */ |
Peter Zijlstra | 499d795 | 2015-09-28 18:52:36 +0200 | [diff] [blame] | 6773 | preempt_disable_notrace(); |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 6774 | preempt_latency_start(1); |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6775 | __schedule(SM_PREEMPT); |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 6776 | preempt_latency_stop(1); |
Peter Zijlstra | 499d795 | 2015-09-28 18:52:36 +0200 | [diff] [blame] | 6777 | preempt_enable_no_resched_notrace(); |
Frederic Weisbecker | a18b5d0 | 2015-01-22 18:08:04 +0100 | [diff] [blame] | 6778 | |
| 6779 | /* |
| 6780 | * Check again in case we missed a preemption opportunity |
| 6781 | * between schedule and now. |
| 6782 | */ |
Frederic Weisbecker | a18b5d0 | 2015-01-22 18:08:04 +0100 | [diff] [blame] | 6783 | } while (need_resched()); |
| 6784 | } |
| 6785 | |
Thomas Gleixner | c1a280b | 2019-07-26 23:19:37 +0200 | [diff] [blame] | 6786 | #ifdef CONFIG_PREEMPTION |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6787 | /* |
Valentin Schneider | a49b4f4 | 2019-09-23 15:36:12 +0100 | [diff] [blame] | 6788 | * This is the entry point to schedule() from in-kernel preemption |
| 6789 | * off of preempt_enable. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6790 | */ |
Andi Kleen | 722a9f9 | 2014-05-02 00:44:38 +0200 | [diff] [blame] | 6791 | asmlinkage __visible void __sched notrace preempt_schedule(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6792 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6793 | /* |
| 6794 | * If there is a non-zero preempt_count or interrupts are disabled, |
Ingo Molnar | 41a2d6c | 2007-12-05 15:46:09 +0100 | [diff] [blame] | 6795 | * we do not want to preempt the current task. Just return.. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6796 | */ |
Frederic Weisbecker | fbb00b5 | 2013-06-19 23:56:22 +0200 | [diff] [blame] | 6797 | if (likely(!preemptible())) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6798 | return; |
Frederic Weisbecker | a18b5d0 | 2015-01-22 18:08:04 +0100 | [diff] [blame] | 6799 | preempt_schedule_common(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6800 | } |
Masami Hiramatsu | 376e242 | 2014-04-17 17:17:05 +0900 | [diff] [blame] | 6801 | NOKPROBE_SYMBOL(preempt_schedule); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6802 | EXPORT_SYMBOL(preempt_schedule); |
Oleg Nesterov | 009f60e | 2014-10-05 22:23:22 +0200 | [diff] [blame] | 6803 | |
Peter Zijlstra (Intel) | 2c9a98d | 2021-01-18 15:12:21 +0100 | [diff] [blame] | 6804 | #ifdef CONFIG_PREEMPT_DYNAMIC |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 6805 | #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 6806 | #ifndef preempt_schedule_dynamic_enabled |
| 6807 | #define preempt_schedule_dynamic_enabled preempt_schedule |
| 6808 | #define preempt_schedule_dynamic_disabled NULL |
Peter Zijlstra (Intel) | 2c9a98d | 2021-01-18 15:12:21 +0100 | [diff] [blame] | 6809 | #endif |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 6810 | DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); |
Peter Zijlstra (Intel) | 2c9a98d | 2021-01-18 15:12:21 +0100 | [diff] [blame] | 6811 | EXPORT_STATIC_CALL_TRAMP(preempt_schedule); |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 6812 | #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) |
| 6813 | static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); |
| 6814 | void __sched notrace dynamic_preempt_schedule(void) |
| 6815 | { |
| 6816 | if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) |
| 6817 | return; |
| 6818 | preempt_schedule(); |
| 6819 | } |
| 6820 | NOKPROBE_SYMBOL(dynamic_preempt_schedule); |
| 6821 | EXPORT_SYMBOL(dynamic_preempt_schedule); |
Peter Zijlstra (Intel) | 2c9a98d | 2021-01-18 15:12:21 +0100 | [diff] [blame] | 6822 | #endif |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 6823 | #endif |
Peter Zijlstra (Intel) | 2c9a98d | 2021-01-18 15:12:21 +0100 | [diff] [blame] | 6824 | |
Oleg Nesterov | 009f60e | 2014-10-05 22:23:22 +0200 | [diff] [blame] | 6825 | /** |
Frederic Weisbecker | 4eaca0a | 2015-06-04 17:39:08 +0200 | [diff] [blame] | 6826 | * preempt_schedule_notrace - preempt_schedule called by tracing |
Oleg Nesterov | 009f60e | 2014-10-05 22:23:22 +0200 | [diff] [blame] | 6827 | * |
| 6828 | * The tracing infrastructure uses preempt_enable_notrace to prevent |
| 6829 | * recursion and tracing preempt enabling caused by the tracing |
| 6830 | * infrastructure itself. But as tracing can happen in areas coming |
| 6831 | * from userspace or just about to enter userspace, a preempt enable |
| 6832 | * can occur before user_exit() is called. This will cause the scheduler |
| 6833 | * to be called when the system is still in usermode. |
| 6834 | * |
| 6835 | * To prevent this, the preempt_enable_notrace will use this function |
| 6836 | * instead of preempt_schedule() to exit user context if needed before |
| 6837 | * calling the scheduler. |
| 6838 | */ |
Frederic Weisbecker | 4eaca0a | 2015-06-04 17:39:08 +0200 | [diff] [blame] | 6839 | asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) |
Oleg Nesterov | 009f60e | 2014-10-05 22:23:22 +0200 | [diff] [blame] | 6840 | { |
| 6841 | enum ctx_state prev_ctx; |
| 6842 | |
| 6843 | if (likely(!preemptible())) |
| 6844 | return; |
| 6845 | |
| 6846 | do { |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 6847 | /* |
| 6848 | * Because the function tracer can trace preempt_count_sub() |
| 6849 | * and it also uses preempt_enable/disable_notrace(), if |
| 6850 | * NEED_RESCHED is set, the preempt_enable_notrace() called |
| 6851 | * by the function tracer will call this function again and |
| 6852 | * cause infinite recursion. |
| 6853 | * |
| 6854 | * Preemption must be disabled here before the function |
| 6855 | * tracer can trace. Break up preempt_disable() into two |
| 6856 | * calls. One to disable preemption without fear of being |
| 6857 | * traced. The other to still record the preemption latency, |
| 6858 | * which can also be traced by the function tracer. |
| 6859 | */ |
Peter Zijlstra | 3d8f74d | 2015-09-28 18:09:19 +0200 | [diff] [blame] | 6860 | preempt_disable_notrace(); |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 6861 | preempt_latency_start(1); |
Oleg Nesterov | 009f60e | 2014-10-05 22:23:22 +0200 | [diff] [blame] | 6862 | /* |
| 6863 | * Needs preempt disabled in case user_exit() is traced |
| 6864 | * and the tracer calls preempt_enable_notrace() causing |
| 6865 | * an infinite recursion. |
| 6866 | */ |
| 6867 | prev_ctx = exception_enter(); |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6868 | __schedule(SM_PREEMPT); |
Oleg Nesterov | 009f60e | 2014-10-05 22:23:22 +0200 | [diff] [blame] | 6869 | exception_exit(prev_ctx); |
| 6870 | |
Steven Rostedt | 47252cf | 2016-03-21 11:23:39 -0400 | [diff] [blame] | 6871 | preempt_latency_stop(1); |
Peter Zijlstra | 3d8f74d | 2015-09-28 18:09:19 +0200 | [diff] [blame] | 6872 | preempt_enable_no_resched_notrace(); |
Oleg Nesterov | 009f60e | 2014-10-05 22:23:22 +0200 | [diff] [blame] | 6873 | } while (need_resched()); |
| 6874 | } |
Frederic Weisbecker | 4eaca0a | 2015-06-04 17:39:08 +0200 | [diff] [blame] | 6875 | EXPORT_SYMBOL_GPL(preempt_schedule_notrace); |
Oleg Nesterov | 009f60e | 2014-10-05 22:23:22 +0200 | [diff] [blame] | 6876 | |
Peter Zijlstra (Intel) | 2c9a98d | 2021-01-18 15:12:21 +0100 | [diff] [blame] | 6877 | #ifdef CONFIG_PREEMPT_DYNAMIC |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 6878 | #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 6879 | #ifndef preempt_schedule_notrace_dynamic_enabled |
| 6880 | #define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace |
| 6881 | #define preempt_schedule_notrace_dynamic_disabled NULL |
| 6882 | #endif |
| 6883 | DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); |
Peter Zijlstra | ef72661 | 2021-01-25 16:26:50 +0100 | [diff] [blame] | 6884 | EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 6885 | #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) |
| 6886 | static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); |
| 6887 | void __sched notrace dynamic_preempt_schedule_notrace(void) |
| 6888 | { |
| 6889 | if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) |
| 6890 | return; |
| 6891 | preempt_schedule_notrace(); |
| 6892 | } |
| 6893 | NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); |
| 6894 | EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); |
| 6895 | #endif |
Peter Zijlstra (Intel) | 2c9a98d | 2021-01-18 15:12:21 +0100 | [diff] [blame] | 6896 | #endif |
| 6897 | |
Thomas Gleixner | c1a280b | 2019-07-26 23:19:37 +0200 | [diff] [blame] | 6898 | #endif /* CONFIG_PREEMPTION */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6899 | |
| 6900 | /* |
Valentin Schneider | a49b4f4 | 2019-09-23 15:36:12 +0100 | [diff] [blame] | 6901 | * This is the entry point to schedule() from kernel preemption |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6902 | * off of irq context. |
| 6903 | * Note, that this is called and return with irqs disabled. This will |
| 6904 | * protect us against recursive calling from irq. |
| 6905 | */ |
Andi Kleen | 722a9f9 | 2014-05-02 00:44:38 +0200 | [diff] [blame] | 6906 | asmlinkage __visible void __sched preempt_schedule_irq(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6907 | { |
Frederic Weisbecker | b22366c | 2013-02-24 12:59:30 +0100 | [diff] [blame] | 6908 | enum ctx_state prev_state; |
Ingo Molnar | 6478d88 | 2008-01-25 21:08:33 +0100 | [diff] [blame] | 6909 | |
Andreas Mohr | 2ed6e34 | 2006-07-10 04:43:52 -0700 | [diff] [blame] | 6910 | /* Catch callers which need to be fixed */ |
Peter Zijlstra | f27dde8 | 2013-08-14 14:55:31 +0200 | [diff] [blame] | 6911 | BUG_ON(preempt_count() || !irqs_disabled()); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6912 | |
Frederic Weisbecker | b22366c | 2013-02-24 12:59:30 +0100 | [diff] [blame] | 6913 | prev_state = exception_enter(); |
| 6914 | |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 6915 | do { |
Peter Zijlstra | 3d8f74d | 2015-09-28 18:09:19 +0200 | [diff] [blame] | 6916 | preempt_disable(); |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 6917 | local_irq_enable(); |
Thomas Gleixner | b4bfa3f | 2021-08-15 23:27:46 +0200 | [diff] [blame] | 6918 | __schedule(SM_PREEMPT); |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 6919 | local_irq_disable(); |
Peter Zijlstra | 3d8f74d | 2015-09-28 18:09:19 +0200 | [diff] [blame] | 6920 | sched_preempt_enable_no_resched(); |
Lai Jiangshan | 5ed0cec | 2009-03-06 19:40:20 +0800 | [diff] [blame] | 6921 | } while (need_resched()); |
Frederic Weisbecker | b22366c | 2013-02-24 12:59:30 +0100 | [diff] [blame] | 6922 | |
| 6923 | exception_exit(prev_state); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6924 | } |
| 6925 | |
Ingo Molnar | ac6424b | 2017-06-20 12:06:13 +0200 | [diff] [blame] | 6926 | int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, |
Ingo Molnar | 95cdf3b | 2005-09-10 00:26:11 -0700 | [diff] [blame] | 6927 | void *key) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6928 | { |
Chris Wilson | 062d3f9 | 2020-07-23 21:10:42 +0100 | [diff] [blame] | 6929 | WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC); |
Peter Zijlstra | 63859d4 | 2009-09-15 19:14:42 +0200 | [diff] [blame] | 6930 | return try_to_wake_up(curr->private, mode, wake_flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6931 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6932 | EXPORT_SYMBOL(default_wake_function); |
| 6933 | |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 6934 | static void __setscheduler_prio(struct task_struct *p, int prio) |
| 6935 | { |
| 6936 | if (dl_prio(prio)) |
| 6937 | p->sched_class = &dl_sched_class; |
| 6938 | else if (rt_prio(prio)) |
| 6939 | p->sched_class = &rt_sched_class; |
| 6940 | else |
| 6941 | p->sched_class = &fair_sched_class; |
| 6942 | |
| 6943 | p->prio = prio; |
| 6944 | } |
| 6945 | |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 6946 | #ifdef CONFIG_RT_MUTEXES |
| 6947 | |
Peter Zijlstra | acd5862 | 2017-03-23 15:56:11 +0100 | [diff] [blame] | 6948 | static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) |
| 6949 | { |
| 6950 | if (pi_task) |
| 6951 | prio = min(prio, pi_task->prio); |
| 6952 | |
| 6953 | return prio; |
| 6954 | } |
| 6955 | |
| 6956 | static inline int rt_effective_prio(struct task_struct *p, int prio) |
| 6957 | { |
| 6958 | struct task_struct *pi_task = rt_mutex_get_top_task(p); |
| 6959 | |
| 6960 | return __rt_effective_prio(pi_task, prio); |
| 6961 | } |
| 6962 | |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 6963 | /* |
| 6964 | * rt_mutex_setprio - set the current priority of a task |
Peter Zijlstra | acd5862 | 2017-03-23 15:56:11 +0100 | [diff] [blame] | 6965 | * @p: task to boost |
| 6966 | * @pi_task: donor task |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 6967 | * |
| 6968 | * This function changes the 'effective' priority of a task. It does |
| 6969 | * not touch ->normal_prio like __setscheduler(). |
| 6970 | * |
Thomas Gleixner | c365c29 | 2014-02-07 20:58:42 +0100 | [diff] [blame] | 6971 | * Used by the rt_mutex code to implement priority inheritance |
| 6972 | * logic. Call site only calls if the priority of the task changed. |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 6973 | */ |
Peter Zijlstra | acd5862 | 2017-03-23 15:56:11 +0100 | [diff] [blame] | 6974 | void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 6975 | { |
Peter Zijlstra | acd5862 | 2017-03-23 15:56:11 +0100 | [diff] [blame] | 6976 | int prio, oldprio, queued, running, queue_flag = |
Peter Zijlstra | 7a57f32 | 2017-02-21 14:47:02 +0100 | [diff] [blame] | 6977 | DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; |
Thomas Gleixner | 83ab0aa | 2010-02-17 09:05:48 +0100 | [diff] [blame] | 6978 | const struct sched_class *prev_class; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 6979 | struct rq_flags rf; |
| 6980 | struct rq *rq; |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 6981 | |
Peter Zijlstra | acd5862 | 2017-03-23 15:56:11 +0100 | [diff] [blame] | 6982 | /* XXX used to be waiter->prio, not waiter->task->prio */ |
| 6983 | prio = __rt_effective_prio(pi_task, p->normal_prio); |
| 6984 | |
| 6985 | /* |
| 6986 | * If nothing changed; bail early. |
| 6987 | */ |
| 6988 | if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) |
| 6989 | return; |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 6990 | |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 6991 | rq = __task_rq_lock(p, &rf); |
Peter Zijlstra | 80f5c1b | 2016-10-03 16:28:37 +0200 | [diff] [blame] | 6992 | update_rq_clock(rq); |
Peter Zijlstra | acd5862 | 2017-03-23 15:56:11 +0100 | [diff] [blame] | 6993 | /* |
| 6994 | * Set under pi_lock && rq->lock, such that the value can be used under |
| 6995 | * either lock. |
| 6996 | * |
| 6997 | * Note that there is loads of tricky to make this pointer cache work |
| 6998 | * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to |
| 6999 | * ensure a task is de-boosted (pi_task is set to NULL) before the |
| 7000 | * task is allowed to run again (and can exit). This ensures the pointer |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 7001 | * points to a blocked task -- which guarantees the task is present. |
Peter Zijlstra | acd5862 | 2017-03-23 15:56:11 +0100 | [diff] [blame] | 7002 | */ |
| 7003 | p->pi_top_task = pi_task; |
| 7004 | |
| 7005 | /* |
| 7006 | * For FIFO/RR we only need to set prio, if that matches we're done. |
| 7007 | */ |
| 7008 | if (prio == p->prio && !dl_prio(prio)) |
| 7009 | goto out_unlock; |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 7010 | |
Thomas Gleixner | 1c4dd99 | 2011-06-06 20:07:38 +0200 | [diff] [blame] | 7011 | /* |
| 7012 | * Idle task boosting is a nono in general. There is one |
| 7013 | * exception, when PREEMPT_RT and NOHZ is active: |
| 7014 | * |
| 7015 | * The idle task calls get_next_timer_interrupt() and holds |
| 7016 | * the timer wheel base->lock on the CPU and another CPU wants |
| 7017 | * to access the timer (probably to cancel it). We can safely |
| 7018 | * ignore the boosting request, as the idle CPU runs this code |
| 7019 | * with interrupts disabled and will complete the lock |
| 7020 | * protected section without being interrupted. So there is no |
| 7021 | * real need to boost. |
| 7022 | */ |
| 7023 | if (unlikely(p == rq->idle)) { |
| 7024 | WARN_ON(p != rq->curr); |
| 7025 | WARN_ON(p->pi_blocked_on); |
| 7026 | goto out_unlock; |
| 7027 | } |
| 7028 | |
Peter Zijlstra | b91473f | 2017-03-23 15:56:12 +0100 | [diff] [blame] | 7029 | trace_sched_pi_setprio(p, pi_task); |
Andrew Morton | d5f9f94 | 2007-05-08 20:27:06 -0700 | [diff] [blame] | 7030 | oldprio = p->prio; |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7031 | |
| 7032 | if (oldprio == prio) |
| 7033 | queue_flag &= ~DEQUEUE_MOVE; |
| 7034 | |
Thomas Gleixner | 83ab0aa | 2010-02-17 09:05:48 +0100 | [diff] [blame] | 7035 | prev_class = p->sched_class; |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 7036 | queued = task_on_rq_queued(p); |
Dmitry Adamushko | 051a1d1 | 2007-12-18 15:21:13 +0100 | [diff] [blame] | 7037 | running = task_current(rq, p); |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 7038 | if (queued) |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7039 | dequeue_task(rq, p, queue_flag); |
Hiroshi Shimamoto | 0e1f348 | 2008-03-10 11:01:20 -0700 | [diff] [blame] | 7040 | if (running) |
Kirill Tkhai | f3cd1c4 | 2014-09-12 17:41:40 +0400 | [diff] [blame] | 7041 | put_prev_task(rq, p); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 7042 | |
Dario Faggioli | 2d3d891 | 2013-11-07 14:43:44 +0100 | [diff] [blame] | 7043 | /* |
| 7044 | * Boosting condition are: |
| 7045 | * 1. -rt task is running and holds mutex A |
| 7046 | * --> -dl task blocks on mutex A |
| 7047 | * |
| 7048 | * 2. -dl task is running and holds mutex A |
| 7049 | * --> -dl task blocks on mutex A and could preempt the |
| 7050 | * running task |
| 7051 | */ |
| 7052 | if (dl_prio(prio)) { |
Oleg Nesterov | 466af29 | 2014-06-06 18:52:06 +0200 | [diff] [blame] | 7053 | if (!dl_prio(p->normal_prio) || |
Juri Lelli | 740797c | 2018-11-19 16:32:01 +0100 | [diff] [blame] | 7054 | (pi_task && dl_prio(pi_task->prio) && |
| 7055 | dl_entity_preempt(&pi_task->dl, &p->dl))) { |
Juri Lelli | 2279f54 | 2020-11-17 07:14:32 +0100 | [diff] [blame] | 7056 | p->dl.pi_se = pi_task->dl.pi_se; |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7057 | queue_flag |= ENQUEUE_REPLENISH; |
Juri Lelli | 2279f54 | 2020-11-17 07:14:32 +0100 | [diff] [blame] | 7058 | } else { |
| 7059 | p->dl.pi_se = &p->dl; |
| 7060 | } |
Dario Faggioli | 2d3d891 | 2013-11-07 14:43:44 +0100 | [diff] [blame] | 7061 | } else if (rt_prio(prio)) { |
| 7062 | if (dl_prio(oldprio)) |
Juri Lelli | 2279f54 | 2020-11-17 07:14:32 +0100 | [diff] [blame] | 7063 | p->dl.pi_se = &p->dl; |
Dario Faggioli | 2d3d891 | 2013-11-07 14:43:44 +0100 | [diff] [blame] | 7064 | if (oldprio < prio) |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7065 | queue_flag |= ENQUEUE_HEAD; |
Dario Faggioli | 2d3d891 | 2013-11-07 14:43:44 +0100 | [diff] [blame] | 7066 | } else { |
| 7067 | if (dl_prio(oldprio)) |
Juri Lelli | 2279f54 | 2020-11-17 07:14:32 +0100 | [diff] [blame] | 7068 | p->dl.pi_se = &p->dl; |
Brian Silverman | 746db94 | 2015-02-18 16:23:56 -0800 | [diff] [blame] | 7069 | if (rt_prio(oldprio)) |
| 7070 | p->rt.timeout = 0; |
Dario Faggioli | 2d3d891 | 2013-11-07 14:43:44 +0100 | [diff] [blame] | 7071 | } |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 7072 | |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 7073 | __setscheduler_prio(p, prio); |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 7074 | |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 7075 | if (queued) |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7076 | enqueue_task(rq, p, queue_flag); |
Vincent Guittot | a399d23 | 2016-09-12 09:47:52 +0200 | [diff] [blame] | 7077 | if (running) |
Peter Zijlstra | 03b7fad | 2019-05-29 20:36:41 +0000 | [diff] [blame] | 7078 | set_next_task(rq, p); |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 7079 | |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 7080 | check_class_changed(rq, p, prev_class, oldprio); |
Thomas Gleixner | 1c4dd99 | 2011-06-06 20:07:38 +0200 | [diff] [blame] | 7081 | out_unlock: |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7082 | /* Avoid rq from going away on us: */ |
| 7083 | preempt_disable(); |
Peter Zijlstra | 4c9a4bc | 2015-06-11 14:46:39 +0200 | [diff] [blame] | 7084 | |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 7085 | rq_unpin_lock(rq, &rf); |
| 7086 | __balance_callbacks(rq); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 7087 | raw_spin_rq_unlock(rq); |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 7088 | |
Peter Zijlstra | 4c9a4bc | 2015-06-11 14:46:39 +0200 | [diff] [blame] | 7089 | preempt_enable(); |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 7090 | } |
Peter Zijlstra | acd5862 | 2017-03-23 15:56:11 +0100 | [diff] [blame] | 7091 | #else |
| 7092 | static inline int rt_effective_prio(struct task_struct *p, int prio) |
| 7093 | { |
| 7094 | return prio; |
| 7095 | } |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 7096 | #endif |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7097 | |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 7098 | void set_user_nice(struct task_struct *p, long nice) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7099 | { |
Peter Zijlstra | 49bd21e | 2016-09-20 22:06:01 +0200 | [diff] [blame] | 7100 | bool queued, running; |
Qian Cai | 53a2336 | 2019-12-19 09:03:14 -0500 | [diff] [blame] | 7101 | int old_prio; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 7102 | struct rq_flags rf; |
Ingo Molnar | 70b97a7 | 2006-07-03 00:25:42 -0700 | [diff] [blame] | 7103 | struct rq *rq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7104 | |
Dongsheng Yang | 75e45d5 | 2014-02-11 15:34:50 +0800 | [diff] [blame] | 7105 | if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7106 | return; |
| 7107 | /* |
| 7108 | * We have to be careful, if called from sys_setpriority(), |
| 7109 | * the task might be in the middle of scheduling on another CPU. |
| 7110 | */ |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 7111 | rq = task_rq_lock(p, &rf); |
Peter Zijlstra | 2fb8d36 | 2016-10-03 16:44:25 +0200 | [diff] [blame] | 7112 | update_rq_clock(rq); |
| 7113 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7114 | /* |
| 7115 | * The RT priorities are set via sched_setscheduler(), but we still |
| 7116 | * allow the 'normal' nice value to be set - but as expected |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 7117 | * it won't have any effect on scheduling until the task is |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 7118 | * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7119 | */ |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 7120 | if (task_has_dl_policy(p) || task_has_rt_policy(p)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7121 | p->static_prio = NICE_TO_PRIO(nice); |
| 7122 | goto out_unlock; |
| 7123 | } |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 7124 | queued = task_on_rq_queued(p); |
Peter Zijlstra | 49bd21e | 2016-09-20 22:06:01 +0200 | [diff] [blame] | 7125 | running = task_current(rq, p); |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 7126 | if (queued) |
Peter Zijlstra | 7a57f32 | 2017-02-21 14:47:02 +0100 | [diff] [blame] | 7127 | dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); |
Peter Zijlstra | 49bd21e | 2016-09-20 22:06:01 +0200 | [diff] [blame] | 7128 | if (running) |
| 7129 | put_prev_task(rq, p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7130 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7131 | p->static_prio = NICE_TO_PRIO(nice); |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 7132 | set_load_weight(p, true); |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 7133 | old_prio = p->prio; |
| 7134 | p->prio = effective_prio(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7135 | |
Frederic Weisbecker | 5443a0b | 2019-12-03 17:01:06 +0100 | [diff] [blame] | 7136 | if (queued) |
Peter Zijlstra | 7134b3e | 2017-02-21 14:23:38 +0100 | [diff] [blame] | 7137 | enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); |
Peter Zijlstra | 49bd21e | 2016-09-20 22:06:01 +0200 | [diff] [blame] | 7138 | if (running) |
Peter Zijlstra | 03b7fad | 2019-05-29 20:36:41 +0000 | [diff] [blame] | 7139 | set_next_task(rq, p); |
Frederic Weisbecker | 5443a0b | 2019-12-03 17:01:06 +0100 | [diff] [blame] | 7140 | |
| 7141 | /* |
| 7142 | * If the task increased its priority or is running and |
| 7143 | * lowered its priority, then reschedule its CPU: |
| 7144 | */ |
| 7145 | p->sched_class->prio_changed(rq, p, old_prio); |
| 7146 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7147 | out_unlock: |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 7148 | task_rq_unlock(rq, p, &rf); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7149 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7150 | EXPORT_SYMBOL(set_user_nice); |
| 7151 | |
Matt Mackall | e43379f | 2005-05-01 08:59:00 -0700 | [diff] [blame] | 7152 | /* |
Christian Göttsche | 700a783 | 2022-06-15 17:25:04 +0200 | [diff] [blame] | 7153 | * is_nice_reduction - check if nice value is an actual reduction |
| 7154 | * |
| 7155 | * Similar to can_nice() but does not perform a capability check. |
| 7156 | * |
| 7157 | * @p: task |
| 7158 | * @nice: nice value |
| 7159 | */ |
| 7160 | static bool is_nice_reduction(const struct task_struct *p, const int nice) |
| 7161 | { |
| 7162 | /* Convert nice value [19,-20] to rlimit style value [1,40]: */ |
| 7163 | int nice_rlim = nice_to_rlimit(nice); |
| 7164 | |
| 7165 | return (nice_rlim <= task_rlimit(p, RLIMIT_NICE)); |
| 7166 | } |
| 7167 | |
| 7168 | /* |
Matt Mackall | e43379f | 2005-05-01 08:59:00 -0700 | [diff] [blame] | 7169 | * can_nice - check if a task can reduce its nice value |
| 7170 | * @p: task |
| 7171 | * @nice: nice value |
| 7172 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 7173 | int can_nice(const struct task_struct *p, const int nice) |
Matt Mackall | e43379f | 2005-05-01 08:59:00 -0700 | [diff] [blame] | 7174 | { |
Christian Göttsche | 700a783 | 2022-06-15 17:25:04 +0200 | [diff] [blame] | 7175 | return is_nice_reduction(p, nice) || capable(CAP_SYS_NICE); |
Matt Mackall | e43379f | 2005-05-01 08:59:00 -0700 | [diff] [blame] | 7176 | } |
| 7177 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7178 | #ifdef __ARCH_WANT_SYS_NICE |
| 7179 | |
| 7180 | /* |
| 7181 | * sys_nice - change the priority of the current process. |
| 7182 | * @increment: priority increment |
| 7183 | * |
| 7184 | * sys_setpriority is a more generic, but much slower function that |
| 7185 | * does similar things. |
| 7186 | */ |
Heiko Carstens | 5add95d | 2009-01-14 14:14:08 +0100 | [diff] [blame] | 7187 | SYSCALL_DEFINE1(nice, int, increment) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7188 | { |
Ingo Molnar | 48f24c4 | 2006-07-03 00:25:40 -0700 | [diff] [blame] | 7189 | long nice, retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7190 | |
| 7191 | /* |
| 7192 | * Setpriority might change our priority at the same moment. |
| 7193 | * We don't have to worry. Conceptually one call occurs first |
| 7194 | * and we have a single winner. |
| 7195 | */ |
Dongsheng Yang | a9467fa | 2014-05-08 18:35:15 +0900 | [diff] [blame] | 7196 | increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); |
Dongsheng Yang | d0ea026 | 2014-01-27 22:00:45 -0500 | [diff] [blame] | 7197 | nice = task_nice(current) + increment; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7198 | |
Dongsheng Yang | a9467fa | 2014-05-08 18:35:15 +0900 | [diff] [blame] | 7199 | nice = clamp_val(nice, MIN_NICE, MAX_NICE); |
Matt Mackall | e43379f | 2005-05-01 08:59:00 -0700 | [diff] [blame] | 7200 | if (increment < 0 && !can_nice(current, nice)) |
| 7201 | return -EPERM; |
| 7202 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7203 | retval = security_task_setnice(current, nice); |
| 7204 | if (retval) |
| 7205 | return retval; |
| 7206 | |
| 7207 | set_user_nice(current, nice); |
| 7208 | return 0; |
| 7209 | } |
| 7210 | |
| 7211 | #endif |
| 7212 | |
| 7213 | /** |
| 7214 | * task_prio - return the priority value of a given task. |
| 7215 | * @p: the task in question. |
| 7216 | * |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7217 | * Return: The priority value as seen by users in /proc. |
Dietmar Eggemann | c541bb7 | 2021-01-28 14:10:40 +0100 | [diff] [blame] | 7218 | * |
| 7219 | * sched policy return value kernel prio user prio/nice |
| 7220 | * |
| 7221 | * normal, batch, idle [0 ... 39] [100 ... 139] 0/[-20 ... 19] |
| 7222 | * fifo, rr [-2 ... -100] [98 ... 0] [1 ... 99] |
| 7223 | * deadline -101 -1 0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7224 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 7225 | int task_prio(const struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7226 | { |
| 7227 | return p->prio - MAX_RT_PRIO; |
| 7228 | } |
| 7229 | |
| 7230 | /** |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7231 | * idle_cpu - is a given CPU idle currently? |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7232 | * @cpu: the processor in question. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7233 | * |
| 7234 | * Return: 1 if the CPU is currently idle. 0 otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7235 | */ |
| 7236 | int idle_cpu(int cpu) |
| 7237 | { |
Thomas Gleixner | 908a328 | 2011-09-15 15:32:06 +0200 | [diff] [blame] | 7238 | struct rq *rq = cpu_rq(cpu); |
| 7239 | |
| 7240 | if (rq->curr != rq->idle) |
| 7241 | return 0; |
| 7242 | |
| 7243 | if (rq->nr_running) |
| 7244 | return 0; |
| 7245 | |
| 7246 | #ifdef CONFIG_SMP |
Peter Zijlstra | 126c209 | 2020-05-26 18:11:03 +0200 | [diff] [blame] | 7247 | if (rq->ttwu_pending) |
Thomas Gleixner | 908a328 | 2011-09-15 15:32:06 +0200 | [diff] [blame] | 7248 | return 0; |
| 7249 | #endif |
| 7250 | |
| 7251 | return 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7252 | } |
| 7253 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7254 | /** |
Rohit Jain | 943d355 | 2018-05-09 09:39:48 -0700 | [diff] [blame] | 7255 | * available_idle_cpu - is a given CPU idle for enqueuing work. |
| 7256 | * @cpu: the CPU in question. |
| 7257 | * |
| 7258 | * Return: 1 if the CPU is currently idle. 0 otherwise. |
| 7259 | */ |
| 7260 | int available_idle_cpu(int cpu) |
| 7261 | { |
| 7262 | if (!idle_cpu(cpu)) |
| 7263 | return 0; |
| 7264 | |
Rohit Jain | 247f2f6 | 2018-05-02 13:52:10 -0700 | [diff] [blame] | 7265 | if (vcpu_is_preempted(cpu)) |
| 7266 | return 0; |
| 7267 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7268 | return 1; |
| 7269 | } |
| 7270 | |
| 7271 | /** |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7272 | * idle_task - return the idle task for a given CPU. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7273 | * @cpu: the processor in question. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7274 | * |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7275 | * Return: The idle task for the CPU @cpu. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7276 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 7277 | struct task_struct *idle_task(int cpu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7278 | { |
| 7279 | return cpu_rq(cpu)->idle; |
| 7280 | } |
| 7281 | |
Viresh Kumar | 7d6a905f | 2020-12-08 09:46:55 +0530 | [diff] [blame] | 7282 | #ifdef CONFIG_SMP |
| 7283 | /* |
| 7284 | * This function computes an effective utilization for the given CPU, to be |
| 7285 | * used for frequency selection given the linear relation: f = u * f_max. |
| 7286 | * |
| 7287 | * The scheduler tracks the following metrics: |
| 7288 | * |
| 7289 | * cpu_util_{cfs,rt,dl,irq}() |
| 7290 | * cpu_bw_dl() |
| 7291 | * |
| 7292 | * Where the cfs,rt and dl util numbers are tracked with the same metric and |
| 7293 | * synchronized windows and are thus directly comparable. |
| 7294 | * |
| 7295 | * The cfs,rt,dl utilization are the running times measured with rq->clock_task |
| 7296 | * which excludes things like IRQ and steal-time. These latter are then accrued |
| 7297 | * in the irq utilization. |
| 7298 | * |
| 7299 | * The DL bandwidth number otoh is not a measured metric but a value computed |
| 7300 | * based on the task model parameters and gives the minimal utilization |
| 7301 | * required to meet deadlines. |
| 7302 | */ |
Viresh Kumar | a5418be | 2020-12-08 09:46:56 +0530 | [diff] [blame] | 7303 | unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, |
Dietmar Eggemann | bb44799 | 2022-06-21 10:04:10 +0100 | [diff] [blame] | 7304 | enum cpu_util_type type, |
Viresh Kumar | 7d6a905f | 2020-12-08 09:46:55 +0530 | [diff] [blame] | 7305 | struct task_struct *p) |
| 7306 | { |
Dietmar Eggemann | bb44799 | 2022-06-21 10:04:10 +0100 | [diff] [blame] | 7307 | unsigned long dl_util, util, irq, max; |
Viresh Kumar | 7d6a905f | 2020-12-08 09:46:55 +0530 | [diff] [blame] | 7308 | struct rq *rq = cpu_rq(cpu); |
| 7309 | |
Dietmar Eggemann | bb44799 | 2022-06-21 10:04:10 +0100 | [diff] [blame] | 7310 | max = arch_scale_cpu_capacity(cpu); |
| 7311 | |
Viresh Kumar | 7d6a905f | 2020-12-08 09:46:55 +0530 | [diff] [blame] | 7312 | if (!uclamp_is_used() && |
| 7313 | type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) { |
| 7314 | return max; |
| 7315 | } |
| 7316 | |
| 7317 | /* |
| 7318 | * Early check to see if IRQ/steal time saturates the CPU, can be |
| 7319 | * because of inaccuracies in how we track these -- see |
| 7320 | * update_irq_load_avg(). |
| 7321 | */ |
| 7322 | irq = cpu_util_irq(rq); |
| 7323 | if (unlikely(irq >= max)) |
| 7324 | return max; |
| 7325 | |
| 7326 | /* |
| 7327 | * Because the time spend on RT/DL tasks is visible as 'lost' time to |
| 7328 | * CFS tasks and we use the same metric to track the effective |
| 7329 | * utilization (PELT windows are synchronized) we can directly add them |
| 7330 | * to obtain the CPU's actual utilization. |
| 7331 | * |
| 7332 | * CFS and RT utilization can be boosted or capped, depending on |
| 7333 | * utilization clamp constraints requested by currently RUNNABLE |
| 7334 | * tasks. |
| 7335 | * When there are no CFS RUNNABLE tasks, clamps are released and |
| 7336 | * frequency will be gracefully reduced with the utilization decay. |
| 7337 | */ |
| 7338 | util = util_cfs + cpu_util_rt(rq); |
| 7339 | if (type == FREQUENCY_UTIL) |
| 7340 | util = uclamp_rq_util_with(rq, util, p); |
| 7341 | |
| 7342 | dl_util = cpu_util_dl(rq); |
| 7343 | |
| 7344 | /* |
| 7345 | * For frequency selection we do not make cpu_util_dl() a permanent part |
| 7346 | * of this sum because we want to use cpu_bw_dl() later on, but we need |
| 7347 | * to check if the CFS+RT+DL sum is saturated (ie. no idle time) such |
| 7348 | * that we select f_max when there is no idle time. |
| 7349 | * |
| 7350 | * NOTE: numerical errors or stop class might cause us to not quite hit |
| 7351 | * saturation when we should -- something for later. |
| 7352 | */ |
| 7353 | if (util + dl_util >= max) |
| 7354 | return max; |
| 7355 | |
| 7356 | /* |
| 7357 | * OTOH, for energy computation we need the estimated running time, so |
| 7358 | * include util_dl and ignore dl_bw. |
| 7359 | */ |
| 7360 | if (type == ENERGY_UTIL) |
| 7361 | util += dl_util; |
| 7362 | |
| 7363 | /* |
| 7364 | * There is still idle time; further improve the number by using the |
| 7365 | * irq metric. Because IRQ/steal time is hidden from the task clock we |
| 7366 | * need to scale the task numbers: |
| 7367 | * |
| 7368 | * max - irq |
| 7369 | * U' = irq + --------- * U |
| 7370 | * max |
| 7371 | */ |
| 7372 | util = scale_irq_capacity(util, irq, max); |
| 7373 | util += irq; |
| 7374 | |
| 7375 | /* |
| 7376 | * Bandwidth required by DEADLINE must always be granted while, for |
| 7377 | * FAIR and RT, we use blocked utilization of IDLE CPUs as a mechanism |
| 7378 | * to gracefully reduce the frequency when no tasks show up for longer |
| 7379 | * periods of time. |
| 7380 | * |
| 7381 | * Ideally we would like to set bw_dl as min/guaranteed freq and util + |
| 7382 | * bw_dl as requested freq. However, cpufreq is not yet ready for such |
| 7383 | * an interface. So, we only do the latter for now. |
| 7384 | */ |
| 7385 | if (type == FREQUENCY_UTIL) |
| 7386 | util += cpu_bw_dl(rq); |
| 7387 | |
| 7388 | return min(max, util); |
| 7389 | } |
Viresh Kumar | a5418be | 2020-12-08 09:46:56 +0530 | [diff] [blame] | 7390 | |
Dietmar Eggemann | bb44799 | 2022-06-21 10:04:10 +0100 | [diff] [blame] | 7391 | unsigned long sched_cpu_util(int cpu) |
Viresh Kumar | a5418be | 2020-12-08 09:46:56 +0530 | [diff] [blame] | 7392 | { |
Dietmar Eggemann | bb44799 | 2022-06-21 10:04:10 +0100 | [diff] [blame] | 7393 | return effective_cpu_util(cpu, cpu_util_cfs(cpu), ENERGY_UTIL, NULL); |
Viresh Kumar | a5418be | 2020-12-08 09:46:56 +0530 | [diff] [blame] | 7394 | } |
Viresh Kumar | 7d6a905f | 2020-12-08 09:46:55 +0530 | [diff] [blame] | 7395 | #endif /* CONFIG_SMP */ |
| 7396 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7397 | /** |
| 7398 | * find_process_by_pid - find a process with a matching PID value. |
| 7399 | * @pid: the pid in question. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7400 | * |
| 7401 | * The task of @pid, if found. %NULL otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7402 | */ |
Alexey Dobriyan | a995744 | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 7403 | static struct task_struct *find_process_by_pid(pid_t pid) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7404 | { |
Pavel Emelyanov | 228ebcb | 2007-10-18 23:40:16 -0700 | [diff] [blame] | 7405 | return pid ? find_task_by_vpid(pid) : current; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7406 | } |
| 7407 | |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 7408 | /* |
Steven Rostedt | c13db6b | 2014-07-23 11:28:26 -0400 | [diff] [blame] | 7409 | * sched_setparam() passes in -1 for its policy, to let the functions |
| 7410 | * it calls know not to change it. |
| 7411 | */ |
| 7412 | #define SETPARAM_POLICY -1 |
| 7413 | |
Thomas Gleixner | c365c29 | 2014-02-07 20:58:42 +0100 | [diff] [blame] | 7414 | static void __setscheduler_params(struct task_struct *p, |
| 7415 | const struct sched_attr *attr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7416 | { |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7417 | int policy = attr->sched_policy; |
| 7418 | |
Steven Rostedt | c13db6b | 2014-07-23 11:28:26 -0400 | [diff] [blame] | 7419 | if (policy == SETPARAM_POLICY) |
Peter Zijlstra | 39fd8fd | 2014-01-15 16:33:20 +0100 | [diff] [blame] | 7420 | policy = p->policy; |
| 7421 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7422 | p->policy = policy; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7423 | |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 7424 | if (dl_policy(policy)) |
| 7425 | __setparam_dl(p, attr); |
Peter Zijlstra | 39fd8fd | 2014-01-15 16:33:20 +0100 | [diff] [blame] | 7426 | else if (fair_policy(policy)) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7427 | p->static_prio = NICE_TO_PRIO(attr->sched_nice); |
| 7428 | |
Peter Zijlstra | 39fd8fd | 2014-01-15 16:33:20 +0100 | [diff] [blame] | 7429 | /* |
| 7430 | * __sched_setscheduler() ensures attr->sched_priority == 0 when |
| 7431 | * !rt_policy. Always setting this ensures that things like |
| 7432 | * getparam()/getattr() don't report silly values for !rt tasks. |
| 7433 | */ |
| 7434 | p->rt_priority = attr->sched_priority; |
Steven Rostedt | 383afd0 | 2014-03-11 19:24:20 -0400 | [diff] [blame] | 7435 | p->normal_prio = normal_prio(p); |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 7436 | set_load_weight(p, true); |
Thomas Gleixner | c365c29 | 2014-02-07 20:58:42 +0100 | [diff] [blame] | 7437 | } |
Peter Zijlstra | 39fd8fd | 2014-01-15 16:33:20 +0100 | [diff] [blame] | 7438 | |
David Howells | c69e8d9 | 2008-11-14 10:39:19 +1100 | [diff] [blame] | 7439 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7440 | * Check the target process has a UID that matches the current process's: |
David Howells | c69e8d9 | 2008-11-14 10:39:19 +1100 | [diff] [blame] | 7441 | */ |
| 7442 | static bool check_same_owner(struct task_struct *p) |
| 7443 | { |
| 7444 | const struct cred *cred = current_cred(), *pcred; |
| 7445 | bool match; |
| 7446 | |
| 7447 | rcu_read_lock(); |
| 7448 | pcred = __task_cred(p); |
Eric W. Biederman | 9c806aa | 2012-02-02 18:54:02 -0800 | [diff] [blame] | 7449 | match = (uid_eq(cred->euid, pcred->euid) || |
| 7450 | uid_eq(cred->euid, pcred->uid)); |
David Howells | c69e8d9 | 2008-11-14 10:39:19 +1100 | [diff] [blame] | 7451 | rcu_read_unlock(); |
| 7452 | return match; |
| 7453 | } |
| 7454 | |
Christian Göttsche | 700a783 | 2022-06-15 17:25:04 +0200 | [diff] [blame] | 7455 | /* |
| 7456 | * Allow unprivileged RT tasks to decrease priority. |
| 7457 | * Only issue a capable test if needed and only once to avoid an audit |
| 7458 | * event on permitted non-privileged operations: |
| 7459 | */ |
| 7460 | static int user_check_sched_setscheduler(struct task_struct *p, |
| 7461 | const struct sched_attr *attr, |
| 7462 | int policy, int reset_on_fork) |
| 7463 | { |
| 7464 | if (fair_policy(policy)) { |
| 7465 | if (attr->sched_nice < task_nice(p) && |
| 7466 | !is_nice_reduction(p, attr->sched_nice)) |
| 7467 | goto req_priv; |
| 7468 | } |
| 7469 | |
| 7470 | if (rt_policy(policy)) { |
| 7471 | unsigned long rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO); |
| 7472 | |
| 7473 | /* Can't set/change the rt policy: */ |
| 7474 | if (policy != p->policy && !rlim_rtprio) |
| 7475 | goto req_priv; |
| 7476 | |
| 7477 | /* Can't increase priority: */ |
| 7478 | if (attr->sched_priority > p->rt_priority && |
| 7479 | attr->sched_priority > rlim_rtprio) |
| 7480 | goto req_priv; |
| 7481 | } |
| 7482 | |
| 7483 | /* |
| 7484 | * Can't set/change SCHED_DEADLINE policy at all for now |
| 7485 | * (safest behavior); in the future we would like to allow |
| 7486 | * unprivileged DL tasks to increase their relative deadline |
| 7487 | * or reduce their runtime (both ways reducing utilization) |
| 7488 | */ |
| 7489 | if (dl_policy(policy)) |
| 7490 | goto req_priv; |
| 7491 | |
| 7492 | /* |
| 7493 | * Treat SCHED_IDLE as nice 20. Only allow a switch to |
| 7494 | * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. |
| 7495 | */ |
| 7496 | if (task_has_idle_policy(p) && !idle_policy(policy)) { |
| 7497 | if (!is_nice_reduction(p, task_nice(p))) |
| 7498 | goto req_priv; |
| 7499 | } |
| 7500 | |
| 7501 | /* Can't change other user's priorities: */ |
| 7502 | if (!check_same_owner(p)) |
| 7503 | goto req_priv; |
| 7504 | |
| 7505 | /* Normal users shall not reset the sched_reset_on_fork flag: */ |
| 7506 | if (p->sched_reset_on_fork && !reset_on_fork) |
| 7507 | goto req_priv; |
| 7508 | |
| 7509 | return 0; |
| 7510 | |
| 7511 | req_priv: |
| 7512 | if (!capable(CAP_SYS_NICE)) |
| 7513 | return -EPERM; |
| 7514 | |
| 7515 | return 0; |
| 7516 | } |
| 7517 | |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7518 | static int __sched_setscheduler(struct task_struct *p, |
| 7519 | const struct sched_attr *attr, |
Peter Zijlstra | dbc7f06 | 2015-06-11 14:46:38 +0200 | [diff] [blame] | 7520 | bool user, bool pi) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7521 | { |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 7522 | int oldpolicy = -1, policy = attr->sched_policy; |
| 7523 | int retval, oldprio, newprio, queued, running; |
Thomas Gleixner | 83ab0aa | 2010-02-17 09:05:48 +0100 | [diff] [blame] | 7524 | const struct sched_class *prev_class; |
Kees Cook | 8e5bad7 | 2022-10-07 17:07:58 -0700 | [diff] [blame] | 7525 | struct balance_callback *head; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 7526 | struct rq_flags rf; |
Lennart Poettering | ca94c44 | 2009-06-15 17:17:47 +0200 | [diff] [blame] | 7527 | int reset_on_fork; |
Peter Zijlstra | 7a57f32 | 2017-02-21 14:47:02 +0100 | [diff] [blame] | 7528 | int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 7529 | struct rq *rq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7530 | |
Steven Rostedt (VMware) | 896bbb2 | 2017-03-09 10:18:42 -0500 | [diff] [blame] | 7531 | /* The pi code expects interrupts enabled */ |
| 7532 | BUG_ON(pi && in_interrupt()); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7533 | recheck: |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7534 | /* Double check policy once rq lock held: */ |
Lennart Poettering | ca94c44 | 2009-06-15 17:17:47 +0200 | [diff] [blame] | 7535 | if (policy < 0) { |
| 7536 | reset_on_fork = p->sched_reset_on_fork; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7537 | policy = oldpolicy = p->policy; |
Lennart Poettering | ca94c44 | 2009-06-15 17:17:47 +0200 | [diff] [blame] | 7538 | } else { |
Peter Zijlstra | 7479f3c9c | 2014-01-15 17:05:04 +0100 | [diff] [blame] | 7539 | reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); |
Lennart Poettering | ca94c44 | 2009-06-15 17:17:47 +0200 | [diff] [blame] | 7540 | |
Henrik Austad | 20f9cd2 | 2015-09-09 17:00:41 +0200 | [diff] [blame] | 7541 | if (!valid_policy(policy)) |
Lennart Poettering | ca94c44 | 2009-06-15 17:17:47 +0200 | [diff] [blame] | 7542 | return -EINVAL; |
| 7543 | } |
| 7544 | |
Juri Lelli | 794a56e | 2017-12-04 11:23:20 +0100 | [diff] [blame] | 7545 | if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) |
Peter Zijlstra | 7479f3c9c | 2014-01-15 17:05:04 +0100 | [diff] [blame] | 7546 | return -EINVAL; |
| 7547 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7548 | /* |
| 7549 | * Valid priorities for SCHED_FIFO and SCHED_RR are |
Dietmar Eggemann | ae18ad2 | 2021-01-28 14:10:38 +0100 | [diff] [blame] | 7550 | * 1..MAX_RT_PRIO-1, valid priority for SCHED_NORMAL, |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 7551 | * SCHED_BATCH and SCHED_IDLE is 0. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7552 | */ |
Dietmar Eggemann | ae18ad2 | 2021-01-28 14:10:38 +0100 | [diff] [blame] | 7553 | if (attr->sched_priority > MAX_RT_PRIO-1) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7554 | return -EINVAL; |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 7555 | if ((dl_policy(policy) && !__checkparam_dl(attr)) || |
| 7556 | (rt_policy(policy) != (attr->sched_priority != 0))) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7557 | return -EINVAL; |
| 7558 | |
Jeremy Fitzhardinge | 725aad2 | 2008-08-03 09:33:03 -0700 | [diff] [blame] | 7559 | if (user) { |
Christian Göttsche | 700a783 | 2022-06-15 17:25:04 +0200 | [diff] [blame] | 7560 | retval = user_check_sched_setscheduler(p, attr, policy, reset_on_fork); |
| 7561 | if (retval) |
| 7562 | return retval; |
| 7563 | |
Juri Lelli | 794a56e | 2017-12-04 11:23:20 +0100 | [diff] [blame] | 7564 | if (attr->sched_flags & SCHED_FLAG_SUGOV) |
| 7565 | return -EINVAL; |
| 7566 | |
KOSAKI Motohiro | b0ae198 | 2010-10-15 04:21:18 +0900 | [diff] [blame] | 7567 | retval = security_task_setscheduler(p); |
Jeremy Fitzhardinge | 725aad2 | 2008-08-03 09:33:03 -0700 | [diff] [blame] | 7568 | if (retval) |
| 7569 | return retval; |
| 7570 | } |
| 7571 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 7572 | /* Update task specific "requested" clamps */ |
| 7573 | if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) { |
| 7574 | retval = uclamp_validate(p, attr); |
| 7575 | if (retval) |
| 7576 | return retval; |
| 7577 | } |
| 7578 | |
Juri Lelli | 710da3c | 2019-07-19 16:00:00 +0200 | [diff] [blame] | 7579 | if (pi) |
| 7580 | cpuset_read_lock(); |
| 7581 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7582 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7583 | * Make sure no PI-waiters arrive (or leave) while we are |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 7584 | * changing the priority of the task: |
Peter Zijlstra | 0122ec5 | 2011-04-05 17:23:51 +0200 | [diff] [blame] | 7585 | * |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 7586 | * To be able to change p->policy safely, the appropriate |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7587 | * runqueue lock must be held. |
| 7588 | */ |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 7589 | rq = task_rq_lock(p, &rf); |
Peter Zijlstra | 80f5c1b | 2016-10-03 16:28:37 +0200 | [diff] [blame] | 7590 | update_rq_clock(rq); |
Peter Zijlstra | dc61b1d | 2010-06-08 11:40:42 +0200 | [diff] [blame] | 7591 | |
Peter Zijlstra | 34f971f | 2010-09-22 13:53:15 +0200 | [diff] [blame] | 7592 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7593 | * Changing the policy of the stop threads its a very bad idea: |
Peter Zijlstra | 34f971f | 2010-09-22 13:53:15 +0200 | [diff] [blame] | 7594 | */ |
| 7595 | if (p == rq->stop) { |
Mathieu Poirier | 4b211f2 | 2019-07-19 15:59:54 +0200 | [diff] [blame] | 7596 | retval = -EINVAL; |
| 7597 | goto unlock; |
Peter Zijlstra | 34f971f | 2010-09-22 13:53:15 +0200 | [diff] [blame] | 7598 | } |
| 7599 | |
Dario Faggioli | a51e919 | 2011-03-24 14:00:18 +0100 | [diff] [blame] | 7600 | /* |
Thomas Gleixner | d6b1e91 | 2014-02-07 20:58:40 +0100 | [diff] [blame] | 7601 | * If not changing anything there's no need to proceed further, |
| 7602 | * but store a possible modification of reset_on_fork. |
Dario Faggioli | a51e919 | 2011-03-24 14:00:18 +0100 | [diff] [blame] | 7603 | */ |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7604 | if (unlikely(policy == p->policy)) { |
Dongsheng Yang | d0ea026 | 2014-01-27 22:00:45 -0500 | [diff] [blame] | 7605 | if (fair_policy(policy) && attr->sched_nice != task_nice(p)) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7606 | goto change; |
| 7607 | if (rt_policy(policy) && attr->sched_priority != p->rt_priority) |
| 7608 | goto change; |
Wanpeng Li | 7538160 | 2014-11-26 08:44:04 +0800 | [diff] [blame] | 7609 | if (dl_policy(policy) && dl_param_changed(p, attr)) |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 7610 | goto change; |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 7611 | if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) |
| 7612 | goto change; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7613 | |
Thomas Gleixner | d6b1e91 | 2014-02-07 20:58:40 +0100 | [diff] [blame] | 7614 | p->sched_reset_on_fork = reset_on_fork; |
Mathieu Poirier | 4b211f2 | 2019-07-19 15:59:54 +0200 | [diff] [blame] | 7615 | retval = 0; |
| 7616 | goto unlock; |
Dario Faggioli | a51e919 | 2011-03-24 14:00:18 +0100 | [diff] [blame] | 7617 | } |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7618 | change: |
Dario Faggioli | a51e919 | 2011-03-24 14:00:18 +0100 | [diff] [blame] | 7619 | |
Peter Zijlstra | dc61b1d | 2010-06-08 11:40:42 +0200 | [diff] [blame] | 7620 | if (user) { |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 7621 | #ifdef CONFIG_RT_GROUP_SCHED |
Peter Zijlstra | dc61b1d | 2010-06-08 11:40:42 +0200 | [diff] [blame] | 7622 | /* |
| 7623 | * Do not allow realtime tasks into groups that have no runtime |
| 7624 | * assigned. |
| 7625 | */ |
| 7626 | if (rt_bandwidth_enabled() && rt_policy(policy) && |
Mike Galbraith | f449377 | 2011-01-13 04:54:50 +0100 | [diff] [blame] | 7627 | task_group(p)->rt_bandwidth.rt_runtime == 0 && |
| 7628 | !task_group_is_autogroup(task_group(p))) { |
Mathieu Poirier | 4b211f2 | 2019-07-19 15:59:54 +0200 | [diff] [blame] | 7629 | retval = -EPERM; |
| 7630 | goto unlock; |
Peter Zijlstra | dc61b1d | 2010-06-08 11:40:42 +0200 | [diff] [blame] | 7631 | } |
Peter Zijlstra | dc61b1d | 2010-06-08 11:40:42 +0200 | [diff] [blame] | 7632 | #endif |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 7633 | #ifdef CONFIG_SMP |
Juri Lelli | 794a56e | 2017-12-04 11:23:20 +0100 | [diff] [blame] | 7634 | if (dl_bandwidth_enabled() && dl_policy(policy) && |
| 7635 | !(attr->sched_flags & SCHED_FLAG_SUGOV)) { |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 7636 | cpumask_t *span = rq->rd->span; |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 7637 | |
| 7638 | /* |
| 7639 | * Don't allow tasks with an affinity mask smaller than |
| 7640 | * the entire root_domain to become SCHED_DEADLINE. We |
| 7641 | * will also fail if there's no bandwidth available. |
| 7642 | */ |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 7643 | if (!cpumask_subset(span, p->cpus_ptr) || |
Peter Zijlstra | e4099a5 | 2013-12-17 10:03:34 +0100 | [diff] [blame] | 7644 | rq->rd->dl_bw.bw == 0) { |
Mathieu Poirier | 4b211f2 | 2019-07-19 15:59:54 +0200 | [diff] [blame] | 7645 | retval = -EPERM; |
| 7646 | goto unlock; |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 7647 | } |
| 7648 | } |
| 7649 | #endif |
| 7650 | } |
Peter Zijlstra | dc61b1d | 2010-06-08 11:40:42 +0200 | [diff] [blame] | 7651 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7652 | /* Re-check policy now with rq lock held: */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7653 | if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { |
| 7654 | policy = oldpolicy = -1; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 7655 | task_rq_unlock(rq, p, &rf); |
Juri Lelli | 710da3c | 2019-07-19 16:00:00 +0200 | [diff] [blame] | 7656 | if (pi) |
| 7657 | cpuset_read_unlock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7658 | goto recheck; |
| 7659 | } |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 7660 | |
| 7661 | /* |
| 7662 | * If setscheduling to SCHED_DEADLINE (or changing the parameters |
| 7663 | * of a SCHED_DEADLINE task) we need to check if enough bandwidth |
| 7664 | * is available. |
| 7665 | */ |
Nicolas Pitre | 06a76fe | 2017-06-21 14:22:01 -0400 | [diff] [blame] | 7666 | if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) { |
Mathieu Poirier | 4b211f2 | 2019-07-19 15:59:54 +0200 | [diff] [blame] | 7667 | retval = -EBUSY; |
| 7668 | goto unlock; |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 7669 | } |
| 7670 | |
Thomas Gleixner | c365c29 | 2014-02-07 20:58:42 +0100 | [diff] [blame] | 7671 | p->sched_reset_on_fork = reset_on_fork; |
| 7672 | oldprio = p->prio; |
| 7673 | |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 7674 | newprio = __normal_prio(policy, attr->sched_priority, attr->sched_nice); |
Peter Zijlstra | dbc7f06 | 2015-06-11 14:46:38 +0200 | [diff] [blame] | 7675 | if (pi) { |
| 7676 | /* |
| 7677 | * Take priority boosted tasks into account. If the new |
| 7678 | * effective priority is unchanged, we just store the new |
| 7679 | * normal parameters and do not touch the scheduler class and |
| 7680 | * the runqueue. This will be done when the task deboost |
| 7681 | * itself. |
| 7682 | */ |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 7683 | newprio = rt_effective_prio(p, newprio); |
| 7684 | if (newprio == oldprio) |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7685 | queue_flags &= ~DEQUEUE_MOVE; |
Thomas Gleixner | c365c29 | 2014-02-07 20:58:42 +0100 | [diff] [blame] | 7686 | } |
| 7687 | |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 7688 | queued = task_on_rq_queued(p); |
Dmitry Adamushko | 051a1d1 | 2007-12-18 15:21:13 +0100 | [diff] [blame] | 7689 | running = task_current(rq, p); |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 7690 | if (queued) |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7691 | dequeue_task(rq, p, queue_flags); |
Hiroshi Shimamoto | 0e1f348 | 2008-03-10 11:01:20 -0700 | [diff] [blame] | 7692 | if (running) |
Kirill Tkhai | f3cd1c4 | 2014-09-12 17:41:40 +0400 | [diff] [blame] | 7693 | put_prev_task(rq, p); |
Dmitry Adamushko | f6b53205 | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 7694 | |
Thomas Gleixner | 83ab0aa | 2010-02-17 09:05:48 +0100 | [diff] [blame] | 7695 | prev_class = p->sched_class; |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 7696 | |
Peter Zijlstra | f558c2b | 2021-08-03 12:45:01 +0200 | [diff] [blame] | 7697 | if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { |
| 7698 | __setscheduler_params(p, attr); |
| 7699 | __setscheduler_prio(p, newprio); |
| 7700 | } |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 7701 | __setscheduler_uclamp(p, attr); |
Dmitry Adamushko | f6b53205 | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 7702 | |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 7703 | if (queued) { |
Thomas Gleixner | 81a44c5 | 2014-02-07 20:58:41 +0100 | [diff] [blame] | 7704 | /* |
| 7705 | * We enqueue to tail when the priority of a task is |
| 7706 | * increased (user space view). |
| 7707 | */ |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7708 | if (oldprio < p->prio) |
| 7709 | queue_flags |= ENQUEUE_HEAD; |
Peter Zijlstra | 1de6444 | 2015-09-30 17:44:13 +0200 | [diff] [blame] | 7710 | |
Peter Zijlstra | ff77e46 | 2016-01-18 15:27:07 +0100 | [diff] [blame] | 7711 | enqueue_task(rq, p, queue_flags); |
Thomas Gleixner | 81a44c5 | 2014-02-07 20:58:41 +0100 | [diff] [blame] | 7712 | } |
Vincent Guittot | a399d23 | 2016-09-12 09:47:52 +0200 | [diff] [blame] | 7713 | if (running) |
Peter Zijlstra | 03b7fad | 2019-05-29 20:36:41 +0000 | [diff] [blame] | 7714 | set_next_task(rq, p); |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 7715 | |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 7716 | check_class_changed(rq, p, prev_class, oldprio); |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7717 | |
| 7718 | /* Avoid rq from going away on us: */ |
| 7719 | preempt_disable(); |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 7720 | head = splice_balance_callbacks(rq); |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 7721 | task_rq_unlock(rq, p, &rf); |
Ingo Molnar | b29739f | 2006-06-27 02:54:51 -0700 | [diff] [blame] | 7722 | |
Juri Lelli | 710da3c | 2019-07-19 16:00:00 +0200 | [diff] [blame] | 7723 | if (pi) { |
| 7724 | cpuset_read_unlock(); |
Peter Zijlstra | dbc7f06 | 2015-06-11 14:46:38 +0200 | [diff] [blame] | 7725 | rt_mutex_adjust_pi(p); |
Juri Lelli | 710da3c | 2019-07-19 16:00:00 +0200 | [diff] [blame] | 7726 | } |
Thomas Gleixner | 95e02ca | 2006-06-27 02:55:02 -0700 | [diff] [blame] | 7727 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7728 | /* Run balance callbacks after we've adjusted the PI chain: */ |
Peter Zijlstra | 565790d2 | 2020-05-11 14:13:00 +0200 | [diff] [blame] | 7729 | balance_callbacks(rq, head); |
Peter Zijlstra | 4c9a4bc | 2015-06-11 14:46:39 +0200 | [diff] [blame] | 7730 | preempt_enable(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7731 | |
| 7732 | return 0; |
Mathieu Poirier | 4b211f2 | 2019-07-19 15:59:54 +0200 | [diff] [blame] | 7733 | |
| 7734 | unlock: |
| 7735 | task_rq_unlock(rq, p, &rf); |
Juri Lelli | 710da3c | 2019-07-19 16:00:00 +0200 | [diff] [blame] | 7736 | if (pi) |
| 7737 | cpuset_read_unlock(); |
Mathieu Poirier | 4b211f2 | 2019-07-19 15:59:54 +0200 | [diff] [blame] | 7738 | return retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7739 | } |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7740 | |
Peter Zijlstra | 7479f3c9c | 2014-01-15 17:05:04 +0100 | [diff] [blame] | 7741 | static int _sched_setscheduler(struct task_struct *p, int policy, |
| 7742 | const struct sched_param *param, bool check) |
| 7743 | { |
| 7744 | struct sched_attr attr = { |
| 7745 | .sched_policy = policy, |
| 7746 | .sched_priority = param->sched_priority, |
| 7747 | .sched_nice = PRIO_TO_NICE(p->static_prio), |
| 7748 | }; |
| 7749 | |
Steven Rostedt | c13db6b | 2014-07-23 11:28:26 -0400 | [diff] [blame] | 7750 | /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ |
| 7751 | if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { |
Peter Zijlstra | 7479f3c9c | 2014-01-15 17:05:04 +0100 | [diff] [blame] | 7752 | attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; |
| 7753 | policy &= ~SCHED_RESET_ON_FORK; |
| 7754 | attr.sched_policy = policy; |
| 7755 | } |
| 7756 | |
Peter Zijlstra | dbc7f06 | 2015-06-11 14:46:38 +0200 | [diff] [blame] | 7757 | return __sched_setscheduler(p, &attr, check, true); |
Peter Zijlstra | 7479f3c9c | 2014-01-15 17:05:04 +0100 | [diff] [blame] | 7758 | } |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7759 | /** |
| 7760 | * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. |
| 7761 | * @p: the task in question. |
| 7762 | * @policy: new policy. |
| 7763 | * @param: structure containing the new RT priority. |
| 7764 | * |
Peter Zijlstra | 7318d4c | 2020-04-21 12:09:13 +0200 | [diff] [blame] | 7765 | * Use sched_set_fifo(), read its comment. |
| 7766 | * |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7767 | * Return: 0 on success. An error code otherwise. |
| 7768 | * |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7769 | * NOTE that the task may be already dead. |
| 7770 | */ |
| 7771 | int sched_setscheduler(struct task_struct *p, int policy, |
KOSAKI Motohiro | fe7de49 | 2010-10-20 16:01:12 -0700 | [diff] [blame] | 7772 | const struct sched_param *param) |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7773 | { |
Peter Zijlstra | 7479f3c9c | 2014-01-15 17:05:04 +0100 | [diff] [blame] | 7774 | return _sched_setscheduler(p, policy, param, true); |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7775 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7776 | |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7777 | int sched_setattr(struct task_struct *p, const struct sched_attr *attr) |
| 7778 | { |
Peter Zijlstra | dbc7f06 | 2015-06-11 14:46:38 +0200 | [diff] [blame] | 7779 | return __sched_setscheduler(p, attr, true, true); |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7780 | } |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7781 | |
Juri Lelli | 794a56e | 2017-12-04 11:23:20 +0100 | [diff] [blame] | 7782 | int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr) |
| 7783 | { |
| 7784 | return __sched_setscheduler(p, attr, false, true); |
| 7785 | } |
Viresh Kumar | 1eb5dde | 2020-06-23 15:49:40 +0530 | [diff] [blame] | 7786 | EXPORT_SYMBOL_GPL(sched_setattr_nocheck); |
Juri Lelli | 794a56e | 2017-12-04 11:23:20 +0100 | [diff] [blame] | 7787 | |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7788 | /** |
| 7789 | * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. |
| 7790 | * @p: the task in question. |
| 7791 | * @policy: new policy. |
| 7792 | * @param: structure containing the new RT priority. |
| 7793 | * |
| 7794 | * Just like sched_setscheduler, only don't bother checking if the |
| 7795 | * current context has permission. For example, this is needed in |
| 7796 | * stop_machine(): we create temporary high priority worker threads, |
| 7797 | * but our caller might not have that capability. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7798 | * |
| 7799 | * Return: 0 on success. An error code otherwise. |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7800 | */ |
| 7801 | int sched_setscheduler_nocheck(struct task_struct *p, int policy, |
KOSAKI Motohiro | fe7de49 | 2010-10-20 16:01:12 -0700 | [diff] [blame] | 7802 | const struct sched_param *param) |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7803 | { |
Peter Zijlstra | 7479f3c9c | 2014-01-15 17:05:04 +0100 | [diff] [blame] | 7804 | return _sched_setscheduler(p, policy, param, false); |
Rusty Russell | 961ccdd | 2008-06-23 13:55:38 +1000 | [diff] [blame] | 7805 | } |
| 7806 | |
Peter Zijlstra | 7318d4c | 2020-04-21 12:09:13 +0200 | [diff] [blame] | 7807 | /* |
| 7808 | * SCHED_FIFO is a broken scheduler model; that is, it is fundamentally |
| 7809 | * incapable of resource management, which is the one thing an OS really should |
| 7810 | * be doing. |
| 7811 | * |
| 7812 | * This is of course the reason it is limited to privileged users only. |
| 7813 | * |
| 7814 | * Worse still; it is fundamentally impossible to compose static priority |
| 7815 | * workloads. You cannot take two correctly working static prio workloads |
| 7816 | * and smash them together and still expect them to work. |
| 7817 | * |
| 7818 | * For this reason 'all' FIFO tasks the kernel creates are basically at: |
| 7819 | * |
| 7820 | * MAX_RT_PRIO / 2 |
| 7821 | * |
| 7822 | * The administrator _MUST_ configure the system, the kernel simply doesn't |
| 7823 | * know enough information to make a sensible choice. |
| 7824 | */ |
Peter Zijlstra | 8b70098 | 2020-04-22 13:10:04 +0200 | [diff] [blame] | 7825 | void sched_set_fifo(struct task_struct *p) |
Peter Zijlstra | 7318d4c | 2020-04-21 12:09:13 +0200 | [diff] [blame] | 7826 | { |
| 7827 | struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 }; |
Peter Zijlstra | 8b70098 | 2020-04-22 13:10:04 +0200 | [diff] [blame] | 7828 | WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); |
Peter Zijlstra | 7318d4c | 2020-04-21 12:09:13 +0200 | [diff] [blame] | 7829 | } |
| 7830 | EXPORT_SYMBOL_GPL(sched_set_fifo); |
| 7831 | |
| 7832 | /* |
| 7833 | * For when you don't much care about FIFO, but want to be above SCHED_NORMAL. |
| 7834 | */ |
Peter Zijlstra | 8b70098 | 2020-04-22 13:10:04 +0200 | [diff] [blame] | 7835 | void sched_set_fifo_low(struct task_struct *p) |
Peter Zijlstra | 7318d4c | 2020-04-21 12:09:13 +0200 | [diff] [blame] | 7836 | { |
| 7837 | struct sched_param sp = { .sched_priority = 1 }; |
Peter Zijlstra | 8b70098 | 2020-04-22 13:10:04 +0200 | [diff] [blame] | 7838 | WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); |
Peter Zijlstra | 7318d4c | 2020-04-21 12:09:13 +0200 | [diff] [blame] | 7839 | } |
| 7840 | EXPORT_SYMBOL_GPL(sched_set_fifo_low); |
| 7841 | |
Peter Zijlstra | 8b70098 | 2020-04-22 13:10:04 +0200 | [diff] [blame] | 7842 | void sched_set_normal(struct task_struct *p, int nice) |
Peter Zijlstra | 7318d4c | 2020-04-21 12:09:13 +0200 | [diff] [blame] | 7843 | { |
| 7844 | struct sched_attr attr = { |
| 7845 | .sched_policy = SCHED_NORMAL, |
| 7846 | .sched_nice = nice, |
| 7847 | }; |
Peter Zijlstra | 8b70098 | 2020-04-22 13:10:04 +0200 | [diff] [blame] | 7848 | WARN_ON_ONCE(sched_setattr_nocheck(p, &attr) != 0); |
Peter Zijlstra | 7318d4c | 2020-04-21 12:09:13 +0200 | [diff] [blame] | 7849 | } |
| 7850 | EXPORT_SYMBOL_GPL(sched_set_normal); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7851 | |
| 7852 | static int |
| 7853 | do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) |
| 7854 | { |
| 7855 | struct sched_param lparam; |
Ingo Molnar | 41a2d6c | 2007-12-05 15:46:09 +0100 | [diff] [blame] | 7856 | struct task_struct *p; |
| 7857 | int retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7858 | |
Jason Baron | c21761f | 2006-01-18 17:43:03 -0800 | [diff] [blame] | 7859 | if (!param || pid < 0) |
| 7860 | return -EINVAL; |
| 7861 | if (copy_from_user(&lparam, param, sizeof(struct sched_param))) |
| 7862 | return -EFAULT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7863 | |
| 7864 | rcu_read_lock(); |
| 7865 | retval = -ESRCH; |
| 7866 | p = find_process_by_pid(pid); |
Juri Lelli | 710da3c | 2019-07-19 16:00:00 +0200 | [diff] [blame] | 7867 | if (likely(p)) |
| 7868 | get_task_struct(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7869 | rcu_read_unlock(); |
| 7870 | |
Juri Lelli | 710da3c | 2019-07-19 16:00:00 +0200 | [diff] [blame] | 7871 | if (likely(p)) { |
| 7872 | retval = sched_setscheduler(p, policy, &lparam); |
| 7873 | put_task_struct(p); |
| 7874 | } |
| 7875 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7876 | return retval; |
| 7877 | } |
| 7878 | |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7879 | /* |
| 7880 | * Mimics kernel/events/core.c perf_copy_attr(). |
| 7881 | */ |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7882 | static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7883 | { |
| 7884 | u32 size; |
| 7885 | int ret; |
| 7886 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7887 | /* Zero the full structure, so that a short copy will be nice: */ |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7888 | memset(attr, 0, sizeof(*attr)); |
| 7889 | |
| 7890 | ret = get_user(size, &uattr->size); |
| 7891 | if (ret) |
| 7892 | return ret; |
| 7893 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7894 | /* ABI compatibility quirk: */ |
| 7895 | if (!size) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7896 | size = SCHED_ATTR_SIZE_VER0; |
Aleksa Sarai | dff3a85 | 2019-10-01 11:10:54 +1000 | [diff] [blame] | 7897 | if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7898 | goto err_size; |
| 7899 | |
Aleksa Sarai | dff3a85 | 2019-10-01 11:10:54 +1000 | [diff] [blame] | 7900 | ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size); |
| 7901 | if (ret) { |
| 7902 | if (ret == -E2BIG) |
| 7903 | goto err_size; |
| 7904 | return ret; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7905 | } |
| 7906 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 7907 | if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) && |
| 7908 | size < SCHED_ATTR_SIZE_VER1) |
| 7909 | return -EINVAL; |
| 7910 | |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7911 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7912 | * XXX: Do we want to be lenient like existing syscalls; or do we want |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7913 | * to be strict and return an error on out-of-bounds values? |
| 7914 | */ |
Dongsheng Yang | 75e45d5 | 2014-02-11 15:34:50 +0800 | [diff] [blame] | 7915 | attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7916 | |
Michael Kerrisk | e78c7bc | 2014-05-09 16:54:28 +0200 | [diff] [blame] | 7917 | return 0; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7918 | |
| 7919 | err_size: |
| 7920 | put_user(sizeof(*attr), &uattr->size); |
Michael Kerrisk | e78c7bc | 2014-05-09 16:54:28 +0200 | [diff] [blame] | 7921 | return -E2BIG; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7922 | } |
| 7923 | |
Quentin Perret | f4dddf9 | 2021-08-05 11:21:54 +0100 | [diff] [blame] | 7924 | static void get_params(struct task_struct *p, struct sched_attr *attr) |
| 7925 | { |
| 7926 | if (task_has_dl_policy(p)) |
| 7927 | __getparam_dl(p, attr); |
| 7928 | else if (task_has_rt_policy(p)) |
| 7929 | attr->sched_priority = p->rt_priority; |
| 7930 | else |
| 7931 | attr->sched_nice = task_nice(p); |
| 7932 | } |
| 7933 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7934 | /** |
| 7935 | * sys_sched_setscheduler - set/change the scheduler policy and RT priority |
| 7936 | * @pid: the pid in question. |
| 7937 | * @policy: new policy. |
| 7938 | * @param: structure containing the new RT priority. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7939 | * |
| 7940 | * Return: 0 on success. An error code otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7941 | */ |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 7942 | SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7943 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7944 | if (policy < 0) |
| 7945 | return -EINVAL; |
| 7946 | |
| 7947 | return do_sched_setscheduler(pid, policy, param); |
| 7948 | } |
| 7949 | |
| 7950 | /** |
| 7951 | * sys_sched_setparam - set/change the RT priority of a thread |
| 7952 | * @pid: the pid in question. |
| 7953 | * @param: structure containing the new RT priority. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7954 | * |
| 7955 | * Return: 0 on success. An error code otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7956 | */ |
Heiko Carstens | 5add95d | 2009-01-14 14:14:08 +0100 | [diff] [blame] | 7957 | SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7958 | { |
Steven Rostedt | c13db6b | 2014-07-23 11:28:26 -0400 | [diff] [blame] | 7959 | return do_sched_setscheduler(pid, SETPARAM_POLICY, param); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7960 | } |
| 7961 | |
| 7962 | /** |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7963 | * sys_sched_setattr - same as above, but with extended sched_attr |
| 7964 | * @pid: the pid in question. |
Juri Lelli | 5778fcc | 2014-01-14 16:10:39 +0100 | [diff] [blame] | 7965 | * @uattr: structure containing the extended parameters. |
Masanari Iida | db66d75 | 2014-04-18 01:59:15 +0900 | [diff] [blame] | 7966 | * @flags: for future extension. |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7967 | */ |
Peter Zijlstra | 6d35ab4 | 2014-02-14 17:19:29 +0100 | [diff] [blame] | 7968 | SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, |
| 7969 | unsigned int, flags) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7970 | { |
| 7971 | struct sched_attr attr; |
| 7972 | struct task_struct *p; |
| 7973 | int retval; |
| 7974 | |
Peter Zijlstra | 6d35ab4 | 2014-02-14 17:19:29 +0100 | [diff] [blame] | 7975 | if (!uattr || pid < 0 || flags) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7976 | return -EINVAL; |
| 7977 | |
Michael Kerrisk | 143cf23 | 2014-05-09 16:54:15 +0200 | [diff] [blame] | 7978 | retval = sched_copy_attr(uattr, &attr); |
| 7979 | if (retval) |
| 7980 | return retval; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7981 | |
Richard Weinberger | b14ed2c | 2014-06-02 22:38:34 +0200 | [diff] [blame] | 7982 | if ((int)attr.sched_policy < 0) |
Peter Zijlstra | dbdb227 | 2014-05-09 10:49:03 +0200 | [diff] [blame] | 7983 | return -EINVAL; |
Patrick Bellasi | 1d6362f | 2019-06-21 09:42:06 +0100 | [diff] [blame] | 7984 | if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY) |
| 7985 | attr.sched_policy = SETPARAM_POLICY; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7986 | |
| 7987 | rcu_read_lock(); |
| 7988 | retval = -ESRCH; |
| 7989 | p = find_process_by_pid(pid); |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 7990 | if (likely(p)) |
| 7991 | get_task_struct(p); |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 7992 | rcu_read_unlock(); |
| 7993 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 7994 | if (likely(p)) { |
Quentin Perret | f4dddf9 | 2021-08-05 11:21:54 +0100 | [diff] [blame] | 7995 | if (attr.sched_flags & SCHED_FLAG_KEEP_PARAMS) |
| 7996 | get_params(p, &attr); |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 7997 | retval = sched_setattr(p, &attr); |
| 7998 | put_task_struct(p); |
| 7999 | } |
| 8000 | |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8001 | return retval; |
| 8002 | } |
| 8003 | |
| 8004 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8005 | * sys_sched_getscheduler - get the policy (scheduling class) of a thread |
| 8006 | * @pid: the pid in question. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 8007 | * |
| 8008 | * Return: On success, the policy of the thread. Otherwise, a negative error |
| 8009 | * code. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8010 | */ |
Heiko Carstens | 5add95d | 2009-01-14 14:14:08 +0100 | [diff] [blame] | 8011 | SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8012 | { |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 8013 | struct task_struct *p; |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 8014 | int retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8015 | |
| 8016 | if (pid < 0) |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 8017 | return -EINVAL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8018 | |
| 8019 | retval = -ESRCH; |
Thomas Gleixner | 5fe85be | 2009-12-09 10:14:58 +0000 | [diff] [blame] | 8020 | rcu_read_lock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8021 | p = find_process_by_pid(pid); |
| 8022 | if (p) { |
| 8023 | retval = security_task_getscheduler(p); |
| 8024 | if (!retval) |
Lennart Poettering | ca94c44 | 2009-06-15 17:17:47 +0200 | [diff] [blame] | 8025 | retval = p->policy |
| 8026 | | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8027 | } |
Thomas Gleixner | 5fe85be | 2009-12-09 10:14:58 +0000 | [diff] [blame] | 8028 | rcu_read_unlock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8029 | return retval; |
| 8030 | } |
| 8031 | |
| 8032 | /** |
Lennart Poettering | ca94c44 | 2009-06-15 17:17:47 +0200 | [diff] [blame] | 8033 | * sys_sched_getparam - get the RT priority of a thread |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8034 | * @pid: the pid in question. |
| 8035 | * @param: structure containing the RT priority. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 8036 | * |
| 8037 | * Return: On success, 0 and the RT priority is in @param. Otherwise, an error |
| 8038 | * code. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8039 | */ |
Heiko Carstens | 5add95d | 2009-01-14 14:14:08 +0100 | [diff] [blame] | 8040 | SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8041 | { |
Peter Zijlstra | ce5f7f8 | 2014-05-12 22:50:34 +0200 | [diff] [blame] | 8042 | struct sched_param lp = { .sched_priority = 0 }; |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 8043 | struct task_struct *p; |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 8044 | int retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8045 | |
| 8046 | if (!param || pid < 0) |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 8047 | return -EINVAL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8048 | |
Thomas Gleixner | 5fe85be | 2009-12-09 10:14:58 +0000 | [diff] [blame] | 8049 | rcu_read_lock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8050 | p = find_process_by_pid(pid); |
| 8051 | retval = -ESRCH; |
| 8052 | if (!p) |
| 8053 | goto out_unlock; |
| 8054 | |
| 8055 | retval = security_task_getscheduler(p); |
| 8056 | if (retval) |
| 8057 | goto out_unlock; |
| 8058 | |
Peter Zijlstra | ce5f7f8 | 2014-05-12 22:50:34 +0200 | [diff] [blame] | 8059 | if (task_has_rt_policy(p)) |
| 8060 | lp.sched_priority = p->rt_priority; |
Thomas Gleixner | 5fe85be | 2009-12-09 10:14:58 +0000 | [diff] [blame] | 8061 | rcu_read_unlock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8062 | |
| 8063 | /* |
| 8064 | * This one might sleep, we cannot do it with a spinlock held ... |
| 8065 | */ |
| 8066 | retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; |
| 8067 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8068 | return retval; |
| 8069 | |
| 8070 | out_unlock: |
Thomas Gleixner | 5fe85be | 2009-12-09 10:14:58 +0000 | [diff] [blame] | 8071 | rcu_read_unlock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8072 | return retval; |
| 8073 | } |
| 8074 | |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8075 | /* |
| 8076 | * Copy the kernel size attribute structure (which might be larger |
| 8077 | * than what user-space knows about) to user-space. |
| 8078 | * |
| 8079 | * Note that all cases are valid: user-space buffer can be larger or |
| 8080 | * smaller than the kernel-space buffer. The usual case is that both |
| 8081 | * have the same size. |
| 8082 | */ |
| 8083 | static int |
| 8084 | sched_attr_copy_to_user(struct sched_attr __user *uattr, |
| 8085 | struct sched_attr *kattr, |
| 8086 | unsigned int usize) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8087 | { |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8088 | unsigned int ksize = sizeof(*kattr); |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8089 | |
Linus Torvalds | 96d4f26 | 2019-01-03 18:57:57 -0800 | [diff] [blame] | 8090 | if (!access_ok(uattr, usize)) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8091 | return -EFAULT; |
| 8092 | |
| 8093 | /* |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8094 | * sched_getattr() ABI forwards and backwards compatibility: |
| 8095 | * |
| 8096 | * If usize == ksize then we just copy everything to user-space and all is good. |
| 8097 | * |
| 8098 | * If usize < ksize then we only copy as much as user-space has space for, |
| 8099 | * this keeps ABI compatibility as well. We skip the rest. |
| 8100 | * |
| 8101 | * If usize > ksize then user-space is using a newer version of the ABI, |
| 8102 | * which part the kernel doesn't know about. Just ignore it - tooling can |
| 8103 | * detect the kernel's knowledge of attributes from the attr->size value |
| 8104 | * which is set to ksize in this case. |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8105 | */ |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8106 | kattr->size = min(usize, ksize); |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8107 | |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8108 | if (copy_to_user(uattr, kattr, kattr->size)) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8109 | return -EFAULT; |
| 8110 | |
Michael Kerrisk | 2240067 | 2014-05-09 16:54:33 +0200 | [diff] [blame] | 8111 | return 0; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8112 | } |
| 8113 | |
| 8114 | /** |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 8115 | * sys_sched_getattr - similar to sched_getparam, but with sched_attr |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8116 | * @pid: the pid in question. |
Juri Lelli | 5778fcc | 2014-01-14 16:10:39 +0100 | [diff] [blame] | 8117 | * @uattr: structure containing the extended parameters. |
Aleksa Sarai | dff3a85 | 2019-10-01 11:10:54 +1000 | [diff] [blame] | 8118 | * @usize: sizeof(attr) for fwd/bwd comp. |
Masanari Iida | db66d75 | 2014-04-18 01:59:15 +0900 | [diff] [blame] | 8119 | * @flags: for future extension. |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8120 | */ |
Peter Zijlstra | 6d35ab4 | 2014-02-14 17:19:29 +0100 | [diff] [blame] | 8121 | SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8122 | unsigned int, usize, unsigned int, flags) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8123 | { |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8124 | struct sched_attr kattr = { }; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8125 | struct task_struct *p; |
| 8126 | int retval; |
| 8127 | |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8128 | if (!uattr || pid < 0 || usize > PAGE_SIZE || |
| 8129 | usize < SCHED_ATTR_SIZE_VER0 || flags) |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8130 | return -EINVAL; |
| 8131 | |
| 8132 | rcu_read_lock(); |
| 8133 | p = find_process_by_pid(pid); |
| 8134 | retval = -ESRCH; |
| 8135 | if (!p) |
| 8136 | goto out_unlock; |
| 8137 | |
| 8138 | retval = security_task_getscheduler(p); |
| 8139 | if (retval) |
| 8140 | goto out_unlock; |
| 8141 | |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8142 | kattr.sched_policy = p->policy; |
Peter Zijlstra | 7479f3c9c | 2014-01-15 17:05:04 +0100 | [diff] [blame] | 8143 | if (p->sched_reset_on_fork) |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8144 | kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; |
Quentin Perret | f4dddf9 | 2021-08-05 11:21:54 +0100 | [diff] [blame] | 8145 | get_params(p, &kattr); |
Quentin Perret | 7ad721b | 2021-07-27 11:11:02 +0100 | [diff] [blame] | 8146 | kattr.sched_flags &= SCHED_FLAG_ALL; |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8147 | |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 8148 | #ifdef CONFIG_UCLAMP_TASK |
Qais Yousef | 13685c4 | 2020-07-16 12:03:45 +0100 | [diff] [blame] | 8149 | /* |
| 8150 | * This could race with another potential updater, but this is fine |
| 8151 | * because it'll correctly read the old or the new value. We don't need |
| 8152 | * to guarantee who wins the race as long as it doesn't return garbage. |
| 8153 | */ |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8154 | kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value; |
| 8155 | kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value; |
Patrick Bellasi | a509a7cd | 2019-06-21 09:42:07 +0100 | [diff] [blame] | 8156 | #endif |
| 8157 | |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8158 | rcu_read_unlock(); |
| 8159 | |
Ingo Molnar | 1251201 | 2019-09-04 09:55:32 +0200 | [diff] [blame] | 8160 | return sched_attr_copy_to_user(uattr, &kattr, usize); |
Dario Faggioli | d50dde5 | 2013-11-07 14:43:36 +0100 | [diff] [blame] | 8161 | |
| 8162 | out_unlock: |
| 8163 | rcu_read_unlock(); |
| 8164 | return retval; |
| 8165 | } |
| 8166 | |
Will Deacon | 234b8ab | 2021-07-30 12:24:36 +0100 | [diff] [blame] | 8167 | #ifdef CONFIG_SMP |
| 8168 | int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask) |
| 8169 | { |
| 8170 | int ret = 0; |
| 8171 | |
| 8172 | /* |
| 8173 | * If the task isn't a deadline task or admission control is |
| 8174 | * disabled then we don't care about affinity changes. |
| 8175 | */ |
| 8176 | if (!task_has_dl_policy(p) || !dl_bandwidth_enabled()) |
| 8177 | return 0; |
| 8178 | |
| 8179 | /* |
| 8180 | * Since bandwidth control happens on root_domain basis, |
| 8181 | * if admission test is enabled, we only admit -deadline |
| 8182 | * tasks allowed to run on all the CPUs in the task's |
| 8183 | * root_domain. |
| 8184 | */ |
| 8185 | rcu_read_lock(); |
| 8186 | if (!cpumask_subset(task_rq(p)->rd->span, mask)) |
| 8187 | ret = -EBUSY; |
| 8188 | rcu_read_unlock(); |
| 8189 | return ret; |
| 8190 | } |
| 8191 | #endif |
| 8192 | |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8193 | static int |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 8194 | __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8195 | { |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 8196 | int retval; |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8197 | cpumask_var_t cpus_allowed, new_mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8198 | |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8199 | if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) |
| 8200 | return -ENOMEM; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8201 | |
Rusty Russell | 5a16f3d | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8202 | if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { |
| 8203 | retval = -ENOMEM; |
| 8204 | goto out_free_cpus_allowed; |
| 8205 | } |
Peter Zijlstra | e4099a5 | 2013-12-17 10:03:34 +0100 | [diff] [blame] | 8206 | |
| 8207 | cpuset_cpus_allowed(p, cpus_allowed); |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 8208 | cpumask_and(new_mask, ctx->new_mask, cpus_allowed); |
| 8209 | |
| 8210 | ctx->new_mask = new_mask; |
| 8211 | ctx->flags |= SCA_CHECK; |
Peter Zijlstra | e4099a5 | 2013-12-17 10:03:34 +0100 | [diff] [blame] | 8212 | |
Will Deacon | 234b8ab | 2021-07-30 12:24:36 +0100 | [diff] [blame] | 8213 | retval = dl_task_check_affinity(p, new_mask); |
| 8214 | if (retval) |
| 8215 | goto out_free_new_mask; |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 8216 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 8217 | retval = __set_cpus_allowed_ptr(p, ctx); |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8218 | if (retval) |
| 8219 | goto out_free_new_mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8220 | |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8221 | cpuset_cpus_allowed(p, cpus_allowed); |
| 8222 | if (!cpumask_subset(new_mask, cpus_allowed)) { |
| 8223 | /* |
| 8224 | * We must have raced with a concurrent cpuset update. |
| 8225 | * Just reset the cpumask to the cpuset's cpus_allowed. |
| 8226 | */ |
| 8227 | cpumask_copy(new_mask, cpus_allowed); |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 8228 | |
| 8229 | /* |
| 8230 | * If SCA_USER is set, a 2nd call to __set_cpus_allowed_ptr() |
| 8231 | * will restore the previous user_cpus_ptr value. |
| 8232 | * |
| 8233 | * In the unlikely event a previous user_cpus_ptr exists, |
| 8234 | * we need to further restrict the mask to what is allowed |
| 8235 | * by that old user_cpus_ptr. |
| 8236 | */ |
| 8237 | if (unlikely((ctx->flags & SCA_USER) && ctx->user_mask)) { |
| 8238 | bool empty = !cpumask_and(new_mask, new_mask, |
| 8239 | ctx->user_mask); |
| 8240 | |
| 8241 | if (WARN_ON_ONCE(empty)) |
| 8242 | cpumask_copy(new_mask, cpus_allowed); |
| 8243 | } |
| 8244 | __set_cpus_allowed_ptr(p, ctx); |
| 8245 | retval = -EINVAL; |
Paul Menage | 8707d8b | 2007-10-18 23:40:22 -0700 | [diff] [blame] | 8246 | } |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8247 | |
Kirill Tkhai | 16303ab | 2014-09-22 22:36:30 +0400 | [diff] [blame] | 8248 | out_free_new_mask: |
Rusty Russell | 5a16f3d | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8249 | free_cpumask_var(new_mask); |
| 8250 | out_free_cpus_allowed: |
| 8251 | free_cpumask_var(cpus_allowed); |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8252 | return retval; |
| 8253 | } |
| 8254 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8255 | long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) |
| 8256 | { |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 8257 | struct affinity_context ac; |
| 8258 | struct cpumask *user_mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8259 | struct task_struct *p; |
| 8260 | int retval; |
| 8261 | |
| 8262 | rcu_read_lock(); |
| 8263 | |
| 8264 | p = find_process_by_pid(pid); |
| 8265 | if (!p) { |
| 8266 | rcu_read_unlock(); |
| 8267 | return -ESRCH; |
| 8268 | } |
| 8269 | |
| 8270 | /* Prevent p going away */ |
| 8271 | get_task_struct(p); |
| 8272 | rcu_read_unlock(); |
| 8273 | |
| 8274 | if (p->flags & PF_NO_SETAFFINITY) { |
| 8275 | retval = -EINVAL; |
| 8276 | goto out_put_task; |
| 8277 | } |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8278 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8279 | if (!check_same_owner(p)) { |
| 8280 | rcu_read_lock(); |
| 8281 | if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { |
| 8282 | rcu_read_unlock(); |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8283 | retval = -EPERM; |
| 8284 | goto out_put_task; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8285 | } |
| 8286 | rcu_read_unlock(); |
| 8287 | } |
| 8288 | |
| 8289 | retval = security_task_setscheduler(p); |
| 8290 | if (retval) |
Will Deacon | db3b02a | 2021-07-30 12:24:34 +0100 | [diff] [blame] | 8291 | goto out_put_task; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8292 | |
Waiman Long | 5657c11 | 2023-01-15 14:31:22 -0500 | [diff] [blame] | 8293 | /* |
| 8294 | * With non-SMP configs, user_cpus_ptr/user_mask isn't used and |
| 8295 | * alloc_user_cpus_ptr() returns NULL. |
| 8296 | */ |
Waiman Long | 9a5418b | 2022-12-30 23:11:20 -0500 | [diff] [blame] | 8297 | user_mask = alloc_user_cpus_ptr(NUMA_NO_NODE); |
Waiman Long | 5657c11 | 2023-01-15 14:31:22 -0500 | [diff] [blame] | 8298 | if (user_mask) { |
| 8299 | cpumask_copy(user_mask, in_mask); |
| 8300 | } else if (IS_ENABLED(CONFIG_SMP)) { |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 8301 | retval = -ENOMEM; |
| 8302 | goto out_put_task; |
| 8303 | } |
Waiman Long | 5657c11 | 2023-01-15 14:31:22 -0500 | [diff] [blame] | 8304 | |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 8305 | ac = (struct affinity_context){ |
| 8306 | .new_mask = in_mask, |
| 8307 | .user_mask = user_mask, |
| 8308 | .flags = SCA_USER, |
| 8309 | }; |
| 8310 | |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 8311 | retval = __sched_setaffinity(p, &ac); |
Waiman Long | 8f9ea86 | 2022-09-22 14:00:38 -0400 | [diff] [blame] | 8312 | kfree(ac.user_mask); |
| 8313 | |
Rusty Russell | 5a16f3d | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8314 | out_put_task: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8315 | put_task_struct(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8316 | return retval; |
| 8317 | } |
| 8318 | |
| 8319 | static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, |
Rusty Russell | 96f874e2 | 2008-11-25 02:35:14 +1030 | [diff] [blame] | 8320 | struct cpumask *new_mask) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8321 | { |
Rusty Russell | 96f874e2 | 2008-11-25 02:35:14 +1030 | [diff] [blame] | 8322 | if (len < cpumask_size()) |
| 8323 | cpumask_clear(new_mask); |
| 8324 | else if (len > cpumask_size()) |
| 8325 | len = cpumask_size(); |
| 8326 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8327 | return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; |
| 8328 | } |
| 8329 | |
| 8330 | /** |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 8331 | * sys_sched_setaffinity - set the CPU affinity of a process |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8332 | * @pid: pid of the process |
| 8333 | * @len: length in bytes of the bitmask pointed to by user_mask_ptr |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 8334 | * @user_mask_ptr: user-space pointer to the new CPU mask |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 8335 | * |
| 8336 | * Return: 0 on success. An error code otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8337 | */ |
Heiko Carstens | 5add95d | 2009-01-14 14:14:08 +0100 | [diff] [blame] | 8338 | SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, |
| 8339 | unsigned long __user *, user_mask_ptr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8340 | { |
Rusty Russell | 5a16f3d | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8341 | cpumask_var_t new_mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8342 | int retval; |
| 8343 | |
Rusty Russell | 5a16f3d | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8344 | if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) |
| 8345 | return -ENOMEM; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8346 | |
Rusty Russell | 5a16f3d | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8347 | retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); |
| 8348 | if (retval == 0) |
| 8349 | retval = sched_setaffinity(pid, new_mask); |
| 8350 | free_cpumask_var(new_mask); |
| 8351 | return retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8352 | } |
| 8353 | |
Rusty Russell | 96f874e2 | 2008-11-25 02:35:14 +1030 | [diff] [blame] | 8354 | long sched_getaffinity(pid_t pid, struct cpumask *mask) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8355 | { |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 8356 | struct task_struct *p; |
Thomas Gleixner | 3160568 | 2009-12-08 20:24:16 +0000 | [diff] [blame] | 8357 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8358 | int retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8359 | |
Thomas Gleixner | 23f5d14 | 2009-12-09 10:15:01 +0000 | [diff] [blame] | 8360 | rcu_read_lock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8361 | |
| 8362 | retval = -ESRCH; |
| 8363 | p = find_process_by_pid(pid); |
| 8364 | if (!p) |
| 8365 | goto out_unlock; |
| 8366 | |
David Quigley | e7834f8 | 2006-06-23 02:03:59 -0700 | [diff] [blame] | 8367 | retval = security_task_getscheduler(p); |
| 8368 | if (retval) |
| 8369 | goto out_unlock; |
| 8370 | |
Peter Zijlstra | 013fdb8 | 2011-04-05 17:23:45 +0200 | [diff] [blame] | 8371 | raw_spin_lock_irqsave(&p->pi_lock, flags); |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 8372 | cpumask_and(mask, &p->cpus_mask, cpu_active_mask); |
Peter Zijlstra | 013fdb8 | 2011-04-05 17:23:45 +0200 | [diff] [blame] | 8373 | raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8374 | |
| 8375 | out_unlock: |
Thomas Gleixner | 23f5d14 | 2009-12-09 10:15:01 +0000 | [diff] [blame] | 8376 | rcu_read_unlock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8377 | |
Ulrich Drepper | 9531b62 | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 8378 | return retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8379 | } |
| 8380 | |
| 8381 | /** |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 8382 | * sys_sched_getaffinity - get the CPU affinity of a process |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8383 | * @pid: pid of the process |
| 8384 | * @len: length in bytes of the bitmask pointed to by user_mask_ptr |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 8385 | * @user_mask_ptr: user-space pointer to hold the current CPU mask |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 8386 | * |
Zev Weiss | 599b484 | 2016-06-26 16:13:23 -0500 | [diff] [blame] | 8387 | * Return: size of CPU mask copied to user_mask_ptr on success. An |
| 8388 | * error code otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8389 | */ |
Heiko Carstens | 5add95d | 2009-01-14 14:14:08 +0100 | [diff] [blame] | 8390 | SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, |
| 8391 | unsigned long __user *, user_mask_ptr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8392 | { |
| 8393 | int ret; |
Rusty Russell | f17c860 | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8394 | cpumask_var_t mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8395 | |
Anton Blanchard | 84fba5e | 2010-04-06 17:02:19 +1000 | [diff] [blame] | 8396 | if ((len * BITS_PER_BYTE) < nr_cpu_ids) |
KOSAKI Motohiro | cd3d803 | 2010-03-12 16:15:36 +0900 | [diff] [blame] | 8397 | return -EINVAL; |
| 8398 | if (len & (sizeof(unsigned long)-1)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8399 | return -EINVAL; |
| 8400 | |
Rusty Russell | f17c860 | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8401 | if (!alloc_cpumask_var(&mask, GFP_KERNEL)) |
| 8402 | return -ENOMEM; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8403 | |
Rusty Russell | f17c860 | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8404 | ret = sched_getaffinity(pid, mask); |
| 8405 | if (ret == 0) { |
Alexey Dobriyan | 4de373a | 2018-02-06 15:39:37 -0800 | [diff] [blame] | 8406 | unsigned int retlen = min(len, cpumask_size()); |
KOSAKI Motohiro | cd3d803 | 2010-03-12 16:15:36 +0900 | [diff] [blame] | 8407 | |
| 8408 | if (copy_to_user(user_mask_ptr, mask, retlen)) |
Rusty Russell | f17c860 | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8409 | ret = -EFAULT; |
| 8410 | else |
KOSAKI Motohiro | cd3d803 | 2010-03-12 16:15:36 +0900 | [diff] [blame] | 8411 | ret = retlen; |
Rusty Russell | f17c860 | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8412 | } |
| 8413 | free_cpumask_var(mask); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8414 | |
Rusty Russell | f17c860 | 2008-11-25 02:35:11 +1030 | [diff] [blame] | 8415 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8416 | } |
| 8417 | |
Dominik Brodowski | 7d4dd4f | 2018-03-14 22:40:35 +0100 | [diff] [blame] | 8418 | static void do_sched_yield(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8419 | { |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8420 | struct rq_flags rf; |
| 8421 | struct rq *rq; |
| 8422 | |
Johannes Weiner | 246b3b3 | 2018-10-26 15:06:23 -0700 | [diff] [blame] | 8423 | rq = this_rq_lock_irq(&rf); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8424 | |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8425 | schedstat_inc(rq->yld_count); |
Dmitry Adamushko | 4530d7a | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 8426 | current->sched_class->yield_task(rq); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8427 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8428 | preempt_disable(); |
Thomas Gleixner | 345a957 | 2020-10-20 16:46:55 +0200 | [diff] [blame] | 8429 | rq_unlock_irq(rq, &rf); |
Thomas Gleixner | ba74c14 | 2011-03-21 13:32:17 +0100 | [diff] [blame] | 8430 | sched_preempt_enable_no_resched(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8431 | |
| 8432 | schedule(); |
Dominik Brodowski | 7d4dd4f | 2018-03-14 22:40:35 +0100 | [diff] [blame] | 8433 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8434 | |
Mauro Carvalho Chehab | 59a74b1 | 2020-12-01 13:09:06 +0100 | [diff] [blame] | 8435 | /** |
| 8436 | * sys_sched_yield - yield the current processor to other threads. |
| 8437 | * |
| 8438 | * This function yields the current CPU to other tasks. If there are no |
| 8439 | * other threads running on this CPU then this function will return. |
| 8440 | * |
| 8441 | * Return: 0. |
| 8442 | */ |
Dominik Brodowski | 7d4dd4f | 2018-03-14 22:40:35 +0100 | [diff] [blame] | 8443 | SYSCALL_DEFINE0(sched_yield) |
| 8444 | { |
| 8445 | do_sched_yield(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8446 | return 0; |
| 8447 | } |
| 8448 | |
Peter Zijlstra (Intel) | b965f1d | 2021-01-18 15:12:20 +0100 | [diff] [blame] | 8449 | #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) |
| 8450 | int __sched __cond_resched(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8451 | { |
Konstantin Khlebnikov | fe32d3c | 2015-07-15 12:52:04 +0300 | [diff] [blame] | 8452 | if (should_resched(0)) { |
Frederic Weisbecker | a18b5d0 | 2015-01-22 18:08:04 +0100 | [diff] [blame] | 8453 | preempt_schedule_common(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8454 | return 1; |
| 8455 | } |
Frederic Weisbecker | 5089582 | 2021-07-06 01:43:43 +0200 | [diff] [blame] | 8456 | /* |
| 8457 | * In preemptible kernels, ->rcu_read_lock_nesting tells the tick |
| 8458 | * whether the current CPU is in an RCU read-side critical section, |
| 8459 | * so the tick can report quiescent states even for CPUs looping |
| 8460 | * in kernel context. In contrast, in non-preemptible kernels, |
| 8461 | * RCU readers leave no in-memory hints, which means that CPU-bound |
| 8462 | * processes executing in kernel context might never report an |
| 8463 | * RCU quiescent state. Therefore, the following code causes |
| 8464 | * cond_resched() to report a quiescent state, but only when RCU |
| 8465 | * is in urgent need of one. |
| 8466 | */ |
Peter Zijlstra (Intel) | b965f1d | 2021-01-18 15:12:20 +0100 | [diff] [blame] | 8467 | #ifndef CONFIG_PREEMPT_RCU |
Paul E. McKenney | f79c3ad | 2016-11-30 06:24:30 -0800 | [diff] [blame] | 8468 | rcu_all_qs(); |
Peter Zijlstra (Intel) | b965f1d | 2021-01-18 15:12:20 +0100 | [diff] [blame] | 8469 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8470 | return 0; |
| 8471 | } |
Peter Zijlstra (Intel) | b965f1d | 2021-01-18 15:12:20 +0100 | [diff] [blame] | 8472 | EXPORT_SYMBOL(__cond_resched); |
| 8473 | #endif |
| 8474 | |
| 8475 | #ifdef CONFIG_PREEMPT_DYNAMIC |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 8476 | #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 8477 | #define cond_resched_dynamic_enabled __cond_resched |
| 8478 | #define cond_resched_dynamic_disabled ((void *)&__static_call_return0) |
Peter Zijlstra (Intel) | b965f1d | 2021-01-18 15:12:20 +0100 | [diff] [blame] | 8479 | DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); |
Peter Zijlstra | ef72661 | 2021-01-25 16:26:50 +0100 | [diff] [blame] | 8480 | EXPORT_STATIC_CALL_TRAMP(cond_resched); |
Peter Zijlstra (Intel) | b965f1d | 2021-01-18 15:12:20 +0100 | [diff] [blame] | 8481 | |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 8482 | #define might_resched_dynamic_enabled __cond_resched |
| 8483 | #define might_resched_dynamic_disabled ((void *)&__static_call_return0) |
Peter Zijlstra (Intel) | b965f1d | 2021-01-18 15:12:20 +0100 | [diff] [blame] | 8484 | DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); |
Peter Zijlstra | ef72661 | 2021-01-25 16:26:50 +0100 | [diff] [blame] | 8485 | EXPORT_STATIC_CALL_TRAMP(might_resched); |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 8486 | #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) |
| 8487 | static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); |
| 8488 | int __sched dynamic_cond_resched(void) |
| 8489 | { |
| 8490 | if (!static_branch_unlikely(&sk_dynamic_cond_resched)) |
| 8491 | return 0; |
| 8492 | return __cond_resched(); |
| 8493 | } |
| 8494 | EXPORT_SYMBOL(dynamic_cond_resched); |
| 8495 | |
| 8496 | static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); |
| 8497 | int __sched dynamic_might_resched(void) |
| 8498 | { |
| 8499 | if (!static_branch_unlikely(&sk_dynamic_might_resched)) |
| 8500 | return 0; |
| 8501 | return __cond_resched(); |
| 8502 | } |
| 8503 | EXPORT_SYMBOL(dynamic_might_resched); |
| 8504 | #endif |
Peter Zijlstra | 35a773a | 2016-09-19 12:57:53 +0200 | [diff] [blame] | 8505 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8506 | |
| 8507 | /* |
Frederic Weisbecker | 613afbf | 2009-07-16 15:44:29 +0200 | [diff] [blame] | 8508 | * __cond_resched_lock() - if a reschedule is pending, drop the given lock, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8509 | * call schedule, and on return reacquire the lock. |
| 8510 | * |
Thomas Gleixner | c1a280b | 2019-07-26 23:19:37 +0200 | [diff] [blame] | 8511 | * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8512 | * operations here to prevent schedule() from being called twice (once via |
| 8513 | * spin_unlock(), once by hand). |
| 8514 | */ |
Frederic Weisbecker | 613afbf | 2009-07-16 15:44:29 +0200 | [diff] [blame] | 8515 | int __cond_resched_lock(spinlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8516 | { |
Konstantin Khlebnikov | fe32d3c | 2015-07-15 12:52:04 +0300 | [diff] [blame] | 8517 | int resched = should_resched(PREEMPT_LOCK_OFFSET); |
Jan Kara | 6df3cec | 2005-06-13 15:52:32 -0700 | [diff] [blame] | 8518 | int ret = 0; |
| 8519 | |
Peter Zijlstra | f607c66 | 2009-07-20 19:16:29 +0200 | [diff] [blame] | 8520 | lockdep_assert_held(lock); |
| 8521 | |
Paul E. McKenney | 4a81e83 | 2014-06-20 16:49:01 -0700 | [diff] [blame] | 8522 | if (spin_needbreak(lock) || resched) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8523 | spin_unlock(lock); |
Peter Zijlstra | 7e406d1f | 2021-12-25 01:04:57 +0100 | [diff] [blame] | 8524 | if (!_cond_resched()) |
Nick Piggin | 95c354f | 2008-01-30 13:31:20 +0100 | [diff] [blame] | 8525 | cpu_relax(); |
Jan Kara | 6df3cec | 2005-06-13 15:52:32 -0700 | [diff] [blame] | 8526 | ret = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8527 | spin_lock(lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8528 | } |
Jan Kara | 6df3cec | 2005-06-13 15:52:32 -0700 | [diff] [blame] | 8529 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8530 | } |
Frederic Weisbecker | 613afbf | 2009-07-16 15:44:29 +0200 | [diff] [blame] | 8531 | EXPORT_SYMBOL(__cond_resched_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8532 | |
Ben Gardon | f3d4b4b | 2021-02-02 10:57:14 -0800 | [diff] [blame] | 8533 | int __cond_resched_rwlock_read(rwlock_t *lock) |
| 8534 | { |
| 8535 | int resched = should_resched(PREEMPT_LOCK_OFFSET); |
| 8536 | int ret = 0; |
| 8537 | |
| 8538 | lockdep_assert_held_read(lock); |
| 8539 | |
| 8540 | if (rwlock_needbreak(lock) || resched) { |
| 8541 | read_unlock(lock); |
Peter Zijlstra | 7e406d1f | 2021-12-25 01:04:57 +0100 | [diff] [blame] | 8542 | if (!_cond_resched()) |
Ben Gardon | f3d4b4b | 2021-02-02 10:57:14 -0800 | [diff] [blame] | 8543 | cpu_relax(); |
| 8544 | ret = 1; |
| 8545 | read_lock(lock); |
| 8546 | } |
| 8547 | return ret; |
| 8548 | } |
| 8549 | EXPORT_SYMBOL(__cond_resched_rwlock_read); |
| 8550 | |
| 8551 | int __cond_resched_rwlock_write(rwlock_t *lock) |
| 8552 | { |
| 8553 | int resched = should_resched(PREEMPT_LOCK_OFFSET); |
| 8554 | int ret = 0; |
| 8555 | |
| 8556 | lockdep_assert_held_write(lock); |
| 8557 | |
| 8558 | if (rwlock_needbreak(lock) || resched) { |
| 8559 | write_unlock(lock); |
Peter Zijlstra | 7e406d1f | 2021-12-25 01:04:57 +0100 | [diff] [blame] | 8560 | if (!_cond_resched()) |
Ben Gardon | f3d4b4b | 2021-02-02 10:57:14 -0800 | [diff] [blame] | 8561 | cpu_relax(); |
| 8562 | ret = 1; |
| 8563 | write_lock(lock); |
| 8564 | } |
| 8565 | return ret; |
| 8566 | } |
| 8567 | EXPORT_SYMBOL(__cond_resched_rwlock_write); |
| 8568 | |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8569 | #ifdef CONFIG_PREEMPT_DYNAMIC |
| 8570 | |
Mark Rutland | 33c6473 | 2022-02-14 16:52:13 +0000 | [diff] [blame] | 8571 | #ifdef CONFIG_GENERIC_ENTRY |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8572 | #include <linux/entry-common.h> |
Mark Rutland | 33c6473 | 2022-02-14 16:52:13 +0000 | [diff] [blame] | 8573 | #endif |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8574 | |
| 8575 | /* |
| 8576 | * SC:cond_resched |
| 8577 | * SC:might_resched |
| 8578 | * SC:preempt_schedule |
| 8579 | * SC:preempt_schedule_notrace |
| 8580 | * SC:irqentry_exit_cond_resched |
| 8581 | * |
| 8582 | * |
| 8583 | * NONE: |
| 8584 | * cond_resched <- __cond_resched |
| 8585 | * might_resched <- RET0 |
| 8586 | * preempt_schedule <- NOP |
| 8587 | * preempt_schedule_notrace <- NOP |
| 8588 | * irqentry_exit_cond_resched <- NOP |
| 8589 | * |
| 8590 | * VOLUNTARY: |
| 8591 | * cond_resched <- __cond_resched |
| 8592 | * might_resched <- __cond_resched |
| 8593 | * preempt_schedule <- NOP |
| 8594 | * preempt_schedule_notrace <- NOP |
| 8595 | * irqentry_exit_cond_resched <- NOP |
| 8596 | * |
| 8597 | * FULL: |
| 8598 | * cond_resched <- RET0 |
| 8599 | * might_resched <- RET0 |
| 8600 | * preempt_schedule <- preempt_schedule |
| 8601 | * preempt_schedule_notrace <- preempt_schedule_notrace |
| 8602 | * irqentry_exit_cond_resched <- irqentry_exit_cond_resched |
| 8603 | */ |
| 8604 | |
| 8605 | enum { |
| 8606 | preempt_dynamic_undefined = -1, |
| 8607 | preempt_dynamic_none, |
| 8608 | preempt_dynamic_voluntary, |
| 8609 | preempt_dynamic_full, |
| 8610 | }; |
| 8611 | |
| 8612 | int preempt_dynamic_mode = preempt_dynamic_undefined; |
| 8613 | |
| 8614 | int sched_dynamic_mode(const char *str) |
| 8615 | { |
| 8616 | if (!strcmp(str, "none")) |
| 8617 | return preempt_dynamic_none; |
| 8618 | |
| 8619 | if (!strcmp(str, "voluntary")) |
| 8620 | return preempt_dynamic_voluntary; |
| 8621 | |
| 8622 | if (!strcmp(str, "full")) |
| 8623 | return preempt_dynamic_full; |
| 8624 | |
| 8625 | return -EINVAL; |
| 8626 | } |
| 8627 | |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 8628 | #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 8629 | #define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) |
| 8630 | #define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) |
Mark Rutland | 99cf983c | 2022-02-14 16:52:14 +0000 | [diff] [blame] | 8631 | #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) |
| 8632 | #define preempt_dynamic_enable(f) static_key_enable(&sk_dynamic_##f.key) |
| 8633 | #define preempt_dynamic_disable(f) static_key_disable(&sk_dynamic_##f.key) |
| 8634 | #else |
| 8635 | #error "Unsupported PREEMPT_DYNAMIC mechanism" |
| 8636 | #endif |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 8637 | |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8638 | void sched_dynamic_update(int mode) |
| 8639 | { |
| 8640 | /* |
| 8641 | * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in |
| 8642 | * the ZERO state, which is invalid. |
| 8643 | */ |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 8644 | preempt_dynamic_enable(cond_resched); |
| 8645 | preempt_dynamic_enable(might_resched); |
| 8646 | preempt_dynamic_enable(preempt_schedule); |
| 8647 | preempt_dynamic_enable(preempt_schedule_notrace); |
| 8648 | preempt_dynamic_enable(irqentry_exit_cond_resched); |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8649 | |
| 8650 | switch (mode) { |
| 8651 | case preempt_dynamic_none: |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 8652 | preempt_dynamic_enable(cond_resched); |
| 8653 | preempt_dynamic_disable(might_resched); |
| 8654 | preempt_dynamic_disable(preempt_schedule); |
| 8655 | preempt_dynamic_disable(preempt_schedule_notrace); |
| 8656 | preempt_dynamic_disable(irqentry_exit_cond_resched); |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8657 | pr_info("Dynamic Preempt: none\n"); |
| 8658 | break; |
| 8659 | |
| 8660 | case preempt_dynamic_voluntary: |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 8661 | preempt_dynamic_enable(cond_resched); |
| 8662 | preempt_dynamic_enable(might_resched); |
| 8663 | preempt_dynamic_disable(preempt_schedule); |
| 8664 | preempt_dynamic_disable(preempt_schedule_notrace); |
| 8665 | preempt_dynamic_disable(irqentry_exit_cond_resched); |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8666 | pr_info("Dynamic Preempt: voluntary\n"); |
| 8667 | break; |
| 8668 | |
| 8669 | case preempt_dynamic_full: |
Mark Rutland | 8a69fe0 | 2022-02-14 16:52:11 +0000 | [diff] [blame] | 8670 | preempt_dynamic_disable(cond_resched); |
| 8671 | preempt_dynamic_disable(might_resched); |
| 8672 | preempt_dynamic_enable(preempt_schedule); |
| 8673 | preempt_dynamic_enable(preempt_schedule_notrace); |
| 8674 | preempt_dynamic_enable(irqentry_exit_cond_resched); |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8675 | pr_info("Dynamic Preempt: full\n"); |
| 8676 | break; |
| 8677 | } |
| 8678 | |
| 8679 | preempt_dynamic_mode = mode; |
| 8680 | } |
| 8681 | |
| 8682 | static int __init setup_preempt_mode(char *str) |
| 8683 | { |
| 8684 | int mode = sched_dynamic_mode(str); |
| 8685 | if (mode < 0) { |
| 8686 | pr_warn("Dynamic Preempt: unsupported mode: %s\n", str); |
| 8687 | return 0; |
| 8688 | } |
| 8689 | |
| 8690 | sched_dynamic_update(mode); |
| 8691 | return 1; |
| 8692 | } |
| 8693 | __setup("preempt=", setup_preempt_mode); |
| 8694 | |
| 8695 | static void __init preempt_dynamic_init(void) |
| 8696 | { |
| 8697 | if (preempt_dynamic_mode == preempt_dynamic_undefined) { |
| 8698 | if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { |
| 8699 | sched_dynamic_update(preempt_dynamic_none); |
| 8700 | } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { |
| 8701 | sched_dynamic_update(preempt_dynamic_voluntary); |
| 8702 | } else { |
| 8703 | /* Default static call setting, nothing to do */ |
| 8704 | WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT)); |
| 8705 | preempt_dynamic_mode = preempt_dynamic_full; |
| 8706 | pr_info("Dynamic Preempt: full\n"); |
| 8707 | } |
| 8708 | } |
| 8709 | } |
| 8710 | |
Valentin Schneider | cfe43f4 | 2021-11-12 18:52:01 +0000 | [diff] [blame] | 8711 | #define PREEMPT_MODEL_ACCESSOR(mode) \ |
| 8712 | bool preempt_model_##mode(void) \ |
| 8713 | { \ |
| 8714 | WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \ |
| 8715 | return preempt_dynamic_mode == preempt_dynamic_##mode; \ |
| 8716 | } \ |
| 8717 | EXPORT_SYMBOL_GPL(preempt_model_##mode) |
| 8718 | |
| 8719 | PREEMPT_MODEL_ACCESSOR(none); |
| 8720 | PREEMPT_MODEL_ACCESSOR(voluntary); |
| 8721 | PREEMPT_MODEL_ACCESSOR(full); |
| 8722 | |
Mark Rutland | 4c74855 | 2022-02-14 16:52:10 +0000 | [diff] [blame] | 8723 | #else /* !CONFIG_PREEMPT_DYNAMIC */ |
| 8724 | |
| 8725 | static inline void preempt_dynamic_init(void) { } |
| 8726 | |
| 8727 | #endif /* #ifdef CONFIG_PREEMPT_DYNAMIC */ |
| 8728 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8729 | /** |
| 8730 | * yield - yield the current processor to other threads. |
| 8731 | * |
Peter Zijlstra | 8e3fabf | 2012-03-06 18:54:26 +0100 | [diff] [blame] | 8732 | * Do not ever use this function, there's a 99% chance you're doing it wrong. |
| 8733 | * |
| 8734 | * The scheduler is at all times free to pick the calling task as the most |
| 8735 | * eligible task to run, if removing the yield() call from your code breaks |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 8736 | * it, it's already broken. |
Peter Zijlstra | 8e3fabf | 2012-03-06 18:54:26 +0100 | [diff] [blame] | 8737 | * |
| 8738 | * Typical broken usage is: |
| 8739 | * |
| 8740 | * while (!event) |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 8741 | * yield(); |
Peter Zijlstra | 8e3fabf | 2012-03-06 18:54:26 +0100 | [diff] [blame] | 8742 | * |
| 8743 | * where one assumes that yield() will let 'the other' process run that will |
| 8744 | * make event true. If the current task is a SCHED_FIFO task that will never |
| 8745 | * happen. Never use yield() as a progress guarantee!! |
| 8746 | * |
| 8747 | * If you want to use yield() to wait for something, use wait_event(). |
| 8748 | * If you want to use yield() to be 'nice' for others, use cond_resched(). |
| 8749 | * If you still want to use yield(), do not! |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8750 | */ |
| 8751 | void __sched yield(void) |
| 8752 | { |
| 8753 | set_current_state(TASK_RUNNING); |
Dominik Brodowski | 7d4dd4f | 2018-03-14 22:40:35 +0100 | [diff] [blame] | 8754 | do_sched_yield(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8755 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8756 | EXPORT_SYMBOL(yield); |
| 8757 | |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8758 | /** |
| 8759 | * yield_to - yield the current processor to another thread in |
| 8760 | * your thread group, or accelerate that thread toward the |
| 8761 | * processor it's on. |
Randy Dunlap | 16addf9 | 2011-03-18 09:34:53 -0700 | [diff] [blame] | 8762 | * @p: target task |
| 8763 | * @preempt: whether task preemption is allowed or not |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8764 | * |
| 8765 | * It's the caller's job to ensure that the target task struct |
| 8766 | * can't go away on us before we can do any checks. |
| 8767 | * |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 8768 | * Return: |
Peter Zijlstra | 7b270f6 | 2013-01-22 13:09:13 +0530 | [diff] [blame] | 8769 | * true (>0) if we indeed boosted the target task. |
| 8770 | * false (0) if we failed to boost the target. |
| 8771 | * -ESRCH if there's no task to yield to. |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8772 | */ |
Dan Carpenter | fa93384 | 2014-05-23 13:20:42 +0300 | [diff] [blame] | 8773 | int __sched yield_to(struct task_struct *p, bool preempt) |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8774 | { |
| 8775 | struct task_struct *curr = current; |
| 8776 | struct rq *rq, *p_rq; |
| 8777 | unsigned long flags; |
Dan Carpenter | c3c1864 | 2013-02-05 14:37:51 +0300 | [diff] [blame] | 8778 | int yielded = 0; |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8779 | |
| 8780 | local_irq_save(flags); |
| 8781 | rq = this_rq(); |
| 8782 | |
| 8783 | again: |
| 8784 | p_rq = task_rq(p); |
Peter Zijlstra | 7b270f6 | 2013-01-22 13:09:13 +0530 | [diff] [blame] | 8785 | /* |
| 8786 | * If we're the only runnable task on the rq and target rq also |
| 8787 | * has only one task, there's absolutely no point in yielding. |
| 8788 | */ |
| 8789 | if (rq->nr_running == 1 && p_rq->nr_running == 1) { |
| 8790 | yielded = -ESRCH; |
| 8791 | goto out_irq; |
| 8792 | } |
| 8793 | |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8794 | double_rq_lock(rq, p_rq); |
Shigeru Yoshida | 39e24d8f | 2013-11-23 18:38:01 +0900 | [diff] [blame] | 8795 | if (task_rq(p) != p_rq) { |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8796 | double_rq_unlock(rq, p_rq); |
| 8797 | goto again; |
| 8798 | } |
| 8799 | |
| 8800 | if (!curr->sched_class->yield_to_task) |
Peter Zijlstra | 7b270f6 | 2013-01-22 13:09:13 +0530 | [diff] [blame] | 8801 | goto out_unlock; |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8802 | |
| 8803 | if (curr->sched_class != p->sched_class) |
Peter Zijlstra | 7b270f6 | 2013-01-22 13:09:13 +0530 | [diff] [blame] | 8804 | goto out_unlock; |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8805 | |
Peter Zijlstra | 0b9d46f | 2022-09-06 12:33:04 +0200 | [diff] [blame] | 8806 | if (task_on_cpu(p_rq, p) || !task_is_running(p)) |
Peter Zijlstra | 7b270f6 | 2013-01-22 13:09:13 +0530 | [diff] [blame] | 8807 | goto out_unlock; |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8808 | |
Dietmar Eggemann | 0900acf | 2020-06-03 10:03:02 +0200 | [diff] [blame] | 8809 | yielded = curr->sched_class->yield_to_task(rq, p); |
Venkatesh Pallipadi | 6d1cafd | 2011-03-01 16:28:21 -0800 | [diff] [blame] | 8810 | if (yielded) { |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8811 | schedstat_inc(rq->yld_count); |
Venkatesh Pallipadi | 6d1cafd | 2011-03-01 16:28:21 -0800 | [diff] [blame] | 8812 | /* |
| 8813 | * Make p's CPU reschedule; pick_next_entity takes care of |
| 8814 | * fairness. |
| 8815 | */ |
| 8816 | if (preempt && rq != p_rq) |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 8817 | resched_curr(p_rq); |
Venkatesh Pallipadi | 6d1cafd | 2011-03-01 16:28:21 -0800 | [diff] [blame] | 8818 | } |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8819 | |
Peter Zijlstra | 7b270f6 | 2013-01-22 13:09:13 +0530 | [diff] [blame] | 8820 | out_unlock: |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8821 | double_rq_unlock(rq, p_rq); |
Peter Zijlstra | 7b270f6 | 2013-01-22 13:09:13 +0530 | [diff] [blame] | 8822 | out_irq: |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8823 | local_irq_restore(flags); |
| 8824 | |
Peter Zijlstra | 7b270f6 | 2013-01-22 13:09:13 +0530 | [diff] [blame] | 8825 | if (yielded > 0) |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 8826 | schedule(); |
| 8827 | |
| 8828 | return yielded; |
| 8829 | } |
| 8830 | EXPORT_SYMBOL_GPL(yield_to); |
| 8831 | |
Tejun Heo | 10ab564 | 2016-10-28 12:58:10 -0400 | [diff] [blame] | 8832 | int io_schedule_prepare(void) |
| 8833 | { |
| 8834 | int old_iowait = current->in_iowait; |
| 8835 | |
| 8836 | current->in_iowait = 1; |
Christoph Hellwig | aa8dccc | 2022-01-27 08:05:49 +0100 | [diff] [blame] | 8837 | blk_flush_plug(current->plug, true); |
Tejun Heo | 10ab564 | 2016-10-28 12:58:10 -0400 | [diff] [blame] | 8838 | return old_iowait; |
| 8839 | } |
| 8840 | |
| 8841 | void io_schedule_finish(int token) |
| 8842 | { |
| 8843 | current->in_iowait = token; |
| 8844 | } |
| 8845 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8846 | /* |
Ingo Molnar | 41a2d6c | 2007-12-05 15:46:09 +0100 | [diff] [blame] | 8847 | * This task is about to go to sleep on IO. Increment rq->nr_iowait so |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8848 | * that process accounting knows that this is a task in IO wait state. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8849 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8850 | long __sched io_schedule_timeout(long timeout) |
| 8851 | { |
Tejun Heo | 10ab564 | 2016-10-28 12:58:10 -0400 | [diff] [blame] | 8852 | int token; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8853 | long ret; |
| 8854 | |
Tejun Heo | 10ab564 | 2016-10-28 12:58:10 -0400 | [diff] [blame] | 8855 | token = io_schedule_prepare(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8856 | ret = schedule_timeout(timeout); |
Tejun Heo | 10ab564 | 2016-10-28 12:58:10 -0400 | [diff] [blame] | 8857 | io_schedule_finish(token); |
NeilBrown | 9cff8ad | 2015-02-13 15:49:17 +1100 | [diff] [blame] | 8858 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8859 | return ret; |
| 8860 | } |
NeilBrown | 9cff8ad | 2015-02-13 15:49:17 +1100 | [diff] [blame] | 8861 | EXPORT_SYMBOL(io_schedule_timeout); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8862 | |
Gao Xiang | e3b929b | 2019-06-03 17:13:38 +0800 | [diff] [blame] | 8863 | void __sched io_schedule(void) |
Tejun Heo | 10ab564 | 2016-10-28 12:58:10 -0400 | [diff] [blame] | 8864 | { |
| 8865 | int token; |
| 8866 | |
| 8867 | token = io_schedule_prepare(); |
| 8868 | schedule(); |
| 8869 | io_schedule_finish(token); |
| 8870 | } |
| 8871 | EXPORT_SYMBOL(io_schedule); |
| 8872 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8873 | /** |
| 8874 | * sys_sched_get_priority_max - return maximum RT priority. |
| 8875 | * @policy: scheduling class. |
| 8876 | * |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 8877 | * Return: On success, this syscall returns the maximum |
| 8878 | * rt_priority that can be used by a given scheduling class. |
| 8879 | * On failure, a negative error code is returned. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8880 | */ |
Heiko Carstens | 5add95d | 2009-01-14 14:14:08 +0100 | [diff] [blame] | 8881 | SYSCALL_DEFINE1(sched_get_priority_max, int, policy) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8882 | { |
| 8883 | int ret = -EINVAL; |
| 8884 | |
| 8885 | switch (policy) { |
| 8886 | case SCHED_FIFO: |
| 8887 | case SCHED_RR: |
Dietmar Eggemann | ae18ad2 | 2021-01-28 14:10:38 +0100 | [diff] [blame] | 8888 | ret = MAX_RT_PRIO-1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8889 | break; |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 8890 | case SCHED_DEADLINE: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8891 | case SCHED_NORMAL: |
Ingo Molnar | b0a9499 | 2006-01-14 13:20:41 -0800 | [diff] [blame] | 8892 | case SCHED_BATCH: |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 8893 | case SCHED_IDLE: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8894 | ret = 0; |
| 8895 | break; |
| 8896 | } |
| 8897 | return ret; |
| 8898 | } |
| 8899 | |
| 8900 | /** |
| 8901 | * sys_sched_get_priority_min - return minimum RT priority. |
| 8902 | * @policy: scheduling class. |
| 8903 | * |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 8904 | * Return: On success, this syscall returns the minimum |
| 8905 | * rt_priority that can be used by a given scheduling class. |
| 8906 | * On failure, a negative error code is returned. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8907 | */ |
Heiko Carstens | 5add95d | 2009-01-14 14:14:08 +0100 | [diff] [blame] | 8908 | SYSCALL_DEFINE1(sched_get_priority_min, int, policy) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8909 | { |
| 8910 | int ret = -EINVAL; |
| 8911 | |
| 8912 | switch (policy) { |
| 8913 | case SCHED_FIFO: |
| 8914 | case SCHED_RR: |
| 8915 | ret = 1; |
| 8916 | break; |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 8917 | case SCHED_DEADLINE: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8918 | case SCHED_NORMAL: |
Ingo Molnar | b0a9499 | 2006-01-14 13:20:41 -0800 | [diff] [blame] | 8919 | case SCHED_BATCH: |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 8920 | case SCHED_IDLE: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8921 | ret = 0; |
| 8922 | } |
| 8923 | return ret; |
| 8924 | } |
| 8925 | |
Al Viro | abca5fc | 2017-09-19 18:17:46 -0400 | [diff] [blame] | 8926 | static int sched_rr_get_interval(pid_t pid, struct timespec64 *t) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8927 | { |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 8928 | struct task_struct *p; |
Dmitry Adamushko | a4ec24b | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 8929 | unsigned int time_slice; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 8930 | struct rq_flags rf; |
Thomas Gleixner | dba091b | 2009-12-09 09:32:03 +0100 | [diff] [blame] | 8931 | struct rq *rq; |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 8932 | int retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8933 | |
| 8934 | if (pid < 0) |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 8935 | return -EINVAL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8936 | |
| 8937 | retval = -ESRCH; |
Thomas Gleixner | 1a551ae | 2009-12-09 10:15:11 +0000 | [diff] [blame] | 8938 | rcu_read_lock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8939 | p = find_process_by_pid(pid); |
| 8940 | if (!p) |
| 8941 | goto out_unlock; |
| 8942 | |
| 8943 | retval = security_task_getscheduler(p); |
| 8944 | if (retval) |
| 8945 | goto out_unlock; |
| 8946 | |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 8947 | rq = task_rq_lock(p, &rf); |
Peter Zijlstra | a57beec | 2014-01-27 11:54:13 +0100 | [diff] [blame] | 8948 | time_slice = 0; |
| 8949 | if (p->sched_class->get_rr_interval) |
| 8950 | time_slice = p->sched_class->get_rr_interval(rq, p); |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 8951 | task_rq_unlock(rq, p, &rf); |
Dmitry Adamushko | a4ec24b | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 8952 | |
Thomas Gleixner | 1a551ae | 2009-12-09 10:15:11 +0000 | [diff] [blame] | 8953 | rcu_read_unlock(); |
Al Viro | abca5fc | 2017-09-19 18:17:46 -0400 | [diff] [blame] | 8954 | jiffies_to_timespec64(time_slice, t); |
| 8955 | return 0; |
Andi Kleen | 3a5c359 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 8956 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8957 | out_unlock: |
Thomas Gleixner | 1a551ae | 2009-12-09 10:15:11 +0000 | [diff] [blame] | 8958 | rcu_read_unlock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8959 | return retval; |
| 8960 | } |
| 8961 | |
Randy Dunlap | 2064a5a | 2017-12-03 13:19:00 -0800 | [diff] [blame] | 8962 | /** |
| 8963 | * sys_sched_rr_get_interval - return the default timeslice of a process. |
| 8964 | * @pid: pid of the process. |
| 8965 | * @interval: userspace pointer to the timeslice value. |
| 8966 | * |
| 8967 | * this syscall writes the default timeslice value of a given process |
| 8968 | * into the user-space timespec buffer. A value of '0' means infinity. |
| 8969 | * |
| 8970 | * Return: On success, 0 and the timeslice is in @interval. Otherwise, |
| 8971 | * an error code. |
| 8972 | */ |
Al Viro | abca5fc | 2017-09-19 18:17:46 -0400 | [diff] [blame] | 8973 | SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, |
Arnd Bergmann | 474b9c7 | 2018-04-17 21:59:47 +0200 | [diff] [blame] | 8974 | struct __kernel_timespec __user *, interval) |
Al Viro | abca5fc | 2017-09-19 18:17:46 -0400 | [diff] [blame] | 8975 | { |
| 8976 | struct timespec64 t; |
| 8977 | int retval = sched_rr_get_interval(pid, &t); |
| 8978 | |
| 8979 | if (retval == 0) |
| 8980 | retval = put_timespec64(&t, interval); |
| 8981 | |
| 8982 | return retval; |
| 8983 | } |
| 8984 | |
Arnd Bergmann | 474b9c7 | 2018-04-17 21:59:47 +0200 | [diff] [blame] | 8985 | #ifdef CONFIG_COMPAT_32BIT_TIME |
Arnd Bergmann | 8dabe72 | 2019-01-07 00:33:08 +0100 | [diff] [blame] | 8986 | SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid, |
| 8987 | struct old_timespec32 __user *, interval) |
Al Viro | abca5fc | 2017-09-19 18:17:46 -0400 | [diff] [blame] | 8988 | { |
| 8989 | struct timespec64 t; |
| 8990 | int retval = sched_rr_get_interval(pid, &t); |
| 8991 | |
| 8992 | if (retval == 0) |
Arnd Bergmann | 9afc5ee | 2018-07-13 12:52:28 +0200 | [diff] [blame] | 8993 | retval = put_old_timespec32(&t, interval); |
Al Viro | abca5fc | 2017-09-19 18:17:46 -0400 | [diff] [blame] | 8994 | return retval; |
| 8995 | } |
| 8996 | #endif |
| 8997 | |
Ingo Molnar | 82a1fcb | 2008-01-25 21:08:02 +0100 | [diff] [blame] | 8998 | void sched_show_task(struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8999 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9000 | unsigned long free = 0; |
Paul E. McKenney | 4e79752 | 2012-11-07 13:35:32 -0800 | [diff] [blame] | 9001 | int ppid; |
Ingo Molnar | c930b2c | 2017-02-03 12:22:54 +0100 | [diff] [blame] | 9002 | |
Tetsuo Handa | 3820050 | 2016-11-02 19:50:29 +0900 | [diff] [blame] | 9003 | if (!try_get_task_stack(p)) |
| 9004 | return; |
Xie XiuQi | 20435d8 | 2017-08-07 16:44:23 +0800 | [diff] [blame] | 9005 | |
Libing Zhou | cc172ff | 2020-08-14 11:02:36 +0800 | [diff] [blame] | 9006 | pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p)); |
Xie XiuQi | 20435d8 | 2017-08-07 16:44:23 +0800 | [diff] [blame] | 9007 | |
Peter Zijlstra | b03fbd4 | 2021-06-11 10:28:12 +0200 | [diff] [blame] | 9008 | if (task_is_running(p)) |
Libing Zhou | cc172ff | 2020-08-14 11:02:36 +0800 | [diff] [blame] | 9009 | pr_cont(" running task "); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9010 | #ifdef CONFIG_DEBUG_STACK_USAGE |
Eric Sandeen | 7c9f886 | 2008-04-22 16:38:23 -0500 | [diff] [blame] | 9011 | free = stack_not_used(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9012 | #endif |
Oleg Nesterov | a90e984 | 2014-12-10 15:45:21 -0800 | [diff] [blame] | 9013 | ppid = 0; |
Paul E. McKenney | 4e79752 | 2012-11-07 13:35:32 -0800 | [diff] [blame] | 9014 | rcu_read_lock(); |
Oleg Nesterov | a90e984 | 2014-12-10 15:45:21 -0800 | [diff] [blame] | 9015 | if (pid_alive(p)) |
| 9016 | ppid = task_pid_nr(rcu_dereference(p->real_parent)); |
Paul E. McKenney | 4e79752 | 2012-11-07 13:35:32 -0800 | [diff] [blame] | 9017 | rcu_read_unlock(); |
Zhen Lei | 0f03d68 | 2022-07-27 14:08:19 +0800 | [diff] [blame] | 9018 | pr_cont(" stack:%-5lu pid:%-5d ppid:%-6d flags:0x%08lx\n", |
Libing Zhou | cc172ff | 2020-08-14 11:02:36 +0800 | [diff] [blame] | 9019 | free, task_pid_nr(p), ppid, |
Mark Rutland | 0569b24 | 2021-11-29 13:06:45 +0000 | [diff] [blame] | 9020 | read_task_thread_flags(p)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9021 | |
Tejun Heo | 3d1cb20 | 2013-04-30 15:27:22 -0700 | [diff] [blame] | 9022 | print_worker_info(KERN_INFO, p); |
Peter Zijlstra | a8b62fd | 2020-09-21 12:58:17 +0200 | [diff] [blame] | 9023 | print_stop_info(KERN_INFO, p); |
Dmitry Safonov | 9cb8f06 | 2020-06-08 21:32:29 -0700 | [diff] [blame] | 9024 | show_stack(p, NULL, KERN_INFO); |
Tetsuo Handa | 3820050 | 2016-11-02 19:50:29 +0900 | [diff] [blame] | 9025 | put_task_stack(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9026 | } |
Paul E. McKenney | 0032f4e | 2017-08-30 10:40:17 -0700 | [diff] [blame] | 9027 | EXPORT_SYMBOL_GPL(sched_show_task); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9028 | |
Peter Zijlstra | 5d68cc9 | 2017-09-22 18:32:41 +0200 | [diff] [blame] | 9029 | static inline bool |
| 9030 | state_filter_match(unsigned long state_filter, struct task_struct *p) |
| 9031 | { |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 9032 | unsigned int state = READ_ONCE(p->__state); |
| 9033 | |
Peter Zijlstra | 5d68cc9 | 2017-09-22 18:32:41 +0200 | [diff] [blame] | 9034 | /* no filter, everything matches */ |
| 9035 | if (!state_filter) |
| 9036 | return true; |
| 9037 | |
| 9038 | /* filter, but doesn't match */ |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 9039 | if (!(state & state_filter)) |
Peter Zijlstra | 5d68cc9 | 2017-09-22 18:32:41 +0200 | [diff] [blame] | 9040 | return false; |
| 9041 | |
| 9042 | /* |
| 9043 | * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows |
| 9044 | * TASK_KILLABLE). |
| 9045 | */ |
Peter Zijlstra | 5aec788 | 2022-09-27 21:02:34 +0200 | [diff] [blame] | 9046 | if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD)) |
Peter Zijlstra | 5d68cc9 | 2017-09-22 18:32:41 +0200 | [diff] [blame] | 9047 | return false; |
| 9048 | |
| 9049 | return true; |
| 9050 | } |
| 9051 | |
| 9052 | |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 9053 | void show_state_filter(unsigned int state_filter) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9054 | { |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 9055 | struct task_struct *g, *p; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9056 | |
Thomas Gleixner | 510f5ac | 2011-07-17 20:47:54 +0200 | [diff] [blame] | 9057 | rcu_read_lock(); |
Oleg Nesterov | 5d07f42 | 2014-08-13 21:19:53 +0200 | [diff] [blame] | 9058 | for_each_process_thread(g, p) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9059 | /* |
| 9060 | * reset the NMI-timeout, listing all files on a slow |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 9061 | * console might take a lot of time: |
Andrey Ryabinin | 57675cb | 2016-06-09 15:20:05 +0300 | [diff] [blame] | 9062 | * Also, reset softlockup watchdogs on all CPUs, because |
| 9063 | * another CPU might be blocked waiting for us to process |
| 9064 | * an IPI. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9065 | */ |
| 9066 | touch_nmi_watchdog(); |
Andrey Ryabinin | 57675cb | 2016-06-09 15:20:05 +0300 | [diff] [blame] | 9067 | touch_all_softlockup_watchdogs(); |
Peter Zijlstra | 5d68cc9 | 2017-09-22 18:32:41 +0200 | [diff] [blame] | 9068 | if (state_filter_match(state_filter, p)) |
Ingo Molnar | 82a1fcb | 2008-01-25 21:08:02 +0100 | [diff] [blame] | 9069 | sched_show_task(p); |
Oleg Nesterov | 5d07f42 | 2014-08-13 21:19:53 +0200 | [diff] [blame] | 9070 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9071 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9072 | #ifdef CONFIG_SCHED_DEBUG |
Rabin Vincent | fb90a6e | 2016-04-04 15:42:02 +0200 | [diff] [blame] | 9073 | if (!state_filter) |
| 9074 | sysrq_sched_debug_show(); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9075 | #endif |
Thomas Gleixner | 510f5ac | 2011-07-17 20:47:54 +0200 | [diff] [blame] | 9076 | rcu_read_unlock(); |
Ingo Molnar | e59e2ae | 2006-12-06 20:35:59 -0800 | [diff] [blame] | 9077 | /* |
| 9078 | * Only show locks if all tasks are dumped: |
| 9079 | */ |
Shmulik Ladkani | 93335a2 | 2009-11-25 15:23:41 +0200 | [diff] [blame] | 9080 | if (!state_filter) |
Ingo Molnar | e59e2ae | 2006-12-06 20:35:59 -0800 | [diff] [blame] | 9081 | debug_show_all_locks(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9082 | } |
| 9083 | |
Ingo Molnar | f340c0d | 2005-06-28 16:40:42 +0200 | [diff] [blame] | 9084 | /** |
| 9085 | * init_idle - set up an idle thread for a given CPU |
| 9086 | * @idle: task in question |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9087 | * @cpu: CPU the idle task belongs to |
Ingo Molnar | f340c0d | 2005-06-28 16:40:42 +0200 | [diff] [blame] | 9088 | * |
| 9089 | * NOTE: this function does not set the idle thread's NEED_RESCHED |
| 9090 | * flag, to make booting more robust. |
| 9091 | */ |
Valentin Schneider | f1a0a37 | 2021-05-12 10:46:36 +0100 | [diff] [blame] | 9092 | void __init init_idle(struct task_struct *idle, int cpu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9093 | { |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 9094 | #ifdef CONFIG_SMP |
| 9095 | struct affinity_context ac = (struct affinity_context) { |
| 9096 | .new_mask = cpumask_of(cpu), |
| 9097 | .flags = 0, |
| 9098 | }; |
| 9099 | #endif |
Ingo Molnar | 70b97a7 | 2006-07-03 00:25:42 -0700 | [diff] [blame] | 9100 | struct rq *rq = cpu_rq(cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9101 | unsigned long flags; |
| 9102 | |
Peter Zijlstra | ff51ff8 | 2019-10-01 11:18:37 +0200 | [diff] [blame] | 9103 | __sched_fork(0, idle); |
| 9104 | |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 9105 | raw_spin_lock_irqsave(&idle->pi_lock, flags); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9106 | raw_spin_rq_lock(rq); |
Ingo Molnar | 5cbd54e | 2008-11-12 20:05:50 +0100 | [diff] [blame] | 9107 | |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 9108 | idle->__state = TASK_RUNNING; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9109 | idle->se.exec_start = sched_clock(); |
Valentin Schneider | 00b89fe | 2021-05-10 16:10:23 +0100 | [diff] [blame] | 9110 | /* |
| 9111 | * PF_KTHREAD should already be set at this point; regardless, make it |
| 9112 | * look like a proper per-CPU kthread. |
| 9113 | */ |
| 9114 | idle->flags |= PF_IDLE | PF_KTHREAD | PF_NO_SETAFFINITY; |
| 9115 | kthread_set_per_cpu(idle, cpu); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9116 | |
Peter Zijlstra | de9b8f5 | 2015-08-13 23:09:29 +0200 | [diff] [blame] | 9117 | #ifdef CONFIG_SMP |
| 9118 | /* |
Tal Zussman | b19a888 | 2020-11-12 19:51:56 -0500 | [diff] [blame] | 9119 | * It's possible that init_idle() gets called multiple times on a task, |
Peter Zijlstra | de9b8f5 | 2015-08-13 23:09:29 +0200 | [diff] [blame] | 9120 | * in that case do_set_cpus_allowed() will not do the right thing. |
| 9121 | * |
| 9122 | * And since this is boot we can forgo the serialization. |
| 9123 | */ |
Waiman Long | 713a2e2 | 2022-09-22 14:00:40 -0400 | [diff] [blame] | 9124 | set_cpus_allowed_common(idle, &ac); |
Peter Zijlstra | de9b8f5 | 2015-08-13 23:09:29 +0200 | [diff] [blame] | 9125 | #endif |
Peter Zijlstra | 6506cf6c | 2010-09-16 17:50:31 +0200 | [diff] [blame] | 9126 | /* |
| 9127 | * We're having a chicken and egg problem, even though we are |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9128 | * holding rq->lock, the CPU isn't yet set to this CPU so the |
Peter Zijlstra | 6506cf6c | 2010-09-16 17:50:31 +0200 | [diff] [blame] | 9129 | * lockdep check in task_group() will fail. |
| 9130 | * |
| 9131 | * Similar case to sched_fork(). / Alternatively we could |
| 9132 | * use task_rq_lock() here and obtain the other rq->lock. |
| 9133 | * |
| 9134 | * Silence PROVE_RCU |
| 9135 | */ |
| 9136 | rcu_read_lock(); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9137 | __set_task_cpu(idle, cpu); |
Peter Zijlstra | 6506cf6c | 2010-09-16 17:50:31 +0200 | [diff] [blame] | 9138 | rcu_read_unlock(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9139 | |
Eric W. Biederman | 5311a98 | 2019-09-14 07:35:02 -0500 | [diff] [blame] | 9140 | rq->idle = idle; |
| 9141 | rcu_assign_pointer(rq->curr, idle); |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 9142 | idle->on_rq = TASK_ON_RQ_QUEUED; |
Peter Zijlstra | de9b8f5 | 2015-08-13 23:09:29 +0200 | [diff] [blame] | 9143 | #ifdef CONFIG_SMP |
Peter Zijlstra | 3ca7a44 | 2011-04-05 17:23:40 +0200 | [diff] [blame] | 9144 | idle->on_cpu = 1; |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 9145 | #endif |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9146 | raw_spin_rq_unlock(rq); |
Peter Zijlstra | 25834c7 | 2015-05-15 17:43:34 +0200 | [diff] [blame] | 9147 | raw_spin_unlock_irqrestore(&idle->pi_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9148 | |
| 9149 | /* Set the preempt count _outside_ the spinlocks! */ |
Peter Zijlstra | 0102874 | 2013-08-14 14:55:46 +0200 | [diff] [blame] | 9150 | init_idle_preempt_count(idle, cpu); |
Jonathan Corbet | 625f2a3 | 2011-04-22 11:19:10 -0600 | [diff] [blame] | 9151 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9152 | /* |
| 9153 | * The idle tasks have their own, simple scheduling class: |
| 9154 | */ |
| 9155 | idle->sched_class = &idle_sched_class; |
Steven Rostedt | 868baf0 | 2011-02-10 21:26:13 -0500 | [diff] [blame] | 9156 | ftrace_graph_init_idle_task(idle, cpu); |
Frederic Weisbecker | 45eacc6 | 2013-05-15 22:16:32 +0200 | [diff] [blame] | 9157 | vtime_init_idle(idle, cpu); |
Peter Zijlstra | de9b8f5 | 2015-08-13 23:09:29 +0200 | [diff] [blame] | 9158 | #ifdef CONFIG_SMP |
Carsten Emde | f1c6f1a | 2011-10-26 23:14:16 +0200 | [diff] [blame] | 9159 | sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); |
| 9160 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9161 | } |
| 9162 | |
Nicolas Pitre | e1d4eee | 2017-06-14 13:19:23 -0400 | [diff] [blame] | 9163 | #ifdef CONFIG_SMP |
| 9164 | |
Juri Lelli | f82f804 | 2014-10-07 09:52:11 +0100 | [diff] [blame] | 9165 | int cpuset_cpumask_can_shrink(const struct cpumask *cur, |
| 9166 | const struct cpumask *trial) |
| 9167 | { |
Nicolas Pitre | 06a76fe | 2017-06-21 14:22:01 -0400 | [diff] [blame] | 9168 | int ret = 1; |
Juri Lelli | f82f804 | 2014-10-07 09:52:11 +0100 | [diff] [blame] | 9169 | |
Yury Norov | 1087ad4 | 2022-02-10 14:49:06 -0800 | [diff] [blame] | 9170 | if (cpumask_empty(cur)) |
Mike Galbraith | bb2bc55 | 2015-01-28 04:53:55 +0100 | [diff] [blame] | 9171 | return ret; |
| 9172 | |
Nicolas Pitre | 06a76fe | 2017-06-21 14:22:01 -0400 | [diff] [blame] | 9173 | ret = dl_cpuset_cpumask_can_shrink(cur, trial); |
Juri Lelli | f82f804 | 2014-10-07 09:52:11 +0100 | [diff] [blame] | 9174 | |
| 9175 | return ret; |
| 9176 | } |
| 9177 | |
Juri Lelli | 7f51412 | 2014-09-19 10:22:40 +0100 | [diff] [blame] | 9178 | int task_can_attach(struct task_struct *p, |
Waiman Long | b6e8d40 | 2022-08-02 21:54:51 -0400 | [diff] [blame] | 9179 | const struct cpumask *cs_effective_cpus) |
Juri Lelli | 7f51412 | 2014-09-19 10:22:40 +0100 | [diff] [blame] | 9180 | { |
| 9181 | int ret = 0; |
| 9182 | |
| 9183 | /* |
| 9184 | * Kthreads which disallow setaffinity shouldn't be moved |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9185 | * to a new cpuset; we don't want to change their CPU |
Juri Lelli | 7f51412 | 2014-09-19 10:22:40 +0100 | [diff] [blame] | 9186 | * affinity and isolating such threads by their set of |
| 9187 | * allowed nodes is unnecessary. Thus, cpusets are not |
| 9188 | * applicable for such threads. This prevents checking for |
| 9189 | * success of set_cpus_allowed_ptr() on all attached tasks |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 9190 | * before cpus_mask may be changed. |
Juri Lelli | 7f51412 | 2014-09-19 10:22:40 +0100 | [diff] [blame] | 9191 | */ |
| 9192 | if (p->flags & PF_NO_SETAFFINITY) { |
| 9193 | ret = -EINVAL; |
| 9194 | goto out; |
| 9195 | } |
| 9196 | |
Juri Lelli | 7f51412 | 2014-09-19 10:22:40 +0100 | [diff] [blame] | 9197 | if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span, |
Waiman Long | b6e8d40 | 2022-08-02 21:54:51 -0400 | [diff] [blame] | 9198 | cs_effective_cpus)) { |
| 9199 | int cpu = cpumask_any_and(cpu_active_mask, cs_effective_cpus); |
Dietmar Eggemann | 772b6539 | 2022-03-02 19:34:30 +0100 | [diff] [blame] | 9200 | |
Waiman Long | b6e8d40 | 2022-08-02 21:54:51 -0400 | [diff] [blame] | 9201 | if (unlikely(cpu >= nr_cpu_ids)) |
| 9202 | return -EINVAL; |
Dietmar Eggemann | 772b6539 | 2022-03-02 19:34:30 +0100 | [diff] [blame] | 9203 | ret = dl_cpu_busy(cpu, p); |
| 9204 | } |
Juri Lelli | 7f51412 | 2014-09-19 10:22:40 +0100 | [diff] [blame] | 9205 | |
Juri Lelli | 7f51412 | 2014-09-19 10:22:40 +0100 | [diff] [blame] | 9206 | out: |
| 9207 | return ret; |
| 9208 | } |
| 9209 | |
Ingo Molnar | f2cb136 | 2017-02-01 13:10:18 +0100 | [diff] [blame] | 9210 | bool sched_smp_initialized __read_mostly; |
Thomas Gleixner | e26fbff | 2016-03-10 12:54:10 +0100 | [diff] [blame] | 9211 | |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 9212 | #ifdef CONFIG_NUMA_BALANCING |
| 9213 | /* Migrate current task p to target_cpu */ |
| 9214 | int migrate_task_to(struct task_struct *p, int target_cpu) |
| 9215 | { |
| 9216 | struct migration_arg arg = { p, target_cpu }; |
| 9217 | int curr_cpu = task_cpu(p); |
| 9218 | |
| 9219 | if (curr_cpu == target_cpu) |
| 9220 | return 0; |
| 9221 | |
Sebastian Andrzej Siewior | 3bd37062 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 9222 | if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 9223 | return -EINVAL; |
| 9224 | |
| 9225 | /* TODO: This is not properly updating schedstats */ |
| 9226 | |
Mel Gorman | 286549d | 2014-01-21 15:51:03 -0800 | [diff] [blame] | 9227 | trace_sched_move_numa(p, curr_cpu, target_cpu); |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 9228 | return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); |
| 9229 | } |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 9230 | |
| 9231 | /* |
| 9232 | * Requeue a task on a given node and accurately track the number of NUMA |
| 9233 | * tasks on the runqueues |
| 9234 | */ |
| 9235 | void sched_setnuma(struct task_struct *p, int nid) |
| 9236 | { |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 9237 | bool queued, running; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 9238 | struct rq_flags rf; |
| 9239 | struct rq *rq; |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 9240 | |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 9241 | rq = task_rq_lock(p, &rf); |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 9242 | queued = task_on_rq_queued(p); |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 9243 | running = task_current(rq, p); |
| 9244 | |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 9245 | if (queued) |
Peter Zijlstra | 1de6444 | 2015-09-30 17:44:13 +0200 | [diff] [blame] | 9246 | dequeue_task(rq, p, DEQUEUE_SAVE); |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 9247 | if (running) |
Kirill Tkhai | f3cd1c4 | 2014-09-12 17:41:40 +0400 | [diff] [blame] | 9248 | put_prev_task(rq, p); |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 9249 | |
| 9250 | p->numa_preferred_nid = nid; |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 9251 | |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 9252 | if (queued) |
Peter Zijlstra | 7134b3e | 2017-02-21 14:23:38 +0100 | [diff] [blame] | 9253 | enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); |
Vincent Guittot | a399d23 | 2016-09-12 09:47:52 +0200 | [diff] [blame] | 9254 | if (running) |
Peter Zijlstra | 03b7fad | 2019-05-29 20:36:41 +0000 | [diff] [blame] | 9255 | set_next_task(rq, p); |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 9256 | task_rq_unlock(rq, p, &rf); |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 9257 | } |
Peter Zijlstra | 5cc389b | 2015-06-11 14:46:50 +0200 | [diff] [blame] | 9258 | #endif /* CONFIG_NUMA_BALANCING */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9259 | |
| 9260 | #ifdef CONFIG_HOTPLUG_CPU |
Ingo Molnar | 48f24c4 | 2006-07-03 00:25:40 -0700 | [diff] [blame] | 9261 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9262 | * Ensure that the idle task is using init_mm right before its CPU goes |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9263 | * offline. |
| 9264 | */ |
| 9265 | void idle_task_exit(void) |
| 9266 | { |
| 9267 | struct mm_struct *mm = current->active_mm; |
| 9268 | |
| 9269 | BUG_ON(cpu_online(smp_processor_id())); |
Peter Zijlstra | bf2c59f | 2020-04-01 17:40:33 -0400 | [diff] [blame] | 9270 | BUG_ON(current != this_rq()->idle); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9271 | |
Martin Schwidefsky | a53efe5 | 2012-10-26 17:17:44 +0200 | [diff] [blame] | 9272 | if (mm != &init_mm) { |
Andy Lutomirski | 252d2a4 | 2017-06-09 11:49:15 -0700 | [diff] [blame] | 9273 | switch_mm(mm, &init_mm, current); |
Martin Schwidefsky | a53efe5 | 2012-10-26 17:17:44 +0200 | [diff] [blame] | 9274 | finish_arch_post_lock_switch(); |
| 9275 | } |
Peter Zijlstra | bf2c59f | 2020-04-01 17:40:33 -0400 | [diff] [blame] | 9276 | |
| 9277 | /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9278 | } |
| 9279 | |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9280 | static int __balance_push_cpu_stop(void *arg) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9281 | { |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9282 | struct task_struct *p = arg; |
| 9283 | struct rq *rq = this_rq(); |
| 9284 | struct rq_flags rf; |
| 9285 | int cpu; |
Peter Zijlstra | 48c5ccae | 2010-11-13 19:32:29 +0100 | [diff] [blame] | 9286 | |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9287 | raw_spin_lock_irq(&p->pi_lock); |
| 9288 | rq_lock(rq, &rf); |
Peter Zijlstra | 10e7071 | 2019-08-06 15:13:17 +0200 | [diff] [blame] | 9289 | |
Frederic Weisbecker | 77bd397 | 2013-04-12 01:50:58 +0200 | [diff] [blame] | 9290 | update_rq_clock(rq); |
| 9291 | |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9292 | if (task_rq(p) == rq && task_on_rq_queued(p)) { |
| 9293 | cpu = select_fallback_rq(rq->cpu, p); |
| 9294 | rq = __migrate_task(rq, &rf, p, cpu); |
Peter Zijlstra | 48c5ccae | 2010-11-13 19:32:29 +0100 | [diff] [blame] | 9295 | } |
| 9296 | |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9297 | rq_unlock(rq, &rf); |
| 9298 | raw_spin_unlock_irq(&p->pi_lock); |
| 9299 | |
| 9300 | put_task_struct(p); |
| 9301 | |
| 9302 | return 0; |
Peter Zijlstra | 48c5ccae | 2010-11-13 19:32:29 +0100 | [diff] [blame] | 9303 | } |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9304 | |
| 9305 | static DEFINE_PER_CPU(struct cpu_stop_work, push_work); |
| 9306 | |
| 9307 | /* |
| 9308 | * Ensure we only run per-cpu kthreads once the CPU goes !active. |
Peter Zijlstra | b5c4477 | 2021-01-21 16:09:32 +0100 | [diff] [blame] | 9309 | * |
| 9310 | * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only |
| 9311 | * effective when the hotplug motion is down. |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9312 | */ |
| 9313 | static void balance_push(struct rq *rq) |
| 9314 | { |
| 9315 | struct task_struct *push_task = rq->curr; |
| 9316 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9317 | lockdep_assert_rq_held(rq); |
Peter Zijlstra | b5c4477 | 2021-01-21 16:09:32 +0100 | [diff] [blame] | 9318 | |
Peter Zijlstra | ae79270 | 2020-12-10 17:14:08 +0100 | [diff] [blame] | 9319 | /* |
| 9320 | * Ensure the thing is persistent until balance_push_set(.on = false); |
| 9321 | */ |
| 9322 | rq->balance_callback = &balance_push_callback; |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9323 | |
| 9324 | /* |
Thomas Gleixner | 868ad33 | 2021-08-28 15:55:52 +0200 | [diff] [blame] | 9325 | * Only active while going offline and when invoked on the outgoing |
| 9326 | * CPU. |
Peter Zijlstra | b5c4477 | 2021-01-21 16:09:32 +0100 | [diff] [blame] | 9327 | */ |
Thomas Gleixner | 868ad33 | 2021-08-28 15:55:52 +0200 | [diff] [blame] | 9328 | if (!cpu_dying(rq->cpu) || rq != this_rq()) |
Peter Zijlstra | b5c4477 | 2021-01-21 16:09:32 +0100 | [diff] [blame] | 9329 | return; |
| 9330 | |
| 9331 | /* |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9332 | * Both the cpu-hotplug and stop task are in this case and are |
| 9333 | * required to complete the hotplug process. |
| 9334 | */ |
Valentin Schneider | 00b89fe | 2021-05-10 16:10:23 +0100 | [diff] [blame] | 9335 | if (kthread_is_per_cpu(push_task) || |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 9336 | is_migration_disabled(push_task)) { |
| 9337 | |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9338 | /* |
| 9339 | * If this is the idle task on the outgoing CPU try to wake |
| 9340 | * up the hotplug control thread which might wait for the |
| 9341 | * last task to vanish. The rcuwait_active() check is |
| 9342 | * accurate here because the waiter is pinned on this CPU |
| 9343 | * and can't obviously be running in parallel. |
Thomas Gleixner | 3015ef4 | 2020-08-26 14:08:10 +0200 | [diff] [blame] | 9344 | * |
| 9345 | * On RT kernels this also has to check whether there are |
| 9346 | * pinned and scheduled out tasks on the runqueue. They |
| 9347 | * need to leave the migrate disabled section first. |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9348 | */ |
Thomas Gleixner | 3015ef4 | 2020-08-26 14:08:10 +0200 | [diff] [blame] | 9349 | if (!rq->nr_running && !rq_has_pinned_tasks(rq) && |
| 9350 | rcuwait_active(&rq->hotplug_wait)) { |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9351 | raw_spin_rq_unlock(rq); |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9352 | rcuwait_wake_up(&rq->hotplug_wait); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9353 | raw_spin_rq_lock(rq); |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9354 | } |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9355 | return; |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9356 | } |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9357 | |
| 9358 | get_task_struct(push_task); |
| 9359 | /* |
| 9360 | * Temporarily drop rq->lock such that we can wake-up the stop task. |
| 9361 | * Both preemption and IRQs are still disabled. |
| 9362 | */ |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9363 | raw_spin_rq_unlock(rq); |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9364 | stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, |
| 9365 | this_cpu_ptr(&push_work)); |
| 9366 | /* |
| 9367 | * At this point need_resched() is true and we'll take the loop in |
| 9368 | * schedule(). The next pick is obviously going to be the stop task |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 9369 | * which kthread_is_per_cpu() and will push this task away. |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9370 | */ |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9371 | raw_spin_rq_lock(rq); |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9372 | } |
| 9373 | |
| 9374 | static void balance_push_set(int cpu, bool on) |
| 9375 | { |
| 9376 | struct rq *rq = cpu_rq(cpu); |
| 9377 | struct rq_flags rf; |
| 9378 | |
| 9379 | rq_lock_irqsave(rq, &rf); |
Peter Zijlstra | 22f667c | 2021-01-15 18:17:45 +0100 | [diff] [blame] | 9380 | if (on) { |
| 9381 | WARN_ON_ONCE(rq->balance_callback); |
Peter Zijlstra | ae79270 | 2020-12-10 17:14:08 +0100 | [diff] [blame] | 9382 | rq->balance_callback = &balance_push_callback; |
Peter Zijlstra | 22f667c | 2021-01-15 18:17:45 +0100 | [diff] [blame] | 9383 | } else if (rq->balance_callback == &balance_push_callback) { |
Peter Zijlstra | ae79270 | 2020-12-10 17:14:08 +0100 | [diff] [blame] | 9384 | rq->balance_callback = NULL; |
Peter Zijlstra | 22f667c | 2021-01-15 18:17:45 +0100 | [diff] [blame] | 9385 | } |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9386 | rq_unlock_irqrestore(rq, &rf); |
| 9387 | } |
| 9388 | |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9389 | /* |
| 9390 | * Invoked from a CPUs hotplug control thread after the CPU has been marked |
| 9391 | * inactive. All tasks which are not per CPU kernel threads are either |
| 9392 | * pushed off this CPU now via balance_push() or placed on a different CPU |
| 9393 | * during wakeup. Wait until the CPU is quiescent. |
| 9394 | */ |
| 9395 | static void balance_hotplug_wait(void) |
| 9396 | { |
| 9397 | struct rq *rq = this_rq(); |
| 9398 | |
Thomas Gleixner | 3015ef4 | 2020-08-26 14:08:10 +0200 | [diff] [blame] | 9399 | rcuwait_wait_event(&rq->hotplug_wait, |
| 9400 | rq->nr_running == 1 && !rq_has_pinned_tasks(rq), |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9401 | TASK_UNINTERRUPTIBLE); |
| 9402 | } |
| 9403 | |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9404 | #else |
| 9405 | |
| 9406 | static inline void balance_push(struct rq *rq) |
| 9407 | { |
| 9408 | } |
| 9409 | |
| 9410 | static inline void balance_push_set(int cpu, bool on) |
| 9411 | { |
| 9412 | } |
| 9413 | |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9414 | static inline void balance_hotplug_wait(void) |
| 9415 | { |
| 9416 | } |
| 9417 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9418 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 9419 | |
Ingo Molnar | f2cb136 | 2017-02-01 13:10:18 +0100 | [diff] [blame] | 9420 | void set_rq_online(struct rq *rq) |
Gregory Haskins | 1f11eb6a | 2008-06-04 15:04:05 -0400 | [diff] [blame] | 9421 | { |
| 9422 | if (!rq->online) { |
| 9423 | const struct sched_class *class; |
| 9424 | |
Rusty Russell | c6c4927 | 2008-11-25 02:35:05 +1030 | [diff] [blame] | 9425 | cpumask_set_cpu(rq->cpu, rq->rd->online); |
Gregory Haskins | 1f11eb6a | 2008-06-04 15:04:05 -0400 | [diff] [blame] | 9426 | rq->online = 1; |
| 9427 | |
| 9428 | for_each_class(class) { |
| 9429 | if (class->rq_online) |
| 9430 | class->rq_online(rq); |
| 9431 | } |
| 9432 | } |
| 9433 | } |
| 9434 | |
Ingo Molnar | f2cb136 | 2017-02-01 13:10:18 +0100 | [diff] [blame] | 9435 | void set_rq_offline(struct rq *rq) |
Gregory Haskins | 1f11eb6a | 2008-06-04 15:04:05 -0400 | [diff] [blame] | 9436 | { |
| 9437 | if (rq->online) { |
| 9438 | const struct sched_class *class; |
| 9439 | |
| 9440 | for_each_class(class) { |
| 9441 | if (class->rq_offline) |
| 9442 | class->rq_offline(rq); |
| 9443 | } |
| 9444 | |
Rusty Russell | c6c4927 | 2008-11-25 02:35:05 +1030 | [diff] [blame] | 9445 | cpumask_clear_cpu(rq->cpu, rq->rd->online); |
Gregory Haskins | 1f11eb6a | 2008-06-04 15:04:05 -0400 | [diff] [blame] | 9446 | rq->online = 0; |
| 9447 | } |
| 9448 | } |
| 9449 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9450 | /* |
| 9451 | * used to mark begin/end of suspend/resume: |
| 9452 | */ |
| 9453 | static int num_cpus_frozen; |
Srivatsa S. Bhat | d35be8b | 2012-05-24 19:46:26 +0530 | [diff] [blame] | 9454 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9455 | /* |
Tejun Heo | 3a101d0 | 2010-06-08 21:40:36 +0200 | [diff] [blame] | 9456 | * Update cpusets according to cpu_active mask. If cpusets are |
| 9457 | * disabled, cpuset_update_active_cpus() becomes a simple wrapper |
| 9458 | * around partition_sched_domains(). |
Srivatsa S. Bhat | d35be8b | 2012-05-24 19:46:26 +0530 | [diff] [blame] | 9459 | * |
| 9460 | * If we come here as part of a suspend/resume, don't touch cpusets because we |
| 9461 | * want to restore it back to its original state upon resume anyway. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9462 | */ |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9463 | static void cpuset_cpu_active(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9464 | { |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9465 | if (cpuhp_tasks_frozen) { |
Srivatsa S. Bhat | d35be8b | 2012-05-24 19:46:26 +0530 | [diff] [blame] | 9466 | /* |
| 9467 | * num_cpus_frozen tracks how many CPUs are involved in suspend |
| 9468 | * resume sequence. As long as this is not the last online |
| 9469 | * operation in the resume sequence, just build a single sched |
| 9470 | * domain, ignoring cpusets. |
| 9471 | */ |
Peter Zijlstra | 50e7663 | 2017-09-07 11:13:38 +0200 | [diff] [blame] | 9472 | partition_sched_domains(1, NULL, NULL); |
| 9473 | if (--num_cpus_frozen) |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9474 | return; |
Srivatsa S. Bhat | d35be8b | 2012-05-24 19:46:26 +0530 | [diff] [blame] | 9475 | /* |
| 9476 | * This is the last CPU online operation. So fall through and |
| 9477 | * restore the original sched domains by considering the |
| 9478 | * cpuset configurations. |
| 9479 | */ |
Peter Zijlstra | 50e7663 | 2017-09-07 11:13:38 +0200 | [diff] [blame] | 9480 | cpuset_force_rebuild(); |
Max Krasnyansky | e761b77 | 2008-07-15 04:43:49 -0700 | [diff] [blame] | 9481 | } |
Rakib Mullick | 30e03ac | 2017-04-09 07:36:14 +0600 | [diff] [blame] | 9482 | cpuset_update_active_cpus(); |
Max Krasnyansky | e761b77 | 2008-07-15 04:43:49 -0700 | [diff] [blame] | 9483 | } |
Tejun Heo | 3a101d0 | 2010-06-08 21:40:36 +0200 | [diff] [blame] | 9484 | |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9485 | static int cpuset_cpu_inactive(unsigned int cpu) |
Tejun Heo | 3a101d0 | 2010-06-08 21:40:36 +0200 | [diff] [blame] | 9486 | { |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9487 | if (!cpuhp_tasks_frozen) { |
Dietmar Eggemann | 772b6539 | 2022-03-02 19:34:30 +0100 | [diff] [blame] | 9488 | int ret = dl_cpu_busy(cpu, NULL); |
| 9489 | |
| 9490 | if (ret) |
| 9491 | return ret; |
Rakib Mullick | 30e03ac | 2017-04-09 07:36:14 +0600 | [diff] [blame] | 9492 | cpuset_update_active_cpus(); |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9493 | } else { |
Srivatsa S. Bhat | d35be8b | 2012-05-24 19:46:26 +0530 | [diff] [blame] | 9494 | num_cpus_frozen++; |
| 9495 | partition_sched_domains(1, NULL, NULL); |
Tejun Heo | 3a101d0 | 2010-06-08 21:40:36 +0200 | [diff] [blame] | 9496 | } |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9497 | return 0; |
Tejun Heo | 3a101d0 | 2010-06-08 21:40:36 +0200 | [diff] [blame] | 9498 | } |
Max Krasnyansky | e761b77 | 2008-07-15 04:43:49 -0700 | [diff] [blame] | 9499 | |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9500 | int sched_cpu_activate(unsigned int cpu) |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9501 | { |
Thomas Gleixner | 7d97669 | 2016-03-10 12:54:17 +0100 | [diff] [blame] | 9502 | struct rq *rq = cpu_rq(cpu); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9503 | struct rq_flags rf; |
Thomas Gleixner | 7d97669 | 2016-03-10 12:54:17 +0100 | [diff] [blame] | 9504 | |
Peter Zijlstra | 22f667c | 2021-01-15 18:17:45 +0100 | [diff] [blame] | 9505 | /* |
Peter Zijlstra | b5c4477 | 2021-01-21 16:09:32 +0100 | [diff] [blame] | 9506 | * Clear the balance_push callback and prepare to schedule |
| 9507 | * regular tasks. |
Peter Zijlstra | 22f667c | 2021-01-15 18:17:45 +0100 | [diff] [blame] | 9508 | */ |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9509 | balance_push_set(cpu, false); |
| 9510 | |
Peter Zijlstra | ba2591a | 2018-05-29 16:43:46 +0200 | [diff] [blame] | 9511 | #ifdef CONFIG_SCHED_SMT |
| 9512 | /* |
Peter Zijlstra (Intel) | c5511d0 | 2018-11-25 19:33:36 +0100 | [diff] [blame] | 9513 | * When going up, increment the number of cores with SMT present. |
Peter Zijlstra | ba2591a | 2018-05-29 16:43:46 +0200 | [diff] [blame] | 9514 | */ |
Peter Zijlstra (Intel) | c5511d0 | 2018-11-25 19:33:36 +0100 | [diff] [blame] | 9515 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) |
| 9516 | static_branch_inc_cpuslocked(&sched_smt_present); |
Peter Zijlstra | ba2591a | 2018-05-29 16:43:46 +0200 | [diff] [blame] | 9517 | #endif |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9518 | set_cpu_active(cpu, true); |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9519 | |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9520 | if (sched_smp_initialized) { |
Huang Ying | 0fb3978b | 2022-02-14 20:15:52 +0800 | [diff] [blame] | 9521 | sched_update_numa(cpu, true); |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9522 | sched_domains_numa_masks_set(cpu); |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9523 | cpuset_cpu_active(); |
Nick Piggin | 5c1e176 | 2006-10-03 01:14:04 -0700 | [diff] [blame] | 9524 | } |
Thomas Gleixner | 7d97669 | 2016-03-10 12:54:17 +0100 | [diff] [blame] | 9525 | |
| 9526 | /* |
| 9527 | * Put the rq online, if not already. This happens: |
| 9528 | * |
| 9529 | * 1) In the early boot process, because we build the real domains |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9530 | * after all CPUs have been brought up. |
Thomas Gleixner | 7d97669 | 2016-03-10 12:54:17 +0100 | [diff] [blame] | 9531 | * |
| 9532 | * 2) At runtime, if cpuset_cpu_active() fails to rebuild the |
| 9533 | * domains. |
| 9534 | */ |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9535 | rq_lock_irqsave(rq, &rf); |
Thomas Gleixner | 7d97669 | 2016-03-10 12:54:17 +0100 | [diff] [blame] | 9536 | if (rq->rd) { |
| 9537 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); |
| 9538 | set_rq_online(rq); |
| 9539 | } |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9540 | rq_unlock_irqrestore(rq, &rf); |
Thomas Gleixner | 7d97669 | 2016-03-10 12:54:17 +0100 | [diff] [blame] | 9541 | |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9542 | return 0; |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9543 | } |
| 9544 | |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9545 | int sched_cpu_deactivate(unsigned int cpu) |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9546 | { |
Peter Zijlstra | 120455c5 | 2020-09-25 16:42:31 +0200 | [diff] [blame] | 9547 | struct rq *rq = cpu_rq(cpu); |
| 9548 | struct rq_flags rf; |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9549 | int ret; |
| 9550 | |
Anna-Maria Behnsen | e0b257c | 2020-12-15 11:44:00 +0100 | [diff] [blame] | 9551 | /* |
| 9552 | * Remove CPU from nohz.idle_cpus_mask to prevent participating in |
| 9553 | * load balancing when not active |
| 9554 | */ |
| 9555 | nohz_balance_exit_idle(rq); |
| 9556 | |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9557 | set_cpu_active(cpu, false); |
Peter Zijlstra | 741ba80 | 2021-01-16 11:56:37 +0100 | [diff] [blame] | 9558 | |
| 9559 | /* |
| 9560 | * From this point forward, this CPU will refuse to run any task that |
| 9561 | * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively |
| 9562 | * push those tasks away until this gets cleared, see |
| 9563 | * sched_cpu_dying(). |
| 9564 | */ |
Peter Zijlstra | 975707f | 2021-01-20 15:05:41 +0100 | [diff] [blame] | 9565 | balance_push_set(cpu, true); |
| 9566 | |
Peter Zijlstra | b2454ca | 2016-03-10 12:54:14 +0100 | [diff] [blame] | 9567 | /* |
Peter Zijlstra | 975707f | 2021-01-20 15:05:41 +0100 | [diff] [blame] | 9568 | * We've cleared cpu_active_mask / set balance_push, wait for all |
| 9569 | * preempt-disabled and RCU users of this state to go away such that |
| 9570 | * all new such users will observe it. |
Peter Zijlstra | b2454ca | 2016-03-10 12:54:14 +0100 | [diff] [blame] | 9571 | * |
Peter Zijlstra | 5ba2ffba | 2021-01-12 11:28:16 +0100 | [diff] [blame] | 9572 | * Specifically, we rely on ttwu to no longer target this CPU, see |
| 9573 | * ttwu_queue_cond() and is_cpu_allowed(). |
| 9574 | * |
Peter Zijlstra | b2454ca | 2016-03-10 12:54:14 +0100 | [diff] [blame] | 9575 | * Do sync before park smpboot threads to take care the rcu boost case. |
| 9576 | */ |
Paul E. McKenney | 309ba85 | 2018-07-11 14:36:49 -0700 | [diff] [blame] | 9577 | synchronize_rcu(); |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9578 | |
Peter Zijlstra | 120455c5 | 2020-09-25 16:42:31 +0200 | [diff] [blame] | 9579 | rq_lock_irqsave(rq, &rf); |
| 9580 | if (rq->rd) { |
| 9581 | update_rq_clock(rq); |
| 9582 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); |
| 9583 | set_rq_offline(rq); |
| 9584 | } |
| 9585 | rq_unlock_irqrestore(rq, &rf); |
| 9586 | |
Peter Zijlstra (Intel) | c5511d0 | 2018-11-25 19:33:36 +0100 | [diff] [blame] | 9587 | #ifdef CONFIG_SCHED_SMT |
| 9588 | /* |
| 9589 | * When going down, decrement the number of cores with SMT present. |
| 9590 | */ |
| 9591 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) |
| 9592 | static_branch_dec_cpuslocked(&sched_smt_present); |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 9593 | |
| 9594 | sched_core_cpu_deactivate(cpu); |
Peter Zijlstra (Intel) | c5511d0 | 2018-11-25 19:33:36 +0100 | [diff] [blame] | 9595 | #endif |
| 9596 | |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9597 | if (!sched_smp_initialized) |
| 9598 | return 0; |
| 9599 | |
Huang Ying | 0fb3978b | 2022-02-14 20:15:52 +0800 | [diff] [blame] | 9600 | sched_update_numa(cpu, false); |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9601 | ret = cpuset_cpu_inactive(cpu); |
| 9602 | if (ret) { |
Peter Zijlstra | 2558aac | 2020-09-11 09:54:27 +0200 | [diff] [blame] | 9603 | balance_push_set(cpu, false); |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9604 | set_cpu_active(cpu, true); |
Huang Ying | 0fb3978b | 2022-02-14 20:15:52 +0800 | [diff] [blame] | 9605 | sched_update_numa(cpu, true); |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9606 | return ret; |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9607 | } |
Thomas Gleixner | 40190a7 | 2016-03-10 12:54:13 +0100 | [diff] [blame] | 9608 | sched_domains_numa_masks_clear(cpu); |
| 9609 | return 0; |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9610 | } |
| 9611 | |
Thomas Gleixner | 94baf7a | 2016-03-10 12:54:15 +0100 | [diff] [blame] | 9612 | static void sched_rq_cpu_starting(unsigned int cpu) |
| 9613 | { |
| 9614 | struct rq *rq = cpu_rq(cpu); |
| 9615 | |
| 9616 | rq->calc_load_update = calc_load_update; |
Thomas Gleixner | 94baf7a | 2016-03-10 12:54:15 +0100 | [diff] [blame] | 9617 | update_max_interval(); |
| 9618 | } |
| 9619 | |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9620 | int sched_cpu_starting(unsigned int cpu) |
| 9621 | { |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 9622 | sched_core_cpu_starting(cpu); |
Thomas Gleixner | 94baf7a | 2016-03-10 12:54:15 +0100 | [diff] [blame] | 9623 | sched_rq_cpu_starting(cpu); |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 9624 | sched_tick_start(cpu); |
Thomas Gleixner | 135fb3e | 2016-03-10 12:54:11 +0100 | [diff] [blame] | 9625 | return 0; |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 9626 | } |
| 9627 | |
Thomas Gleixner | f2785dd | 2016-03-10 12:54:18 +0100 | [diff] [blame] | 9628 | #ifdef CONFIG_HOTPLUG_CPU |
Thomas Gleixner | 1cf12e0 | 2020-09-16 09:27:18 +0200 | [diff] [blame] | 9629 | |
| 9630 | /* |
| 9631 | * Invoked immediately before the stopper thread is invoked to bring the |
| 9632 | * CPU down completely. At this point all per CPU kthreads except the |
| 9633 | * hotplug thread (current) and the stopper thread (inactive) have been |
| 9634 | * either parked or have been unbound from the outgoing CPU. Ensure that |
| 9635 | * any of those which might be on the way out are gone. |
| 9636 | * |
| 9637 | * If after this point a bound task is being woken on this CPU then the |
| 9638 | * responsible hotplug callback has failed to do it's job. |
| 9639 | * sched_cpu_dying() will catch it with the appropriate fireworks. |
| 9640 | */ |
| 9641 | int sched_cpu_wait_empty(unsigned int cpu) |
| 9642 | { |
| 9643 | balance_hotplug_wait(); |
| 9644 | return 0; |
| 9645 | } |
| 9646 | |
| 9647 | /* |
| 9648 | * Since this CPU is going 'away' for a while, fold any nr_active delta we |
| 9649 | * might have. Called from the CPU stopper task after ensuring that the |
| 9650 | * stopper is the last running task on the CPU, so nr_active count is |
| 9651 | * stable. We need to take the teardown thread which is calling this into |
| 9652 | * account, so we hand in adjust = 1 to the load calculation. |
| 9653 | * |
| 9654 | * Also see the comment "Global load-average calculations". |
| 9655 | */ |
| 9656 | static void calc_load_migrate(struct rq *rq) |
| 9657 | { |
| 9658 | long delta = calc_load_fold_active(rq, 1); |
| 9659 | |
| 9660 | if (delta) |
| 9661 | atomic_long_add(delta, &calc_load_tasks); |
| 9662 | } |
| 9663 | |
Valentin Schneider | 36c6e17 | 2021-01-13 18:31:41 +0000 | [diff] [blame] | 9664 | static void dump_rq_tasks(struct rq *rq, const char *loglvl) |
| 9665 | { |
| 9666 | struct task_struct *g, *p; |
| 9667 | int cpu = cpu_of(rq); |
| 9668 | |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9669 | lockdep_assert_rq_held(rq); |
Valentin Schneider | 36c6e17 | 2021-01-13 18:31:41 +0000 | [diff] [blame] | 9670 | |
| 9671 | printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); |
| 9672 | for_each_process_thread(g, p) { |
| 9673 | if (task_cpu(p) != cpu) |
| 9674 | continue; |
| 9675 | |
| 9676 | if (!task_on_rq_queued(p)) |
| 9677 | continue; |
| 9678 | |
| 9679 | printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm); |
| 9680 | } |
| 9681 | } |
| 9682 | |
Thomas Gleixner | f2785dd | 2016-03-10 12:54:18 +0100 | [diff] [blame] | 9683 | int sched_cpu_dying(unsigned int cpu) |
| 9684 | { |
| 9685 | struct rq *rq = cpu_rq(cpu); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9686 | struct rq_flags rf; |
Thomas Gleixner | f2785dd | 2016-03-10 12:54:18 +0100 | [diff] [blame] | 9687 | |
| 9688 | /* Handle pending wakeups and then migrate everything off */ |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 9689 | sched_tick_stop(cpu); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9690 | |
| 9691 | rq_lock_irqsave(rq, &rf); |
Valentin Schneider | 36c6e17 | 2021-01-13 18:31:41 +0000 | [diff] [blame] | 9692 | if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { |
| 9693 | WARN(true, "Dying CPU not properly vacated!"); |
| 9694 | dump_rq_tasks(rq, KERN_WARNING); |
| 9695 | } |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9696 | rq_unlock_irqrestore(rq, &rf); |
| 9697 | |
Thomas Gleixner | f2785dd | 2016-03-10 12:54:18 +0100 | [diff] [blame] | 9698 | calc_load_migrate(rq); |
| 9699 | update_max_interval(); |
Thomas Gleixner | e5ef27d | 2016-03-10 12:54:21 +0100 | [diff] [blame] | 9700 | hrtick_clear(rq); |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 9701 | sched_core_cpu_dying(cpu); |
Thomas Gleixner | f2785dd | 2016-03-10 12:54:18 +0100 | [diff] [blame] | 9702 | return 0; |
| 9703 | } |
| 9704 | #endif |
| 9705 | |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 9706 | void __init sched_init_smp(void) |
| 9707 | { |
Huang Ying | 0fb3978b | 2022-02-14 20:15:52 +0800 | [diff] [blame] | 9708 | sched_init_numa(NUMA_NO_NODE); |
Dhaval Giani | ec7dc8a | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9709 | |
Peter Zijlstra | 6acce3e | 2013-10-11 14:38:20 +0200 | [diff] [blame] | 9710 | /* |
| 9711 | * There's no userspace yet to cause hotplug operations; hence all the |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9712 | * CPU masks are stable and all blatant races in the below code cannot |
Valentin Schneider | b5a4e2b | 2018-12-19 18:23:16 +0000 | [diff] [blame] | 9713 | * happen. |
Peter Zijlstra | 6acce3e | 2013-10-11 14:38:20 +0200 | [diff] [blame] | 9714 | */ |
Peter Zijlstra | fa85ae2 | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 9715 | mutex_lock(&sched_domains_mutex); |
Peter Zijlstra | 8d5dc51 | 2017-04-25 15:29:40 +0200 | [diff] [blame] | 9716 | sched_init_domains(cpu_active_mask); |
Christoph Lameter | 476f353 | 2007-05-06 14:48:58 -0700 | [diff] [blame] | 9717 | mutex_unlock(&sched_domains_mutex); |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9718 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9719 | /* Move init over to a non-isolated CPU */ |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 9720 | if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0) |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9721 | BUG(); |
Peter Zijlstra | 15faafc | 2021-05-31 12:21:13 +0200 | [diff] [blame] | 9722 | current->flags &= ~PF_NO_SETAFFINITY; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9723 | sched_init_granularity(); |
Rusty Russell | 4212823 | 2008-11-25 02:35:12 +1030 | [diff] [blame] | 9724 | |
Rusty Russell | 0e3900e | 2008-11-25 02:35:13 +1030 | [diff] [blame] | 9725 | init_sched_rt_class(); |
Juri Lelli | 1baca4c | 2013-11-07 14:43:38 +0100 | [diff] [blame] | 9726 | init_sched_dl_class(); |
Peter Zijlstra | 1b568f0 | 2016-05-09 10:38:41 +0200 | [diff] [blame] | 9727 | |
Thomas Gleixner | e26fbff | 2016-03-10 12:54:10 +0100 | [diff] [blame] | 9728 | sched_smp_initialized = true; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9729 | } |
Thomas Gleixner | e26fbff | 2016-03-10 12:54:10 +0100 | [diff] [blame] | 9730 | |
| 9731 | static int __init migration_init(void) |
| 9732 | { |
Nicholas Piggin | 77a5352 | 2019-04-11 13:34:44 +1000 | [diff] [blame] | 9733 | sched_cpu_starting(smp_processor_id()); |
Thomas Gleixner | e26fbff | 2016-03-10 12:54:10 +0100 | [diff] [blame] | 9734 | return 0; |
| 9735 | } |
| 9736 | early_initcall(migration_init); |
| 9737 | |
Christoph Lameter | 476f353 | 2007-05-06 14:48:58 -0700 | [diff] [blame] | 9738 | #else |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9739 | void __init sched_init_smp(void) |
| 9740 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9741 | sched_init_granularity(); |
| 9742 | } |
Peter Williams | 2dd73a4 | 2006-06-27 02:54:34 -0700 | [diff] [blame] | 9743 | #endif /* CONFIG_SMP */ |
Heiko Carstens | b50f60c | 2006-07-30 03:03:52 -0700 | [diff] [blame] | 9744 | |
Avi Kivity | e107be3 | 2007-07-26 13:40:43 +0200 | [diff] [blame] | 9745 | int in_sched_functions(unsigned long addr) |
| 9746 | { |
| 9747 | return in_lock_functions(addr) || |
| 9748 | (addr >= (unsigned long)__sched_text_start |
Christoph Lameter | c9819f4 | 2006-12-10 02:20:25 -0800 | [diff] [blame] | 9749 | && addr < (unsigned long)__sched_text_end); |
Christoph Lameter | 476f353 | 2007-05-06 14:48:58 -0700 | [diff] [blame] | 9750 | } |
Christoph Lameter | c9819f4 | 2006-12-10 02:20:25 -0800 | [diff] [blame] | 9751 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9752 | #ifdef CONFIG_CGROUP_SCHED |
Li Zefan | 27b4b93 | 2013-03-05 16:07:52 +0800 | [diff] [blame] | 9753 | /* |
| 9754 | * Default task group. |
| 9755 | * Every task in system belongs to this group at bootup. |
| 9756 | */ |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9757 | struct task_group root_task_group; |
Mike Galbraith | 35cf4e5 | 2012-08-07 05:00:13 +0200 | [diff] [blame] | 9758 | LIST_HEAD(task_groups); |
Waiman Long | b036762 | 2015-12-02 13:41:49 -0500 | [diff] [blame] | 9759 | |
| 9760 | /* Cacheline aligned slab cache for task_group */ |
| 9761 | static struct kmem_cache *task_group_cache __read_mostly; |
Heiko Carstens | b50f60c | 2006-07-30 03:03:52 -0700 | [diff] [blame] | 9762 | #endif |
| 9763 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9764 | void __init sched_init(void) |
| 9765 | { |
Qian Cai | a1dc044 | 2019-07-19 21:23:19 -0400 | [diff] [blame] | 9766 | unsigned long ptr = 0; |
Dietmar Eggemann | 55627e3 | 2019-05-27 07:21:13 +0100 | [diff] [blame] | 9767 | int i; |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9768 | |
Steven Rostedt (VMware) | c3a340f | 2019-12-19 16:44:53 -0500 | [diff] [blame] | 9769 | /* Make sure the linker didn't screw up */ |
Peter Zijlstra | 546a3fe | 2022-05-17 13:46:54 +0200 | [diff] [blame] | 9770 | BUG_ON(&idle_sched_class != &fair_sched_class + 1 || |
| 9771 | &fair_sched_class != &rt_sched_class + 1 || |
| 9772 | &rt_sched_class != &dl_sched_class + 1); |
Steven Rostedt (VMware) | c3a340f | 2019-12-19 16:44:53 -0500 | [diff] [blame] | 9773 | #ifdef CONFIG_SMP |
Peter Zijlstra | 546a3fe | 2022-05-17 13:46:54 +0200 | [diff] [blame] | 9774 | BUG_ON(&dl_sched_class != &stop_sched_class + 1); |
Steven Rostedt (VMware) | c3a340f | 2019-12-19 16:44:53 -0500 | [diff] [blame] | 9775 | #endif |
| 9776 | |
Ingo Molnar | 5822a45 | 2017-03-05 13:09:07 +0100 | [diff] [blame] | 9777 | wait_bit_init(); |
Linus Torvalds | 9dcb8b6 | 2016-10-26 10:15:30 -0700 | [diff] [blame] | 9778 | |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9779 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Qian Cai | a1dc044 | 2019-07-19 21:23:19 -0400 | [diff] [blame] | 9780 | ptr += 2 * nr_cpu_ids * sizeof(void **); |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9781 | #endif |
| 9782 | #ifdef CONFIG_RT_GROUP_SCHED |
Qian Cai | a1dc044 | 2019-07-19 21:23:19 -0400 | [diff] [blame] | 9783 | ptr += 2 * nr_cpu_ids * sizeof(void **); |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9784 | #endif |
Qian Cai | a1dc044 | 2019-07-19 21:23:19 -0400 | [diff] [blame] | 9785 | if (ptr) { |
| 9786 | ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9787 | |
| 9788 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9789 | root_task_group.se = (struct sched_entity **)ptr; |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9790 | ptr += nr_cpu_ids * sizeof(void **); |
| 9791 | |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9792 | root_task_group.cfs_rq = (struct cfs_rq **)ptr; |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9793 | ptr += nr_cpu_ids * sizeof(void **); |
Peter Zijlstra | eff766a | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 9794 | |
Wei Yang | b1d1779 | 2020-04-23 21:44:43 +0000 | [diff] [blame] | 9795 | root_task_group.shares = ROOT_TASK_GROUP_LOAD; |
| 9796 | init_cfs_bandwidth(&root_task_group.cfs_bandwidth); |
Dhaval Giani | 6d6bc0a | 2008-05-30 14:23:45 +0200 | [diff] [blame] | 9797 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9798 | #ifdef CONFIG_RT_GROUP_SCHED |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9799 | root_task_group.rt_se = (struct sched_rt_entity **)ptr; |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9800 | ptr += nr_cpu_ids * sizeof(void **); |
| 9801 | |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9802 | root_task_group.rt_rq = (struct rt_rq **)ptr; |
Peter Zijlstra | eff766a | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 9803 | ptr += nr_cpu_ids * sizeof(void **); |
| 9804 | |
Dhaval Giani | 6d6bc0a | 2008-05-30 14:23:45 +0200 | [diff] [blame] | 9805 | #endif /* CONFIG_RT_GROUP_SCHED */ |
Mike Travis | 434d53b | 2008-04-04 18:11:04 -0700 | [diff] [blame] | 9806 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9807 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9808 | init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime()); |
Dario Faggioli | 332ac17 | 2013-11-07 14:43:45 +0100 | [diff] [blame] | 9809 | |
Gregory Haskins | 57d885f | 2008-01-25 21:08:18 +0100 | [diff] [blame] | 9810 | #ifdef CONFIG_SMP |
| 9811 | init_defrootdomain(); |
| 9812 | #endif |
| 9813 | |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 9814 | #ifdef CONFIG_RT_GROUP_SCHED |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9815 | init_rt_bandwidth(&root_task_group.rt_bandwidth, |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 9816 | global_rt_period(), global_rt_runtime()); |
Dhaval Giani | 6d6bc0a | 2008-05-30 14:23:45 +0200 | [diff] [blame] | 9817 | #endif /* CONFIG_RT_GROUP_SCHED */ |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 9818 | |
Dhaval Giani | 7c94143 | 2010-01-20 13:26:18 +0100 | [diff] [blame] | 9819 | #ifdef CONFIG_CGROUP_SCHED |
Waiman Long | b036762 | 2015-12-02 13:41:49 -0500 | [diff] [blame] | 9820 | task_group_cache = KMEM_CACHE(task_group, 0); |
| 9821 | |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9822 | list_add(&root_task_group.list, &task_groups); |
| 9823 | INIT_LIST_HEAD(&root_task_group.children); |
Glauber Costa | f4d6f6c | 2011-11-01 19:19:07 -0200 | [diff] [blame] | 9824 | INIT_LIST_HEAD(&root_task_group.siblings); |
Mike Galbraith | 5091faa | 2010-11-30 14:18:03 +0100 | [diff] [blame] | 9825 | autogroup_init(&init_task); |
Dhaval Giani | 7c94143 | 2010-01-20 13:26:18 +0100 | [diff] [blame] | 9826 | #endif /* CONFIG_CGROUP_SCHED */ |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 9827 | |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9828 | for_each_possible_cpu(i) { |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9829 | struct rq *rq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9830 | |
| 9831 | rq = cpu_rq(i); |
Peter Zijlstra | 5cb9eaa | 2020-11-17 18:19:31 -0500 | [diff] [blame] | 9832 | raw_spin_lock_init(&rq->__lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9833 | rq->nr_running = 0; |
Thomas Gleixner | dce48a8 | 2009-04-11 10:43:41 +0200 | [diff] [blame] | 9834 | rq->calc_load_active = 0; |
| 9835 | rq->calc_load_update = jiffies + LOAD_FREQ; |
Jan H. Schönherr | acb5a9b | 2011-07-14 18:32:43 +0200 | [diff] [blame] | 9836 | init_cfs_rq(&rq->cfs); |
Abel Vesa | 07c54f7 | 2015-03-03 13:50:27 +0200 | [diff] [blame] | 9837 | init_rt_rq(&rq->rt); |
| 9838 | init_dl_rq(&rq->dl); |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 9839 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 9840 | INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); |
Vincent Guittot | 9c2791f | 2016-11-08 10:53:43 +0100 | [diff] [blame] | 9841 | rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; |
Dhaval Giani | 354d60c | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9842 | /* |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9843 | * How much CPU bandwidth does root_task_group get? |
Dhaval Giani | 354d60c | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9844 | * |
| 9845 | * In case of task-groups formed thr' the cgroup filesystem, it |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9846 | * gets 100% of the CPU resources in the system. This overall |
| 9847 | * system CPU resource is divided among the tasks of |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9848 | * root_task_group and its child task-groups in a fair manner, |
Dhaval Giani | 354d60c | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9849 | * based on each entity's (task or task-group's) weight |
| 9850 | * (se->load.weight). |
| 9851 | * |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9852 | * In other words, if root_task_group has 10 tasks of weight |
Dhaval Giani | 354d60c | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9853 | * 1024) and two child groups A0 and A1 (of weight 1024 each), |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9854 | * then A0's share of the CPU resource is: |
Dhaval Giani | 354d60c | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9855 | * |
Ingo Molnar | 0d905bc | 2009-05-04 19:13:30 +0200 | [diff] [blame] | 9856 | * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% |
Dhaval Giani | 354d60c | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9857 | * |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9858 | * We achieve this by letting root_task_group's tasks sit |
| 9859 | * directly in rq->cfs (i.e root_task_group->se[] = NULL). |
Dhaval Giani | 354d60c | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9860 | */ |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9861 | init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); |
Dhaval Giani | 354d60c | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 9862 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
| 9863 | |
| 9864 | rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; |
Peter Zijlstra | 052f1dc | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 9865 | #ifdef CONFIG_RT_GROUP_SCHED |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 9866 | init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 9867 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9868 | #ifdef CONFIG_SMP |
Peter Zijlstra | a4c410f | 2006-12-06 20:37:21 -0800 | [diff] [blame] | 9869 | rq->sd = NULL; |
Gregory Haskins | 57d885f | 2008-01-25 21:08:18 +0100 | [diff] [blame] | 9870 | rq->rd = NULL; |
Vincent Guittot | ca6d75e | 2015-02-27 16:54:09 +0100 | [diff] [blame] | 9871 | rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; |
Peter Zijlstra | b5c4477 | 2021-01-21 16:09:32 +0100 | [diff] [blame] | 9872 | rq->balance_callback = &balance_push_callback; |
Ingo Molnar | 3117df0 | 2006-12-13 00:34:43 -0800 | [diff] [blame] | 9873 | rq->active_balance = 0; |
| 9874 | rq->next_balance = jiffies; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9875 | rq->push_cpu = 0; |
| 9876 | rq->cpu = i; |
Gregory Haskins | 1f11eb6a | 2008-06-04 15:04:05 -0400 | [diff] [blame] | 9877 | rq->online = 0; |
Mike Galbraith | eae0c9d | 2009-11-10 03:50:02 +0100 | [diff] [blame] | 9878 | rq->idle_stamp = 0; |
| 9879 | rq->avg_idle = 2*sysctl_sched_migration_cost; |
Peter Zijlstra | 94aafc3 | 2021-06-15 12:16:11 +0100 | [diff] [blame] | 9880 | rq->wake_stamp = jiffies; |
| 9881 | rq->wake_avg_idle = rq->avg_idle; |
Jason Low | 9bd721c | 2013-09-13 11:26:52 -0700 | [diff] [blame] | 9882 | rq->max_idle_balance_cost = sysctl_sched_migration_cost; |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 9883 | |
| 9884 | INIT_LIST_HEAD(&rq->cfs_tasks); |
| 9885 | |
Gregory Haskins | dc93852 | 2008-01-25 21:08:26 +0100 | [diff] [blame] | 9886 | rq_attach_root(rq, &def_root_domain); |
Frederic Weisbecker | 3451d02 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 9887 | #ifdef CONFIG_NO_HZ_COMMON |
Peter Zijlstra | e022e0d | 2017-12-21 11:20:23 +0100 | [diff] [blame] | 9888 | rq->last_blocked_load_update_tick = jiffies; |
Peter Zijlstra | a22e47a | 2017-12-21 10:01:24 +0100 | [diff] [blame] | 9889 | atomic_set(&rq->nohz_flags, 0); |
Peter Zijlstra (Intel) | 90b5363 | 2020-03-27 11:44:56 +0100 | [diff] [blame] | 9890 | |
Peter Zijlstra | 545b8c8 | 2020-06-15 11:29:31 +0200 | [diff] [blame] | 9891 | INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 9892 | #endif |
Thomas Gleixner | f2469a1 | 2020-09-14 14:47:28 +0200 | [diff] [blame] | 9893 | #ifdef CONFIG_HOTPLUG_CPU |
| 9894 | rcuwait_init(&rq->hotplug_wait); |
Ingo Molnar | a0f98a1 | 2007-06-17 18:37:45 +0200 | [diff] [blame] | 9895 | #endif |
Frederic Weisbecker | 9fd81dd | 2016-04-19 17:36:51 +0200 | [diff] [blame] | 9896 | #endif /* CONFIG_SMP */ |
Frederic Weisbecker | 77a021b | 2018-02-21 05:17:23 +0100 | [diff] [blame] | 9897 | hrtick_rq_init(rq); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9898 | atomic_set(&rq->nr_iowait, 0); |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 9899 | |
| 9900 | #ifdef CONFIG_SCHED_CORE |
Peter Zijlstra | 3c474b3 | 2021-08-19 13:09:17 +0200 | [diff] [blame] | 9901 | rq->core = rq; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 9902 | rq->core_pick = NULL; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 9903 | rq->core_enabled = 0; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 9904 | rq->core_tree = RB_ROOT; |
Josh Don | 4feee7d1 | 2021-10-18 13:34:28 -0700 | [diff] [blame] | 9905 | rq->core_forceidle_count = 0; |
| 9906 | rq->core_forceidle_occupation = 0; |
| 9907 | rq->core_forceidle_start = 0; |
Peter Zijlstra | 539f651 | 2020-11-17 18:19:37 -0500 | [diff] [blame] | 9908 | |
| 9909 | rq->core_cookie = 0UL; |
Peter Zijlstra | 9edeaea | 2020-11-17 18:19:34 -0500 | [diff] [blame] | 9910 | #endif |
Waiman Long | da01903 | 2022-09-22 14:00:39 -0400 | [diff] [blame] | 9911 | zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9912 | } |
Ingo Molnar | a0f98a1 | 2007-06-17 18:37:45 +0200 | [diff] [blame] | 9913 | |
Peter Zijlstra | b1e8206 | 2022-02-14 10:16:57 +0100 | [diff] [blame] | 9914 | set_load_weight(&init_task, false); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 9915 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9916 | /* |
| 9917 | * The boot idle thread does lazy MMU switching as well: |
| 9918 | */ |
Vegard Nossum | f1f1007 | 2017-02-27 14:30:07 -0800 | [diff] [blame] | 9919 | mmgrab(&init_mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9920 | enter_lazy_tlb(&init_mm, current); |
| 9921 | |
| 9922 | /* |
Eric W. Biederman | 40966e3 | 2021-12-02 09:56:14 -0600 | [diff] [blame] | 9923 | * The idle task doesn't need the kthread struct to function, but it |
| 9924 | * is dressed up as a per-CPU kthread and thus needs to play the part |
| 9925 | * if we want to avoid special-casing it in code that deals with per-CPU |
| 9926 | * kthreads. |
| 9927 | */ |
Eric W. Biederman | dd621ee | 2021-12-21 11:41:14 -0600 | [diff] [blame] | 9928 | WARN_ON(!set_kthread_struct(current)); |
Eric W. Biederman | 40966e3 | 2021-12-02 09:56:14 -0600 | [diff] [blame] | 9929 | |
| 9930 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9931 | * Make us the idle thread. Technically, schedule() should not be |
| 9932 | * called from this thread, however somewhere below it might be, |
| 9933 | * but because we are the idle thread, we just pick up running again |
| 9934 | * when this runqueue becomes "idle". |
| 9935 | */ |
| 9936 | init_idle(current, smp_processor_id()); |
Thomas Gleixner | dce48a8 | 2009-04-11 10:43:41 +0200 | [diff] [blame] | 9937 | |
| 9938 | calc_load_update = jiffies + LOAD_FREQ; |
| 9939 | |
Rusty Russell | bf4d83f | 2008-11-25 09:57:51 +1030 | [diff] [blame] | 9940 | #ifdef CONFIG_SMP |
Thomas Gleixner | 29d5e04 | 2012-04-20 13:05:45 +0000 | [diff] [blame] | 9941 | idle_thread_set_boot_cpu(); |
Peter Zijlstra | b5c4477 | 2021-01-21 16:09:32 +0100 | [diff] [blame] | 9942 | balance_push_set(smp_processor_id(), false); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9943 | #endif |
| 9944 | init_sched_fair_class(); |
Rusty Russell | 6a7b3dc | 2008-11-25 02:35:04 +1030 | [diff] [blame] | 9945 | |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 9946 | psi_init(); |
| 9947 | |
Patrick Bellasi | 69842cb | 2019-06-21 09:42:02 +0100 | [diff] [blame] | 9948 | init_uclamp(); |
| 9949 | |
Frederic Weisbecker | c597bfd | 2021-09-14 12:31:34 +0200 | [diff] [blame] | 9950 | preempt_dynamic_init(); |
| 9951 | |
Ingo Molnar | 6892b75 | 2008-02-13 14:02:36 +0100 | [diff] [blame] | 9952 | scheduler_running = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9953 | } |
| 9954 | |
Frederic Weisbecker | d902db1 | 2011-06-08 19:31:56 +0200 | [diff] [blame] | 9955 | #ifdef CONFIG_DEBUG_ATOMIC_SLEEP |
Frederic Weisbecker | e4aafea | 2009-07-16 15:44:29 +0200 | [diff] [blame] | 9956 | |
Thomas Gleixner | 42a3875 | 2021-09-23 18:54:38 +0200 | [diff] [blame] | 9957 | void __might_sleep(const char *file, int line) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9958 | { |
Peter Zijlstra | d6c23bb | 2021-06-11 10:28:14 +0200 | [diff] [blame] | 9959 | unsigned int state = get_current_state(); |
Peter Zijlstra | 8eb23b9 | 2014-09-24 10:18:55 +0200 | [diff] [blame] | 9960 | /* |
| 9961 | * Blocking primitives will set (and therefore destroy) current->state, |
| 9962 | * since we will exit with TASK_RUNNING make sure we enter with it, |
| 9963 | * otherwise we will destroy state. |
| 9964 | */ |
Peter Zijlstra | d6c23bb | 2021-06-11 10:28:14 +0200 | [diff] [blame] | 9965 | WARN_ONCE(state != TASK_RUNNING && current->task_state_change, |
Peter Zijlstra | 8eb23b9 | 2014-09-24 10:18:55 +0200 | [diff] [blame] | 9966 | "do not call blocking ops when !TASK_RUNNING; " |
Peter Zijlstra | d6c23bb | 2021-06-11 10:28:14 +0200 | [diff] [blame] | 9967 | "state=%x set at [<%p>] %pS\n", state, |
Peter Zijlstra | 8eb23b9 | 2014-09-24 10:18:55 +0200 | [diff] [blame] | 9968 | (void *)current->task_state_change, |
Linus Torvalds | 00845eb | 2015-02-01 12:23:32 -0800 | [diff] [blame] | 9969 | (void *)current->task_state_change); |
Peter Zijlstra | 8eb23b9 | 2014-09-24 10:18:55 +0200 | [diff] [blame] | 9970 | |
Thomas Gleixner | 42a3875 | 2021-09-23 18:54:38 +0200 | [diff] [blame] | 9971 | __might_resched(file, line, 0); |
Peter Zijlstra | 3427445 | 2014-09-24 10:18:56 +0200 | [diff] [blame] | 9972 | } |
| 9973 | EXPORT_SYMBOL(__might_sleep); |
| 9974 | |
Thomas Gleixner | 8d713b6 | 2021-09-23 18:54:41 +0200 | [diff] [blame] | 9975 | static void print_preempt_disable_ip(int preempt_offset, unsigned long ip) |
| 9976 | { |
| 9977 | if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT)) |
| 9978 | return; |
| 9979 | |
| 9980 | if (preempt_count() == preempt_offset) |
| 9981 | return; |
| 9982 | |
| 9983 | pr_err("Preemption disabled at:"); |
| 9984 | print_ip_sym(KERN_ERR, ip); |
| 9985 | } |
| 9986 | |
Thomas Gleixner | 50e081b | 2021-09-23 18:54:43 +0200 | [diff] [blame] | 9987 | static inline bool resched_offsets_ok(unsigned int offsets) |
| 9988 | { |
| 9989 | unsigned int nested = preempt_count(); |
| 9990 | |
| 9991 | nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT; |
| 9992 | |
| 9993 | return nested == offsets; |
| 9994 | } |
| 9995 | |
| 9996 | void __might_resched(const char *file, int line, unsigned int offsets) |
Peter Zijlstra | 3427445 | 2014-09-24 10:18:56 +0200 | [diff] [blame] | 9997 | { |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 9998 | /* Ratelimiting timestamp: */ |
| 9999 | static unsigned long prev_jiffy; |
| 10000 | |
Vegard Nossum | d1c6d14 | 2016-07-23 09:46:39 +0200 | [diff] [blame] | 10001 | unsigned long preempt_disable_ip; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10002 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10003 | /* WARN_ON_ONCE() by default, no rate limit required: */ |
| 10004 | rcu_sleep_check(); |
| 10005 | |
Thomas Gleixner | 50e081b | 2021-09-23 18:54:43 +0200 | [diff] [blame] | 10006 | if ((resched_offsets_ok(offsets) && !irqs_disabled() && |
Daniel Vetter | 312364f3 | 2019-08-26 22:14:23 +0200 | [diff] [blame] | 10007 | !is_idle_task(current) && !current->non_block_count) || |
Thomas Gleixner | 1c3c5ea | 2017-05-16 20:42:48 +0200 | [diff] [blame] | 10008 | system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || |
| 10009 | oops_in_progress) |
Ingo Molnar | aef745f | 2008-08-28 11:34:43 +0200 | [diff] [blame] | 10010 | return; |
Thomas Gleixner | 1c3c5ea | 2017-05-16 20:42:48 +0200 | [diff] [blame] | 10011 | |
Ingo Molnar | aef745f | 2008-08-28 11:34:43 +0200 | [diff] [blame] | 10012 | if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) |
| 10013 | return; |
| 10014 | prev_jiffy = jiffies; |
| 10015 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10016 | /* Save this before calling printk(), since that will clobber it: */ |
Vegard Nossum | d1c6d14 | 2016-07-23 09:46:39 +0200 | [diff] [blame] | 10017 | preempt_disable_ip = get_preempt_disable_ip(current); |
| 10018 | |
Thomas Gleixner | a45ed30 | 2021-09-23 18:54:40 +0200 | [diff] [blame] | 10019 | pr_err("BUG: sleeping function called from invalid context at %s:%d\n", |
| 10020 | file, line); |
| 10021 | pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", |
| 10022 | in_atomic(), irqs_disabled(), current->non_block_count, |
| 10023 | current->pid, current->comm); |
Thomas Gleixner | 8d713b6 | 2021-09-23 18:54:41 +0200 | [diff] [blame] | 10024 | pr_err("preempt_count: %x, expected: %x\n", preempt_count(), |
Thomas Gleixner | 50e081b | 2021-09-23 18:54:43 +0200 | [diff] [blame] | 10025 | offsets & MIGHT_RESCHED_PREEMPT_MASK); |
Thomas Gleixner | 8d713b6 | 2021-09-23 18:54:41 +0200 | [diff] [blame] | 10026 | |
| 10027 | if (IS_ENABLED(CONFIG_PREEMPT_RCU)) { |
Thomas Gleixner | 50e081b | 2021-09-23 18:54:43 +0200 | [diff] [blame] | 10028 | pr_err("RCU nest depth: %d, expected: %u\n", |
| 10029 | rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT); |
Thomas Gleixner | 8d713b6 | 2021-09-23 18:54:41 +0200 | [diff] [blame] | 10030 | } |
Ingo Molnar | aef745f | 2008-08-28 11:34:43 +0200 | [diff] [blame] | 10031 | |
Eric Sandeen | a8b686b | 2014-12-16 16:25:28 -0600 | [diff] [blame] | 10032 | if (task_stack_end_corrupted(current)) |
Thomas Gleixner | a45ed30 | 2021-09-23 18:54:40 +0200 | [diff] [blame] | 10033 | pr_emerg("Thread overran stack, or stack corrupted\n"); |
Eric Sandeen | a8b686b | 2014-12-16 16:25:28 -0600 | [diff] [blame] | 10034 | |
Ingo Molnar | aef745f | 2008-08-28 11:34:43 +0200 | [diff] [blame] | 10035 | debug_show_held_locks(current); |
| 10036 | if (irqs_disabled()) |
| 10037 | print_irqtrace_events(current); |
Thomas Gleixner | 8d713b6 | 2021-09-23 18:54:41 +0200 | [diff] [blame] | 10038 | |
Thomas Gleixner | 50e081b | 2021-09-23 18:54:43 +0200 | [diff] [blame] | 10039 | print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK, |
| 10040 | preempt_disable_ip); |
Thomas Gleixner | 8d713b6 | 2021-09-23 18:54:41 +0200 | [diff] [blame] | 10041 | |
Ingo Molnar | aef745f | 2008-08-28 11:34:43 +0200 | [diff] [blame] | 10042 | dump_stack(); |
Vegard Nossum | f0b22e3 | 2016-07-22 21:46:02 +0200 | [diff] [blame] | 10043 | add_taint(TAINT_WARN, LOCKDEP_STILL_OK); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10044 | } |
Thomas Gleixner | 874f670 | 2021-09-23 18:54:35 +0200 | [diff] [blame] | 10045 | EXPORT_SYMBOL(__might_resched); |
Peter Zijlstra | 568f196 | 2019-01-28 17:21:52 -0800 | [diff] [blame] | 10046 | |
| 10047 | void __cant_sleep(const char *file, int line, int preempt_offset) |
| 10048 | { |
| 10049 | static unsigned long prev_jiffy; |
| 10050 | |
| 10051 | if (irqs_disabled()) |
| 10052 | return; |
| 10053 | |
| 10054 | if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) |
| 10055 | return; |
| 10056 | |
| 10057 | if (preempt_count() > preempt_offset) |
| 10058 | return; |
| 10059 | |
| 10060 | if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) |
| 10061 | return; |
| 10062 | prev_jiffy = jiffies; |
| 10063 | |
| 10064 | printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); |
| 10065 | printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", |
| 10066 | in_atomic(), irqs_disabled(), |
| 10067 | current->pid, current->comm); |
| 10068 | |
| 10069 | debug_show_held_locks(current); |
| 10070 | dump_stack(); |
| 10071 | add_taint(TAINT_WARN, LOCKDEP_STILL_OK); |
| 10072 | } |
| 10073 | EXPORT_SYMBOL_GPL(__cant_sleep); |
Thomas Gleixner | 74d862b6 | 2020-11-18 20:48:42 +0100 | [diff] [blame] | 10074 | |
| 10075 | #ifdef CONFIG_SMP |
| 10076 | void __cant_migrate(const char *file, int line) |
| 10077 | { |
| 10078 | static unsigned long prev_jiffy; |
| 10079 | |
| 10080 | if (irqs_disabled()) |
| 10081 | return; |
| 10082 | |
| 10083 | if (is_migration_disabled(current)) |
| 10084 | return; |
| 10085 | |
| 10086 | if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) |
| 10087 | return; |
| 10088 | |
| 10089 | if (preempt_count() > 0) |
| 10090 | return; |
| 10091 | |
| 10092 | if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) |
| 10093 | return; |
| 10094 | prev_jiffy = jiffies; |
| 10095 | |
| 10096 | pr_err("BUG: assuming non migratable context at %s:%d\n", file, line); |
| 10097 | pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n", |
| 10098 | in_atomic(), irqs_disabled(), is_migration_disabled(current), |
| 10099 | current->pid, current->comm); |
| 10100 | |
| 10101 | debug_show_held_locks(current); |
| 10102 | dump_stack(); |
| 10103 | add_taint(TAINT_WARN, LOCKDEP_STILL_OK); |
| 10104 | } |
| 10105 | EXPORT_SYMBOL_GPL(__cant_migrate); |
| 10106 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10107 | #endif |
| 10108 | |
| 10109 | #ifdef CONFIG_MAGIC_SYSRQ |
| 10110 | void normalize_rt_tasks(void) |
| 10111 | { |
| 10112 | struct task_struct *g, *p; |
Peter Zijlstra | dbc7f06 | 2015-06-11 14:46:38 +0200 | [diff] [blame] | 10113 | struct sched_attr attr = { |
| 10114 | .sched_policy = SCHED_NORMAL, |
| 10115 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10116 | |
Oleg Nesterov | 3472eaa | 2014-09-21 21:33:38 +0200 | [diff] [blame] | 10117 | read_lock(&tasklist_lock); |
Oleg Nesterov | 5d07f42 | 2014-08-13 21:19:53 +0200 | [diff] [blame] | 10118 | for_each_process_thread(g, p) { |
Ingo Molnar | 178be79 | 2007-10-15 17:00:18 +0200 | [diff] [blame] | 10119 | /* |
| 10120 | * Only normalize user tasks: |
| 10121 | */ |
Oleg Nesterov | 3472eaa | 2014-09-21 21:33:38 +0200 | [diff] [blame] | 10122 | if (p->flags & PF_KTHREAD) |
Ingo Molnar | 178be79 | 2007-10-15 17:00:18 +0200 | [diff] [blame] | 10123 | continue; |
| 10124 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 10125 | p->se.exec_start = 0; |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 10126 | schedstat_set(p->stats.wait_start, 0); |
| 10127 | schedstat_set(p->stats.sleep_start, 0); |
| 10128 | schedstat_set(p->stats.block_start, 0); |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 10129 | |
Dario Faggioli | aab03e0 | 2013-11-28 11:14:43 +0100 | [diff] [blame] | 10130 | if (!dl_task(p) && !rt_task(p)) { |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 10131 | /* |
| 10132 | * Renice negative nice level userspace |
| 10133 | * tasks back to 0: |
| 10134 | */ |
Oleg Nesterov | 3472eaa | 2014-09-21 21:33:38 +0200 | [diff] [blame] | 10135 | if (task_nice(p) < 0) |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 10136 | set_user_nice(p, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10137 | continue; |
Ingo Molnar | dd41f59 | 2007-07-09 18:51:59 +0200 | [diff] [blame] | 10138 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10139 | |
Peter Zijlstra | dbc7f06 | 2015-06-11 14:46:38 +0200 | [diff] [blame] | 10140 | __sched_setscheduler(p, &attr, false, false); |
Oleg Nesterov | 5d07f42 | 2014-08-13 21:19:53 +0200 | [diff] [blame] | 10141 | } |
Oleg Nesterov | 3472eaa | 2014-09-21 21:33:38 +0200 | [diff] [blame] | 10142 | read_unlock(&tasklist_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10143 | } |
| 10144 | |
| 10145 | #endif /* CONFIG_MAGIC_SYSRQ */ |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10146 | |
Jason Wessel | 67fc4e0 | 2010-05-20 21:04:21 -0500 | [diff] [blame] | 10147 | #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10148 | /* |
Jason Wessel | 67fc4e0 | 2010-05-20 21:04:21 -0500 | [diff] [blame] | 10149 | * These functions are only useful for the IA64 MCA handling, or kdb. |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10150 | * |
| 10151 | * They can only be called when the whole system has been |
| 10152 | * stopped - every CPU needs to be quiescent, and no scheduling |
| 10153 | * activity can take place. Using them for anything else would |
| 10154 | * be a serious bug, and as a result, they aren't even visible |
| 10155 | * under any other configuration. |
| 10156 | */ |
| 10157 | |
| 10158 | /** |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10159 | * curr_task - return the current task for a given CPU. |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10160 | * @cpu: the processor in question. |
| 10161 | * |
| 10162 | * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 10163 | * |
| 10164 | * Return: The current task for @cpu. |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10165 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 10166 | struct task_struct *curr_task(int cpu) |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10167 | { |
| 10168 | return cpu_curr(cpu); |
| 10169 | } |
| 10170 | |
Jason Wessel | 67fc4e0 | 2010-05-20 21:04:21 -0500 | [diff] [blame] | 10171 | #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ |
| 10172 | |
| 10173 | #ifdef CONFIG_IA64 |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10174 | /** |
Peter Zijlstra | 5feeb78 | 2019-05-29 20:36:38 +0000 | [diff] [blame] | 10175 | * ia64_set_curr_task - set the current task for a given CPU. |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10176 | * @cpu: the processor in question. |
| 10177 | * @p: the task pointer to set. |
| 10178 | * |
| 10179 | * Description: This function must only be used when non-maskable interrupts |
Ingo Molnar | 41a2d6c | 2007-12-05 15:46:09 +0100 | [diff] [blame] | 10180 | * are serviced on a separate stack. It allows the architecture to switch the |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10181 | * notion of the current task on a CPU in a non-blocking manner. This function |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10182 | * must be called with all CPU's synchronized, and interrupts disabled, the |
| 10183 | * and caller must save the original value of the current task (see |
| 10184 | * curr_task() above) and restore that value before reenabling interrupts and |
| 10185 | * re-starting the system. |
| 10186 | * |
| 10187 | * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! |
| 10188 | */ |
Peter Zijlstra | a458ae2 | 2016-09-20 20:29:40 +0200 | [diff] [blame] | 10189 | void ia64_set_curr_task(int cpu, struct task_struct *p) |
Linus Torvalds | 1df5c10 | 2005-09-12 07:59:21 -0700 | [diff] [blame] | 10190 | { |
| 10191 | cpu_curr(cpu) = p; |
| 10192 | } |
| 10193 | |
| 10194 | #endif |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10195 | |
Dhaval Giani | 7c94143 | 2010-01-20 13:26:18 +0100 | [diff] [blame] | 10196 | #ifdef CONFIG_CGROUP_SCHED |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10197 | /* task_group_lock serializes the addition/removal of task groups */ |
| 10198 | static DEFINE_SPINLOCK(task_group_lock); |
| 10199 | |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10200 | static inline void alloc_uclamp_sched_group(struct task_group *tg, |
| 10201 | struct task_group *parent) |
| 10202 | { |
| 10203 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 10204 | enum uclamp_id clamp_id; |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10205 | |
| 10206 | for_each_clamp_id(clamp_id) { |
| 10207 | uclamp_se_set(&tg->uclamp_req[clamp_id], |
| 10208 | uclamp_none(clamp_id), false); |
Patrick Bellasi | 0b60ba2 | 2019-08-22 14:28:07 +0100 | [diff] [blame] | 10209 | tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10210 | } |
| 10211 | #endif |
| 10212 | } |
| 10213 | |
Peter Zijlstra | 2f5177f | 2016-03-16 16:22:45 +0100 | [diff] [blame] | 10214 | static void sched_free_group(struct task_group *tg) |
Peter Zijlstra | bccbe08 | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10215 | { |
| 10216 | free_fair_sched_group(tg); |
| 10217 | free_rt_sched_group(tg); |
Mike Galbraith | e9aa1dd | 2011-01-05 11:11:25 +0100 | [diff] [blame] | 10218 | autogroup_free(tg); |
Waiman Long | b036762 | 2015-12-02 13:41:49 -0500 | [diff] [blame] | 10219 | kmem_cache_free(task_group_cache, tg); |
Peter Zijlstra | bccbe08 | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10220 | } |
| 10221 | |
Mathias Krause | b027789 | 2021-11-03 20:06:13 +0100 | [diff] [blame] | 10222 | static void sched_free_group_rcu(struct rcu_head *rcu) |
| 10223 | { |
| 10224 | sched_free_group(container_of(rcu, struct task_group, rcu)); |
| 10225 | } |
| 10226 | |
| 10227 | static void sched_unregister_group(struct task_group *tg) |
| 10228 | { |
| 10229 | unregister_fair_sched_group(tg); |
| 10230 | unregister_rt_sched_group(tg); |
| 10231 | /* |
| 10232 | * We have to wait for yet another RCU grace period to expire, as |
| 10233 | * print_cfs_stats() might run concurrently. |
| 10234 | */ |
| 10235 | call_rcu(&tg->rcu, sched_free_group_rcu); |
| 10236 | } |
| 10237 | |
Peter Zijlstra | bccbe08 | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10238 | /* allocate runqueue etc for a new task group */ |
Dhaval Giani | ec7dc8a | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 10239 | struct task_group *sched_create_group(struct task_group *parent) |
Peter Zijlstra | bccbe08 | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10240 | { |
| 10241 | struct task_group *tg; |
Peter Zijlstra | bccbe08 | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10242 | |
Waiman Long | b036762 | 2015-12-02 13:41:49 -0500 | [diff] [blame] | 10243 | tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); |
Peter Zijlstra | bccbe08 | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10244 | if (!tg) |
| 10245 | return ERR_PTR(-ENOMEM); |
| 10246 | |
Dhaval Giani | ec7dc8a | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 10247 | if (!alloc_fair_sched_group(tg, parent)) |
Peter Zijlstra | bccbe08 | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10248 | goto err; |
| 10249 | |
Dhaval Giani | ec7dc8a | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 10250 | if (!alloc_rt_sched_group(tg, parent)) |
Peter Zijlstra | bccbe08 | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10251 | goto err; |
| 10252 | |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10253 | alloc_uclamp_sched_group(tg, parent); |
| 10254 | |
Li Zefan | ace783b | 2013-01-24 14:30:48 +0800 | [diff] [blame] | 10255 | return tg; |
| 10256 | |
| 10257 | err: |
Peter Zijlstra | 2f5177f | 2016-03-16 16:22:45 +0100 | [diff] [blame] | 10258 | sched_free_group(tg); |
Li Zefan | ace783b | 2013-01-24 14:30:48 +0800 | [diff] [blame] | 10259 | return ERR_PTR(-ENOMEM); |
| 10260 | } |
| 10261 | |
| 10262 | void sched_online_group(struct task_group *tg, struct task_group *parent) |
| 10263 | { |
| 10264 | unsigned long flags; |
| 10265 | |
Peter Zijlstra | 8ed3699 | 2008-02-13 15:45:39 +0100 | [diff] [blame] | 10266 | spin_lock_irqsave(&task_group_lock, flags); |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 10267 | list_add_rcu(&tg->list, &task_groups); |
Peter Zijlstra | f473aa5 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 10268 | |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10269 | /* Root should already exist: */ |
| 10270 | WARN_ON(!parent); |
Peter Zijlstra | f473aa5 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 10271 | |
| 10272 | tg->parent = parent; |
Peter Zijlstra | f473aa5 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 10273 | INIT_LIST_HEAD(&tg->children); |
Zhang, Yanmin | 09f2724 | 2030-08-14 15:56:40 +0800 | [diff] [blame] | 10274 | list_add_rcu(&tg->siblings, &parent->children); |
Peter Zijlstra | 8ed3699 | 2008-02-13 15:45:39 +0100 | [diff] [blame] | 10275 | spin_unlock_irqrestore(&task_group_lock, flags); |
Peter Zijlstra | 8663e24 | 2016-06-22 14:58:02 +0200 | [diff] [blame] | 10276 | |
| 10277 | online_fair_sched_group(tg); |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10278 | } |
| 10279 | |
Srivatsa Vaddagiri | 9b5b775 | 2007-10-15 17:00:09 +0200 | [diff] [blame] | 10280 | /* rcu callback to free various structures associated with a task group */ |
Mathias Krause | b027789 | 2021-11-03 20:06:13 +0100 | [diff] [blame] | 10281 | static void sched_unregister_group_rcu(struct rcu_head *rhp) |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10282 | { |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10283 | /* Now it should be safe to free those cfs_rqs: */ |
Mathias Krause | b027789 | 2021-11-03 20:06:13 +0100 | [diff] [blame] | 10284 | sched_unregister_group(container_of(rhp, struct task_group, rcu)); |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10285 | } |
| 10286 | |
Ingo Molnar | 4cf86d7 | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 10287 | void sched_destroy_group(struct task_group *tg) |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10288 | { |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10289 | /* Wait for possible concurrent references to cfs_rqs complete: */ |
Mathias Krause | b027789 | 2021-11-03 20:06:13 +0100 | [diff] [blame] | 10290 | call_rcu(&tg->rcu, sched_unregister_group_rcu); |
Li Zefan | ace783b | 2013-01-24 14:30:48 +0800 | [diff] [blame] | 10291 | } |
| 10292 | |
Mathias Krause | b027789 | 2021-11-03 20:06:13 +0100 | [diff] [blame] | 10293 | void sched_release_group(struct task_group *tg) |
Li Zefan | ace783b | 2013-01-24 14:30:48 +0800 | [diff] [blame] | 10294 | { |
Peter Zijlstra | 8ed3699 | 2008-02-13 15:45:39 +0100 | [diff] [blame] | 10295 | unsigned long flags; |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10296 | |
Mathias Krause | b027789 | 2021-11-03 20:06:13 +0100 | [diff] [blame] | 10297 | /* |
| 10298 | * Unlink first, to avoid walk_tg_tree_from() from finding us (via |
| 10299 | * sched_cfs_period_timer()). |
| 10300 | * |
| 10301 | * For this to be effective, we have to wait for all pending users of |
| 10302 | * this task group to leave their RCU critical section to ensure no new |
| 10303 | * user will see our dying task group any more. Specifically ensure |
| 10304 | * that tg_unthrottle_up() won't add decayed cfs_rq's to it. |
| 10305 | * |
| 10306 | * We therefore defer calling unregister_fair_sched_group() to |
| 10307 | * sched_unregister_group() which is guarantied to get called only after the |
| 10308 | * current RCU grace period has expired. |
| 10309 | */ |
Peter Zijlstra | 3d4b47b | 2010-11-15 15:47:01 -0800 | [diff] [blame] | 10310 | spin_lock_irqsave(&task_group_lock, flags); |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 10311 | list_del_rcu(&tg->list); |
Peter Zijlstra | f473aa5 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 10312 | list_del_rcu(&tg->siblings); |
Peter Zijlstra | 8ed3699 | 2008-02-13 15:45:39 +0100 | [diff] [blame] | 10313 | spin_unlock_irqrestore(&task_group_lock, flags); |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10314 | } |
| 10315 | |
Chengming Zhou | 39c4261 | 2022-08-18 20:47:58 +0800 | [diff] [blame] | 10316 | static void sched_change_group(struct task_struct *tsk) |
Vincent Guittot | ea86cb4 | 2016-06-17 13:38:55 +0200 | [diff] [blame] | 10317 | { |
| 10318 | struct task_group *tg; |
| 10319 | |
| 10320 | /* |
| 10321 | * All callers are synchronized by task_rq_lock(); we do not use RCU |
| 10322 | * which is pointless here. Thus, we pass "true" to task_css_check() |
| 10323 | * to prevent lockdep warnings. |
| 10324 | */ |
| 10325 | tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), |
| 10326 | struct task_group, css); |
| 10327 | tg = autogroup_task_group(tsk, tg); |
| 10328 | tsk->sched_task_group = tg; |
| 10329 | |
| 10330 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 10331 | if (tsk->sched_class->task_change_group) |
Chengming Zhou | 39c4261 | 2022-08-18 20:47:58 +0800 | [diff] [blame] | 10332 | tsk->sched_class->task_change_group(tsk); |
Vincent Guittot | ea86cb4 | 2016-06-17 13:38:55 +0200 | [diff] [blame] | 10333 | else |
| 10334 | #endif |
| 10335 | set_task_rq(tsk, task_cpu(tsk)); |
| 10336 | } |
| 10337 | |
| 10338 | /* |
| 10339 | * Change task's runqueue when it moves between groups. |
| 10340 | * |
| 10341 | * The caller of this function should have put the task in its new group by |
| 10342 | * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect |
| 10343 | * its new group. |
Srivatsa Vaddagiri | 9b5b775 | 2007-10-15 17:00:09 +0200 | [diff] [blame] | 10344 | */ |
| 10345 | void sched_move_task(struct task_struct *tsk) |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10346 | { |
Peter Zijlstra | 7a57f32 | 2017-02-21 14:47:02 +0100 | [diff] [blame] | 10347 | int queued, running, queue_flags = |
| 10348 | DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 10349 | struct rq_flags rf; |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10350 | struct rq *rq; |
| 10351 | |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 10352 | rq = task_rq_lock(tsk, &rf); |
Peter Zijlstra | 1b1d622 | 2017-01-23 16:05:55 +0100 | [diff] [blame] | 10353 | update_rq_clock(rq); |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10354 | |
Dmitry Adamushko | 051a1d1 | 2007-12-18 15:21:13 +0100 | [diff] [blame] | 10355 | running = task_current(rq, tsk); |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 10356 | queued = task_on_rq_queued(tsk); |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10357 | |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 10358 | if (queued) |
Peter Zijlstra | 7a57f32 | 2017-02-21 14:47:02 +0100 | [diff] [blame] | 10359 | dequeue_task(rq, tsk, queue_flags); |
Steven Rostedt (VMware) | bb3bac2 | 2017-02-06 11:04:26 -0500 | [diff] [blame] | 10360 | if (running) |
Kirill Tkhai | f3cd1c4 | 2014-09-12 17:41:40 +0400 | [diff] [blame] | 10361 | put_prev_task(rq, tsk); |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10362 | |
Chengming Zhou | 39c4261 | 2022-08-18 20:47:58 +0800 | [diff] [blame] | 10363 | sched_change_group(tsk); |
Peter Zijlstra | 810b381 | 2008-02-29 15:21:01 -0500 | [diff] [blame] | 10364 | |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 10365 | if (queued) |
Peter Zijlstra | 7a57f32 | 2017-02-21 14:47:02 +0100 | [diff] [blame] | 10366 | enqueue_task(rq, tsk, queue_flags); |
Vincent Guittot | 2a4b03f | 2020-01-14 15:13:56 +0100 | [diff] [blame] | 10367 | if (running) { |
Peter Zijlstra | 03b7fad | 2019-05-29 20:36:41 +0000 | [diff] [blame] | 10368 | set_next_task(rq, tsk); |
Vincent Guittot | 2a4b03f | 2020-01-14 15:13:56 +0100 | [diff] [blame] | 10369 | /* |
| 10370 | * After changing group, the running task may have joined a |
| 10371 | * throttled one but it's still the running task. Trigger a |
| 10372 | * resched to make sure that task can still run. |
| 10373 | */ |
| 10374 | resched_curr(rq); |
| 10375 | } |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10376 | |
Peter Zijlstra | eb58075 | 2015-07-31 21:28:18 +0200 | [diff] [blame] | 10377 | task_rq_unlock(rq, tsk, &rf); |
Srivatsa Vaddagiri | 29f59db | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 10378 | } |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10379 | |
Tejun Heo | a7c6d55 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10380 | static inline struct task_group *css_tg(struct cgroup_subsys_state *css) |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10381 | { |
Tejun Heo | a7c6d55 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10382 | return css ? container_of(css, struct task_group, css) : NULL; |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10383 | } |
| 10384 | |
Tejun Heo | eb95419 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10385 | static struct cgroup_subsys_state * |
| 10386 | cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10387 | { |
Tejun Heo | eb95419 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10388 | struct task_group *parent = css_tg(parent_css); |
| 10389 | struct task_group *tg; |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10390 | |
Tejun Heo | eb95419 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10391 | if (!parent) { |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10392 | /* This is early initialization for the top cgroup */ |
Yong Zhang | 07e06b0 | 2011-01-07 15:17:36 +0800 | [diff] [blame] | 10393 | return &root_task_group.css; |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10394 | } |
| 10395 | |
Dhaval Giani | ec7dc8a | 2008-04-19 19:44:59 +0200 | [diff] [blame] | 10396 | tg = sched_create_group(parent); |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10397 | if (IS_ERR(tg)) |
| 10398 | return ERR_PTR(-ENOMEM); |
| 10399 | |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10400 | return &tg->css; |
| 10401 | } |
| 10402 | |
Konstantin Khlebnikov | 96b7774 | 2017-02-08 14:27:27 +0300 | [diff] [blame] | 10403 | /* Expose task group only after completing cgroup initialization */ |
| 10404 | static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) |
| 10405 | { |
| 10406 | struct task_group *tg = css_tg(css); |
| 10407 | struct task_group *parent = css_tg(css->parent); |
| 10408 | |
| 10409 | if (parent) |
| 10410 | sched_online_group(tg, parent); |
Qais Yousef | 7226017 | 2019-12-24 11:54:04 +0000 | [diff] [blame] | 10411 | |
| 10412 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
| 10413 | /* Propagate the effective uclamp value for the new group */ |
Qais Yousef | 93b7385 | 2021-05-10 15:50:32 +0100 | [diff] [blame] | 10414 | mutex_lock(&uclamp_mutex); |
| 10415 | rcu_read_lock(); |
Qais Yousef | 7226017 | 2019-12-24 11:54:04 +0000 | [diff] [blame] | 10416 | cpu_util_update_eff(css); |
Qais Yousef | 93b7385 | 2021-05-10 15:50:32 +0100 | [diff] [blame] | 10417 | rcu_read_unlock(); |
| 10418 | mutex_unlock(&uclamp_mutex); |
Qais Yousef | 7226017 | 2019-12-24 11:54:04 +0000 | [diff] [blame] | 10419 | #endif |
| 10420 | |
Konstantin Khlebnikov | 96b7774 | 2017-02-08 14:27:27 +0300 | [diff] [blame] | 10421 | return 0; |
| 10422 | } |
| 10423 | |
Peter Zijlstra | 2f5177f | 2016-03-16 16:22:45 +0100 | [diff] [blame] | 10424 | static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) |
Li Zefan | ace783b | 2013-01-24 14:30:48 +0800 | [diff] [blame] | 10425 | { |
Tejun Heo | eb95419 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10426 | struct task_group *tg = css_tg(css); |
Li Zefan | ace783b | 2013-01-24 14:30:48 +0800 | [diff] [blame] | 10427 | |
Mathias Krause | b027789 | 2021-11-03 20:06:13 +0100 | [diff] [blame] | 10428 | sched_release_group(tg); |
Li Zefan | ace783b | 2013-01-24 14:30:48 +0800 | [diff] [blame] | 10429 | } |
| 10430 | |
Tejun Heo | eb95419 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10431 | static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10432 | { |
Tejun Heo | eb95419 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10433 | struct task_group *tg = css_tg(css); |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10434 | |
Peter Zijlstra | 2f5177f | 2016-03-16 16:22:45 +0100 | [diff] [blame] | 10435 | /* |
| 10436 | * Relies on the RCU grace period between css_released() and this. |
| 10437 | */ |
Mathias Krause | b027789 | 2021-11-03 20:06:13 +0100 | [diff] [blame] | 10438 | sched_unregister_group(tg); |
Li Zefan | ace783b | 2013-01-24 14:30:48 +0800 | [diff] [blame] | 10439 | } |
| 10440 | |
Chengming Zhou | df16b71 | 2022-08-18 20:48:03 +0800 | [diff] [blame] | 10441 | #ifdef CONFIG_RT_GROUP_SCHED |
Tejun Heo | 1f7dd3e5 | 2015-12-03 10:18:21 -0500 | [diff] [blame] | 10442 | static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10443 | { |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 10444 | struct task_struct *task; |
Tejun Heo | 1f7dd3e5 | 2015-12-03 10:18:21 -0500 | [diff] [blame] | 10445 | struct cgroup_subsys_state *css; |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 10446 | |
Tejun Heo | 1f7dd3e5 | 2015-12-03 10:18:21 -0500 | [diff] [blame] | 10447 | cgroup_taskset_for_each(task, css, tset) { |
Tejun Heo | eb95419 | 2013-08-08 20:11:23 -0400 | [diff] [blame] | 10448 | if (!sched_rt_can_attach(css_tg(css), task)) |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 10449 | return -EINVAL; |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 10450 | } |
Chengming Zhou | df16b71 | 2022-08-18 20:48:03 +0800 | [diff] [blame] | 10451 | return 0; |
Ben Blum | be367d0 | 2009-09-23 15:56:31 -0700 | [diff] [blame] | 10452 | } |
Chengming Zhou | df16b71 | 2022-08-18 20:48:03 +0800 | [diff] [blame] | 10453 | #endif |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10454 | |
Tejun Heo | 1f7dd3e5 | 2015-12-03 10:18:21 -0500 | [diff] [blame] | 10455 | static void cpu_cgroup_attach(struct cgroup_taskset *tset) |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10456 | { |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 10457 | struct task_struct *task; |
Tejun Heo | 1f7dd3e5 | 2015-12-03 10:18:21 -0500 | [diff] [blame] | 10458 | struct cgroup_subsys_state *css; |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 10459 | |
Tejun Heo | 1f7dd3e5 | 2015-12-03 10:18:21 -0500 | [diff] [blame] | 10460 | cgroup_taskset_for_each(task, css, tset) |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 10461 | sched_move_task(task); |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10462 | } |
| 10463 | |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10464 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
Patrick Bellasi | 0b60ba2 | 2019-08-22 14:28:07 +0100 | [diff] [blame] | 10465 | static void cpu_util_update_eff(struct cgroup_subsys_state *css) |
| 10466 | { |
| 10467 | struct cgroup_subsys_state *top_css = css; |
| 10468 | struct uclamp_se *uc_parent = NULL; |
| 10469 | struct uclamp_se *uc_se = NULL; |
| 10470 | unsigned int eff[UCLAMP_CNT]; |
Patrick Bellasi | 0413d7f | 2019-08-22 14:28:11 +0100 | [diff] [blame] | 10471 | enum uclamp_id clamp_id; |
Patrick Bellasi | 0b60ba2 | 2019-08-22 14:28:07 +0100 | [diff] [blame] | 10472 | unsigned int clamps; |
| 10473 | |
Qais Yousef | 93b7385 | 2021-05-10 15:50:32 +0100 | [diff] [blame] | 10474 | lockdep_assert_held(&uclamp_mutex); |
| 10475 | SCHED_WARN_ON(!rcu_read_lock_held()); |
| 10476 | |
Patrick Bellasi | 0b60ba2 | 2019-08-22 14:28:07 +0100 | [diff] [blame] | 10477 | css_for_each_descendant_pre(css, top_css) { |
| 10478 | uc_parent = css_tg(css)->parent |
| 10479 | ? css_tg(css)->parent->uclamp : NULL; |
| 10480 | |
| 10481 | for_each_clamp_id(clamp_id) { |
| 10482 | /* Assume effective clamps matches requested clamps */ |
| 10483 | eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; |
| 10484 | /* Cap effective clamps with parent's effective clamps */ |
| 10485 | if (uc_parent && |
| 10486 | eff[clamp_id] > uc_parent[clamp_id].value) { |
| 10487 | eff[clamp_id] = uc_parent[clamp_id].value; |
| 10488 | } |
| 10489 | } |
| 10490 | /* Ensure protection is always capped by limit */ |
| 10491 | eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); |
| 10492 | |
| 10493 | /* Propagate most restrictive effective clamps */ |
| 10494 | clamps = 0x0; |
| 10495 | uc_se = css_tg(css)->uclamp; |
| 10496 | for_each_clamp_id(clamp_id) { |
| 10497 | if (eff[clamp_id] == uc_se[clamp_id].value) |
| 10498 | continue; |
| 10499 | uc_se[clamp_id].value = eff[clamp_id]; |
| 10500 | uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); |
| 10501 | clamps |= (0x1 << clamp_id); |
| 10502 | } |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 10503 | if (!clamps) { |
Patrick Bellasi | 0b60ba2 | 2019-08-22 14:28:07 +0100 | [diff] [blame] | 10504 | css = css_rightmost_descendant(css); |
Patrick Bellasi | babbe17 | 2019-08-22 14:28:10 +0100 | [diff] [blame] | 10505 | continue; |
| 10506 | } |
| 10507 | |
| 10508 | /* Immediately update descendants RUNNABLE tasks */ |
Qais Yousef | 0213b70 | 2021-06-17 17:51:55 +0100 | [diff] [blame] | 10509 | uclamp_update_active_tasks(css); |
Patrick Bellasi | 0b60ba2 | 2019-08-22 14:28:07 +0100 | [diff] [blame] | 10510 | } |
| 10511 | } |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10512 | |
| 10513 | /* |
| 10514 | * Integer 10^N with a given N exponent by casting to integer the literal "1eN" |
| 10515 | * C expression. Since there is no way to convert a macro argument (N) into a |
| 10516 | * character constant, use two levels of macros. |
| 10517 | */ |
| 10518 | #define _POW10(exp) ((unsigned int)1e##exp) |
| 10519 | #define POW10(exp) _POW10(exp) |
| 10520 | |
| 10521 | struct uclamp_request { |
| 10522 | #define UCLAMP_PERCENT_SHIFT 2 |
| 10523 | #define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) |
| 10524 | s64 percent; |
| 10525 | u64 util; |
| 10526 | int ret; |
| 10527 | }; |
| 10528 | |
| 10529 | static inline struct uclamp_request |
| 10530 | capacity_from_percent(char *buf) |
| 10531 | { |
| 10532 | struct uclamp_request req = { |
| 10533 | .percent = UCLAMP_PERCENT_SCALE, |
| 10534 | .util = SCHED_CAPACITY_SCALE, |
| 10535 | .ret = 0, |
| 10536 | }; |
| 10537 | |
| 10538 | buf = strim(buf); |
| 10539 | if (strcmp(buf, "max")) { |
| 10540 | req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, |
| 10541 | &req.percent); |
| 10542 | if (req.ret) |
| 10543 | return req; |
Qais Yousef | b562d14 | 2020-01-14 21:09:47 +0000 | [diff] [blame] | 10544 | if ((u64)req.percent > UCLAMP_PERCENT_SCALE) { |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10545 | req.ret = -ERANGE; |
| 10546 | return req; |
| 10547 | } |
| 10548 | |
| 10549 | req.util = req.percent << SCHED_CAPACITY_SHIFT; |
| 10550 | req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); |
| 10551 | } |
| 10552 | |
| 10553 | return req; |
| 10554 | } |
| 10555 | |
| 10556 | static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, |
| 10557 | size_t nbytes, loff_t off, |
| 10558 | enum uclamp_id clamp_id) |
| 10559 | { |
| 10560 | struct uclamp_request req; |
| 10561 | struct task_group *tg; |
| 10562 | |
| 10563 | req = capacity_from_percent(buf); |
| 10564 | if (req.ret) |
| 10565 | return req.ret; |
| 10566 | |
Qais Yousef | 46609ce | 2020-06-30 12:21:23 +0100 | [diff] [blame] | 10567 | static_branch_enable(&sched_uclamp_used); |
| 10568 | |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10569 | mutex_lock(&uclamp_mutex); |
| 10570 | rcu_read_lock(); |
| 10571 | |
| 10572 | tg = css_tg(of_css(of)); |
| 10573 | if (tg->uclamp_req[clamp_id].value != req.util) |
| 10574 | uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); |
| 10575 | |
| 10576 | /* |
| 10577 | * Because of not recoverable conversion rounding we keep track of the |
| 10578 | * exact requested value |
| 10579 | */ |
| 10580 | tg->uclamp_pct[clamp_id] = req.percent; |
| 10581 | |
Patrick Bellasi | 0b60ba2 | 2019-08-22 14:28:07 +0100 | [diff] [blame] | 10582 | /* Update effective clamps to track the most restrictive value */ |
| 10583 | cpu_util_update_eff(of_css(of)); |
| 10584 | |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 10585 | rcu_read_unlock(); |
| 10586 | mutex_unlock(&uclamp_mutex); |
| 10587 | |
| 10588 | return nbytes; |
| 10589 | } |
| 10590 | |
| 10591 | static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, |
| 10592 | char *buf, size_t nbytes, |
| 10593 | loff_t off) |
| 10594 | { |
| 10595 | return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); |
| 10596 | } |
| 10597 | |
| 10598 | static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, |
| 10599 | char *buf, size_t nbytes, |
| 10600 | loff_t off) |
| 10601 | { |
| 10602 | return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); |
| 10603 | } |
| 10604 | |
| 10605 | static inline void cpu_uclamp_print(struct seq_file *sf, |
| 10606 | enum uclamp_id clamp_id) |
| 10607 | { |
| 10608 | struct task_group *tg; |
| 10609 | u64 util_clamp; |
| 10610 | u64 percent; |
| 10611 | u32 rem; |
| 10612 | |
| 10613 | rcu_read_lock(); |
| 10614 | tg = css_tg(seq_css(sf)); |
| 10615 | util_clamp = tg->uclamp_req[clamp_id].value; |
| 10616 | rcu_read_unlock(); |
| 10617 | |
| 10618 | if (util_clamp == SCHED_CAPACITY_SCALE) { |
| 10619 | seq_puts(sf, "max\n"); |
| 10620 | return; |
| 10621 | } |
| 10622 | |
| 10623 | percent = tg->uclamp_pct[clamp_id]; |
| 10624 | percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); |
| 10625 | seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); |
| 10626 | } |
| 10627 | |
| 10628 | static int cpu_uclamp_min_show(struct seq_file *sf, void *v) |
| 10629 | { |
| 10630 | cpu_uclamp_print(sf, UCLAMP_MIN); |
| 10631 | return 0; |
| 10632 | } |
| 10633 | |
| 10634 | static int cpu_uclamp_max_show(struct seq_file *sf, void *v) |
| 10635 | { |
| 10636 | cpu_uclamp_print(sf, UCLAMP_MAX); |
| 10637 | return 0; |
| 10638 | } |
| 10639 | #endif /* CONFIG_UCLAMP_TASK_GROUP */ |
| 10640 | |
Peter Zijlstra | 052f1dc | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10641 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10642 | static int cpu_shares_write_u64(struct cgroup_subsys_state *css, |
| 10643 | struct cftype *cftype, u64 shareval) |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10644 | { |
Konstantin Khlebnikov | 5b61d50 | 2019-02-27 11:10:18 +0300 | [diff] [blame] | 10645 | if (shareval > scale_load_down(ULONG_MAX)) |
| 10646 | shareval = MAX_SHARES; |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10647 | return sched_group_set_shares(css_tg(css), scale_load(shareval)); |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10648 | } |
| 10649 | |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10650 | static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, |
| 10651 | struct cftype *cft) |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10652 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10653 | struct task_group *tg = css_tg(css); |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10654 | |
Nikhil Rao | c8b2811 | 2011-05-18 14:37:48 -0700 | [diff] [blame] | 10655 | return (u64) scale_load_down(tg->shares); |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10656 | } |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10657 | |
| 10658 | #ifdef CONFIG_CFS_BANDWIDTH |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10659 | static DEFINE_MUTEX(cfs_constraints_mutex); |
| 10660 | |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10661 | const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ |
YueHaibing | b1546ed | 2019-04-18 22:47:13 +0800 | [diff] [blame] | 10662 | static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ |
Huaixin Chang | d505b8a | 2020-04-25 18:52:48 +0800 | [diff] [blame] | 10663 | /* More than 203 days if BW_SHIFT equals 20. */ |
| 10664 | static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10665 | |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10666 | static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); |
| 10667 | |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10668 | static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, |
| 10669 | u64 burst) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10670 | { |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 10671 | int i, ret = 0, runtime_enabled, runtime_was_enabled; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10672 | struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10673 | |
| 10674 | if (tg == &root_task_group) |
| 10675 | return -EINVAL; |
| 10676 | |
| 10677 | /* |
| 10678 | * Ensure we have at some amount of bandwidth every period. This is |
| 10679 | * to prevent reaching a state of large arrears when throttled via |
| 10680 | * entity_tick() resulting in prolonged exit starvation. |
| 10681 | */ |
| 10682 | if (quota < min_cfs_quota_period || period < min_cfs_quota_period) |
| 10683 | return -EINVAL; |
| 10684 | |
| 10685 | /* |
Ingo Molnar | 3b03706 | 2021-03-18 13:38:50 +0100 | [diff] [blame] | 10686 | * Likewise, bound things on the other side by preventing insane quota |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10687 | * periods. This also allows us to normalize in computing quota |
| 10688 | * feasibility. |
| 10689 | */ |
| 10690 | if (period > max_cfs_quota_period) |
| 10691 | return -EINVAL; |
| 10692 | |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 10693 | /* |
Huaixin Chang | d505b8a | 2020-04-25 18:52:48 +0800 | [diff] [blame] | 10694 | * Bound quota to defend quota against overflow during bandwidth shift. |
| 10695 | */ |
| 10696 | if (quota != RUNTIME_INF && quota > max_cfs_runtime) |
| 10697 | return -EINVAL; |
| 10698 | |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10699 | if (quota != RUNTIME_INF && (burst > quota || |
| 10700 | burst + quota > max_cfs_runtime)) |
| 10701 | return -EINVAL; |
| 10702 | |
Huaixin Chang | d505b8a | 2020-04-25 18:52:48 +0800 | [diff] [blame] | 10703 | /* |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 10704 | * Prevent race between setting of cfs_rq->runtime_enabled and |
| 10705 | * unthrottle_offline_cfs_rqs(). |
| 10706 | */ |
Sebastian Andrzej Siewior | 746f5ea | 2021-08-03 16:16:15 +0200 | [diff] [blame] | 10707 | cpus_read_lock(); |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10708 | mutex_lock(&cfs_constraints_mutex); |
| 10709 | ret = __cfs_schedulable(tg, period, quota); |
| 10710 | if (ret) |
| 10711 | goto out_unlock; |
| 10712 | |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 10713 | runtime_enabled = quota != RUNTIME_INF; |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 10714 | runtime_was_enabled = cfs_b->quota != RUNTIME_INF; |
Ben Segall | 1ee14e6 | 2013-10-16 11:16:12 -0700 | [diff] [blame] | 10715 | /* |
| 10716 | * If we need to toggle cfs_bandwidth_used, off->on must occur |
| 10717 | * before making related changes, and on->off must occur afterwards |
| 10718 | */ |
| 10719 | if (runtime_enabled && !runtime_was_enabled) |
| 10720 | cfs_bandwidth_usage_inc(); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10721 | raw_spin_lock_irq(&cfs_b->lock); |
| 10722 | cfs_b->period = ns_to_ktime(period); |
| 10723 | cfs_b->quota = quota; |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10724 | cfs_b->burst = burst; |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 10725 | |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 10726 | __refill_cfs_bandwidth_runtime(cfs_b); |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10727 | |
| 10728 | /* Restart the period timer (if active) to handle new period expiry: */ |
Peter Zijlstra | 77a4d1a | 2015-04-15 11:41:57 +0200 | [diff] [blame] | 10729 | if (runtime_enabled) |
| 10730 | start_cfs_bandwidth(cfs_b); |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10731 | |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10732 | raw_spin_unlock_irq(&cfs_b->lock); |
| 10733 | |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 10734 | for_each_online_cpu(i) { |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10735 | struct cfs_rq *cfs_rq = tg->cfs_rq[i]; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10736 | struct rq *rq = cfs_rq->rq; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 10737 | struct rq_flags rf; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10738 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 10739 | rq_lock_irq(rq, &rf); |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 10740 | cfs_rq->runtime_enabled = runtime_enabled; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10741 | cfs_rq->runtime_remaining = 0; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 10742 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10743 | if (cfs_rq->throttled) |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 10744 | unthrottle_cfs_rq(cfs_rq); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 10745 | rq_unlock_irq(rq, &rf); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10746 | } |
Ben Segall | 1ee14e6 | 2013-10-16 11:16:12 -0700 | [diff] [blame] | 10747 | if (runtime_was_enabled && !runtime_enabled) |
| 10748 | cfs_bandwidth_usage_dec(); |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10749 | out_unlock: |
| 10750 | mutex_unlock(&cfs_constraints_mutex); |
Sebastian Andrzej Siewior | 746f5ea | 2021-08-03 16:16:15 +0200 | [diff] [blame] | 10751 | cpus_read_unlock(); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10752 | |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10753 | return ret; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10754 | } |
| 10755 | |
YueHaibing | b1546ed | 2019-04-18 22:47:13 +0800 | [diff] [blame] | 10756 | static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10757 | { |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10758 | u64 quota, period, burst; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10759 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10760 | period = ktime_to_ns(tg->cfs_bandwidth.period); |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10761 | burst = tg->cfs_bandwidth.burst; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10762 | if (cfs_quota_us < 0) |
| 10763 | quota = RUNTIME_INF; |
Konstantin Khlebnikov | 1a8b454 | 2019-02-27 11:10:20 +0300 | [diff] [blame] | 10764 | else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10765 | quota = (u64)cfs_quota_us * NSEC_PER_USEC; |
Konstantin Khlebnikov | 1a8b454 | 2019-02-27 11:10:20 +0300 | [diff] [blame] | 10766 | else |
| 10767 | return -EINVAL; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10768 | |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10769 | return tg_set_cfs_bandwidth(tg, period, quota, burst); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10770 | } |
| 10771 | |
YueHaibing | b1546ed | 2019-04-18 22:47:13 +0800 | [diff] [blame] | 10772 | static long tg_get_cfs_quota(struct task_group *tg) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10773 | { |
| 10774 | u64 quota_us; |
| 10775 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10776 | if (tg->cfs_bandwidth.quota == RUNTIME_INF) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10777 | return -1; |
| 10778 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10779 | quota_us = tg->cfs_bandwidth.quota; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10780 | do_div(quota_us, NSEC_PER_USEC); |
| 10781 | |
| 10782 | return quota_us; |
| 10783 | } |
| 10784 | |
YueHaibing | b1546ed | 2019-04-18 22:47:13 +0800 | [diff] [blame] | 10785 | static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10786 | { |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10787 | u64 quota, period, burst; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10788 | |
Konstantin Khlebnikov | 1a8b454 | 2019-02-27 11:10:20 +0300 | [diff] [blame] | 10789 | if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC) |
| 10790 | return -EINVAL; |
| 10791 | |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10792 | period = (u64)cfs_period_us * NSEC_PER_USEC; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10793 | quota = tg->cfs_bandwidth.quota; |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10794 | burst = tg->cfs_bandwidth.burst; |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10795 | |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10796 | return tg_set_cfs_bandwidth(tg, period, quota, burst); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10797 | } |
| 10798 | |
YueHaibing | b1546ed | 2019-04-18 22:47:13 +0800 | [diff] [blame] | 10799 | static long tg_get_cfs_period(struct task_group *tg) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10800 | { |
| 10801 | u64 cfs_period_us; |
| 10802 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10803 | cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10804 | do_div(cfs_period_us, NSEC_PER_USEC); |
| 10805 | |
| 10806 | return cfs_period_us; |
| 10807 | } |
| 10808 | |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10809 | static int tg_set_cfs_burst(struct task_group *tg, long cfs_burst_us) |
| 10810 | { |
| 10811 | u64 quota, period, burst; |
| 10812 | |
| 10813 | if ((u64)cfs_burst_us > U64_MAX / NSEC_PER_USEC) |
| 10814 | return -EINVAL; |
| 10815 | |
| 10816 | burst = (u64)cfs_burst_us * NSEC_PER_USEC; |
| 10817 | period = ktime_to_ns(tg->cfs_bandwidth.period); |
| 10818 | quota = tg->cfs_bandwidth.quota; |
| 10819 | |
| 10820 | return tg_set_cfs_bandwidth(tg, period, quota, burst); |
| 10821 | } |
| 10822 | |
| 10823 | static long tg_get_cfs_burst(struct task_group *tg) |
| 10824 | { |
| 10825 | u64 burst_us; |
| 10826 | |
| 10827 | burst_us = tg->cfs_bandwidth.burst; |
| 10828 | do_div(burst_us, NSEC_PER_USEC); |
| 10829 | |
| 10830 | return burst_us; |
| 10831 | } |
| 10832 | |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10833 | static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, |
| 10834 | struct cftype *cft) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10835 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10836 | return tg_get_cfs_quota(css_tg(css)); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10837 | } |
| 10838 | |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10839 | static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, |
| 10840 | struct cftype *cftype, s64 cfs_quota_us) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10841 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10842 | return tg_set_cfs_quota(css_tg(css), cfs_quota_us); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10843 | } |
| 10844 | |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10845 | static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, |
| 10846 | struct cftype *cft) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10847 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10848 | return tg_get_cfs_period(css_tg(css)); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10849 | } |
| 10850 | |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10851 | static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, |
| 10852 | struct cftype *cftype, u64 cfs_period_us) |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10853 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10854 | return tg_set_cfs_period(css_tg(css), cfs_period_us); |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10855 | } |
| 10856 | |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 10857 | static u64 cpu_cfs_burst_read_u64(struct cgroup_subsys_state *css, |
| 10858 | struct cftype *cft) |
| 10859 | { |
| 10860 | return tg_get_cfs_burst(css_tg(css)); |
| 10861 | } |
| 10862 | |
| 10863 | static int cpu_cfs_burst_write_u64(struct cgroup_subsys_state *css, |
| 10864 | struct cftype *cftype, u64 cfs_burst_us) |
| 10865 | { |
| 10866 | return tg_set_cfs_burst(css_tg(css), cfs_burst_us); |
| 10867 | } |
| 10868 | |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10869 | struct cfs_schedulable_data { |
| 10870 | struct task_group *tg; |
| 10871 | u64 period, quota; |
| 10872 | }; |
| 10873 | |
| 10874 | /* |
| 10875 | * normalize group quota/period to be quota/max_period |
| 10876 | * note: units are usecs |
| 10877 | */ |
| 10878 | static u64 normalize_cfs_quota(struct task_group *tg, |
| 10879 | struct cfs_schedulable_data *d) |
| 10880 | { |
| 10881 | u64 quota, period; |
| 10882 | |
| 10883 | if (tg == d->tg) { |
| 10884 | period = d->period; |
| 10885 | quota = d->quota; |
| 10886 | } else { |
| 10887 | period = tg_get_cfs_period(tg); |
| 10888 | quota = tg_get_cfs_quota(tg); |
| 10889 | } |
| 10890 | |
| 10891 | /* note: these should typically be equivalent */ |
| 10892 | if (quota == RUNTIME_INF || quota == -1) |
| 10893 | return RUNTIME_INF; |
| 10894 | |
| 10895 | return to_ratio(period, quota); |
| 10896 | } |
| 10897 | |
| 10898 | static int tg_cfs_schedulable_down(struct task_group *tg, void *data) |
| 10899 | { |
| 10900 | struct cfs_schedulable_data *d = data; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10901 | struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10902 | s64 quota = 0, parent_quota = -1; |
| 10903 | |
| 10904 | if (!tg->parent) { |
| 10905 | quota = RUNTIME_INF; |
| 10906 | } else { |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10907 | struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10908 | |
| 10909 | quota = normalize_cfs_quota(tg, d); |
Zhihui Zhang | 9c58c79 | 2014-09-20 21:24:36 -0400 | [diff] [blame] | 10910 | parent_quota = parent_b->hierarchical_quota; |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10911 | |
| 10912 | /* |
Tejun Heo | c53593e | 2018-01-22 11:26:18 -0800 | [diff] [blame] | 10913 | * Ensure max(child_quota) <= parent_quota. On cgroup2, |
| 10914 | * always take the min. On cgroup1, only inherit when no |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 10915 | * limit is set: |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10916 | */ |
Tejun Heo | c53593e | 2018-01-22 11:26:18 -0800 | [diff] [blame] | 10917 | if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { |
| 10918 | quota = min(quota, parent_quota); |
| 10919 | } else { |
| 10920 | if (quota == RUNTIME_INF) |
| 10921 | quota = parent_quota; |
| 10922 | else if (parent_quota != RUNTIME_INF && quota > parent_quota) |
| 10923 | return -EINVAL; |
| 10924 | } |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10925 | } |
Zhihui Zhang | 9c58c79 | 2014-09-20 21:24:36 -0400 | [diff] [blame] | 10926 | cfs_b->hierarchical_quota = quota; |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10927 | |
| 10928 | return 0; |
| 10929 | } |
| 10930 | |
| 10931 | static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) |
| 10932 | { |
Paul Turner | 8277434 | 2011-07-21 09:43:35 -0700 | [diff] [blame] | 10933 | int ret; |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10934 | struct cfs_schedulable_data data = { |
| 10935 | .tg = tg, |
| 10936 | .period = period, |
| 10937 | .quota = quota, |
| 10938 | }; |
| 10939 | |
| 10940 | if (quota != RUNTIME_INF) { |
| 10941 | do_div(data.period, NSEC_PER_USEC); |
| 10942 | do_div(data.quota, NSEC_PER_USEC); |
| 10943 | } |
| 10944 | |
Paul Turner | 8277434 | 2011-07-21 09:43:35 -0700 | [diff] [blame] | 10945 | rcu_read_lock(); |
| 10946 | ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); |
| 10947 | rcu_read_unlock(); |
| 10948 | |
| 10949 | return ret; |
Paul Turner | a790de9 | 2011-07-21 09:43:29 -0700 | [diff] [blame] | 10950 | } |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 10951 | |
Tejun Heo | a1f7164 | 2017-09-25 09:00:18 -0700 | [diff] [blame] | 10952 | static int cpu_cfs_stat_show(struct seq_file *sf, void *v) |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 10953 | { |
Tejun Heo | 2da8ca8 | 2013-12-05 12:28:04 -0500 | [diff] [blame] | 10954 | struct task_group *tg = css_tg(seq_css(sf)); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 10955 | struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 10956 | |
Tejun Heo | 44ffc75 | 2013-12-05 12:28:01 -0500 | [diff] [blame] | 10957 | seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); |
| 10958 | seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); |
| 10959 | seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 10960 | |
Yun Wang | 3d6c50c | 2018-07-04 11:27:27 +0800 | [diff] [blame] | 10961 | if (schedstat_enabled() && tg != &root_task_group) { |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 10962 | struct sched_statistics *stats; |
Yun Wang | 3d6c50c | 2018-07-04 11:27:27 +0800 | [diff] [blame] | 10963 | u64 ws = 0; |
| 10964 | int i; |
| 10965 | |
Yafang Shao | ceeadb8 | 2021-09-05 14:35:41 +0000 | [diff] [blame] | 10966 | for_each_possible_cpu(i) { |
| 10967 | stats = __schedstats_from_se(tg->se[i]); |
| 10968 | ws += schedstat_val(stats->wait_sum); |
| 10969 | } |
Yun Wang | 3d6c50c | 2018-07-04 11:27:27 +0800 | [diff] [blame] | 10970 | |
| 10971 | seq_printf(sf, "wait_sum %llu\n", ws); |
| 10972 | } |
| 10973 | |
Huaixin Chang | bcb1704 | 2021-08-30 11:22:14 +0800 | [diff] [blame] | 10974 | seq_printf(sf, "nr_bursts %d\n", cfs_b->nr_burst); |
| 10975 | seq_printf(sf, "burst_time %llu\n", cfs_b->burst_time); |
| 10976 | |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 10977 | return 0; |
| 10978 | } |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 10979 | #endif /* CONFIG_CFS_BANDWIDTH */ |
Dhaval Giani | 6d6bc0a | 2008-05-30 14:23:45 +0200 | [diff] [blame] | 10980 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 10981 | |
Peter Zijlstra | 052f1dc | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 10982 | #ifdef CONFIG_RT_GROUP_SCHED |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10983 | static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, |
| 10984 | struct cftype *cft, s64 val) |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 10985 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10986 | return sched_group_set_rt_runtime(css_tg(css), val); |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 10987 | } |
| 10988 | |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10989 | static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, |
| 10990 | struct cftype *cft) |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 10991 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10992 | return sched_group_rt_runtime(css_tg(css)); |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 10993 | } |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 10994 | |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10995 | static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, |
| 10996 | struct cftype *cftype, u64 rt_period_us) |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 10997 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 10998 | return sched_group_set_rt_period(css_tg(css), rt_period_us); |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 10999 | } |
| 11000 | |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 11001 | static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, |
| 11002 | struct cftype *cft) |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 11003 | { |
Tejun Heo | 182446d | 2013-08-08 20:11:24 -0400 | [diff] [blame] | 11004 | return sched_group_rt_period(css_tg(css)); |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 11005 | } |
Dhaval Giani | 6d6bc0a | 2008-05-30 14:23:45 +0200 | [diff] [blame] | 11006 | #endif /* CONFIG_RT_GROUP_SCHED */ |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 11007 | |
Josh Don | 3040003 | 2021-07-29 19:00:18 -0700 | [diff] [blame] | 11008 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 11009 | static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css, |
| 11010 | struct cftype *cft) |
| 11011 | { |
| 11012 | return css_tg(css)->idle; |
| 11013 | } |
| 11014 | |
| 11015 | static int cpu_idle_write_s64(struct cgroup_subsys_state *css, |
| 11016 | struct cftype *cft, s64 idle) |
| 11017 | { |
| 11018 | return sched_group_set_idle(css_tg(css), idle); |
| 11019 | } |
| 11020 | #endif |
| 11021 | |
Tejun Heo | a1f7164 | 2017-09-25 09:00:18 -0700 | [diff] [blame] | 11022 | static struct cftype cpu_legacy_files[] = { |
Peter Zijlstra | 052f1dc | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 11023 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Paul Menage | fe5c7cc | 2007-10-29 21:18:11 +0100 | [diff] [blame] | 11024 | { |
| 11025 | .name = "shares", |
Paul Menage | f4c753b | 2008-04-29 00:59:56 -0700 | [diff] [blame] | 11026 | .read_u64 = cpu_shares_read_u64, |
| 11027 | .write_u64 = cpu_shares_write_u64, |
Paul Menage | fe5c7cc | 2007-10-29 21:18:11 +0100 | [diff] [blame] | 11028 | }, |
Josh Don | 3040003 | 2021-07-29 19:00:18 -0700 | [diff] [blame] | 11029 | { |
| 11030 | .name = "idle", |
| 11031 | .read_s64 = cpu_idle_read_s64, |
| 11032 | .write_s64 = cpu_idle_write_s64, |
| 11033 | }, |
Peter Zijlstra | 052f1dc | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 11034 | #endif |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 11035 | #ifdef CONFIG_CFS_BANDWIDTH |
| 11036 | { |
| 11037 | .name = "cfs_quota_us", |
| 11038 | .read_s64 = cpu_cfs_quota_read_s64, |
| 11039 | .write_s64 = cpu_cfs_quota_write_s64, |
| 11040 | }, |
| 11041 | { |
| 11042 | .name = "cfs_period_us", |
| 11043 | .read_u64 = cpu_cfs_period_read_u64, |
| 11044 | .write_u64 = cpu_cfs_period_write_u64, |
| 11045 | }, |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 11046 | { |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 11047 | .name = "cfs_burst_us", |
| 11048 | .read_u64 = cpu_cfs_burst_read_u64, |
| 11049 | .write_u64 = cpu_cfs_burst_write_u64, |
| 11050 | }, |
| 11051 | { |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 11052 | .name = "stat", |
Tejun Heo | a1f7164 | 2017-09-25 09:00:18 -0700 | [diff] [blame] | 11053 | .seq_show = cpu_cfs_stat_show, |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 11054 | }, |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 11055 | #endif |
Peter Zijlstra | 052f1dc | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 11056 | #ifdef CONFIG_RT_GROUP_SCHED |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 11057 | { |
Peter Zijlstra | 9f0c1e5 | 2008-02-13 15:45:39 +0100 | [diff] [blame] | 11058 | .name = "rt_runtime_us", |
Paul Menage | 06ecb27 | 2008-04-29 01:00:06 -0700 | [diff] [blame] | 11059 | .read_s64 = cpu_rt_runtime_read, |
| 11060 | .write_s64 = cpu_rt_runtime_write, |
Peter Zijlstra | 6f505b1 | 2008-01-25 21:08:30 +0100 | [diff] [blame] | 11061 | }, |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 11062 | { |
| 11063 | .name = "rt_period_us", |
Paul Menage | f4c753b | 2008-04-29 00:59:56 -0700 | [diff] [blame] | 11064 | .read_u64 = cpu_rt_period_read_uint, |
| 11065 | .write_u64 = cpu_rt_period_write_uint, |
Peter Zijlstra | d0b27fa | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 11066 | }, |
Peter Zijlstra | 052f1dc | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 11067 | #endif |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 11068 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
| 11069 | { |
| 11070 | .name = "uclamp.min", |
| 11071 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11072 | .seq_show = cpu_uclamp_min_show, |
| 11073 | .write = cpu_uclamp_min_write, |
| 11074 | }, |
| 11075 | { |
| 11076 | .name = "uclamp.max", |
| 11077 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11078 | .seq_show = cpu_uclamp_max_show, |
| 11079 | .write = cpu_uclamp_max_write, |
| 11080 | }, |
| 11081 | #endif |
Ingo Molnar | d1ccc66 | 2017-02-01 11:46:42 +0100 | [diff] [blame] | 11082 | { } /* Terminate */ |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 11083 | }; |
| 11084 | |
Tejun Heo | d41bf8c9 | 2017-10-23 16:18:27 -0700 | [diff] [blame] | 11085 | static int cpu_extra_stat_show(struct seq_file *sf, |
| 11086 | struct cgroup_subsys_state *css) |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11087 | { |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11088 | #ifdef CONFIG_CFS_BANDWIDTH |
| 11089 | { |
Tejun Heo | d41bf8c9 | 2017-10-23 16:18:27 -0700 | [diff] [blame] | 11090 | struct task_group *tg = css_tg(css); |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11091 | struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; |
Huaixin Chang | bcb1704 | 2021-08-30 11:22:14 +0800 | [diff] [blame] | 11092 | u64 throttled_usec, burst_usec; |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11093 | |
| 11094 | throttled_usec = cfs_b->throttled_time; |
| 11095 | do_div(throttled_usec, NSEC_PER_USEC); |
Huaixin Chang | bcb1704 | 2021-08-30 11:22:14 +0800 | [diff] [blame] | 11096 | burst_usec = cfs_b->burst_time; |
| 11097 | do_div(burst_usec, NSEC_PER_USEC); |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11098 | |
| 11099 | seq_printf(sf, "nr_periods %d\n" |
| 11100 | "nr_throttled %d\n" |
Huaixin Chang | bcb1704 | 2021-08-30 11:22:14 +0800 | [diff] [blame] | 11101 | "throttled_usec %llu\n" |
| 11102 | "nr_bursts %d\n" |
| 11103 | "burst_usec %llu\n", |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11104 | cfs_b->nr_periods, cfs_b->nr_throttled, |
Huaixin Chang | bcb1704 | 2021-08-30 11:22:14 +0800 | [diff] [blame] | 11105 | throttled_usec, cfs_b->nr_burst, burst_usec); |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11106 | } |
| 11107 | #endif |
| 11108 | return 0; |
| 11109 | } |
| 11110 | |
| 11111 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 11112 | static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, |
| 11113 | struct cftype *cft) |
| 11114 | { |
| 11115 | struct task_group *tg = css_tg(css); |
| 11116 | u64 weight = scale_load_down(tg->shares); |
| 11117 | |
| 11118 | return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024); |
| 11119 | } |
| 11120 | |
| 11121 | static int cpu_weight_write_u64(struct cgroup_subsys_state *css, |
| 11122 | struct cftype *cft, u64 weight) |
| 11123 | { |
| 11124 | /* |
| 11125 | * cgroup weight knobs should use the common MIN, DFL and MAX |
| 11126 | * values which are 1, 100 and 10000 respectively. While it loses |
| 11127 | * a bit of range on both ends, it maps pretty well onto the shares |
| 11128 | * value used by scheduler and the round-trip conversions preserve |
| 11129 | * the original value over the entire range. |
| 11130 | */ |
| 11131 | if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX) |
| 11132 | return -ERANGE; |
| 11133 | |
| 11134 | weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL); |
| 11135 | |
| 11136 | return sched_group_set_shares(css_tg(css), scale_load(weight)); |
| 11137 | } |
| 11138 | |
| 11139 | static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, |
| 11140 | struct cftype *cft) |
| 11141 | { |
| 11142 | unsigned long weight = scale_load_down(css_tg(css)->shares); |
| 11143 | int last_delta = INT_MAX; |
| 11144 | int prio, delta; |
| 11145 | |
| 11146 | /* find the closest nice value to the current weight */ |
| 11147 | for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { |
| 11148 | delta = abs(sched_prio_to_weight[prio] - weight); |
| 11149 | if (delta >= last_delta) |
| 11150 | break; |
| 11151 | last_delta = delta; |
| 11152 | } |
| 11153 | |
| 11154 | return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); |
| 11155 | } |
| 11156 | |
| 11157 | static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, |
| 11158 | struct cftype *cft, s64 nice) |
| 11159 | { |
| 11160 | unsigned long weight; |
Peter Zijlstra | 7281c8d | 2018-04-20 14:29:51 +0200 | [diff] [blame] | 11161 | int idx; |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11162 | |
| 11163 | if (nice < MIN_NICE || nice > MAX_NICE) |
| 11164 | return -ERANGE; |
| 11165 | |
Peter Zijlstra | 7281c8d | 2018-04-20 14:29:51 +0200 | [diff] [blame] | 11166 | idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; |
| 11167 | idx = array_index_nospec(idx, 40); |
| 11168 | weight = sched_prio_to_weight[idx]; |
| 11169 | |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11170 | return sched_group_set_shares(css_tg(css), scale_load(weight)); |
| 11171 | } |
| 11172 | #endif |
| 11173 | |
| 11174 | static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, |
| 11175 | long period, long quota) |
| 11176 | { |
| 11177 | if (quota < 0) |
| 11178 | seq_puts(sf, "max"); |
| 11179 | else |
| 11180 | seq_printf(sf, "%ld", quota); |
| 11181 | |
| 11182 | seq_printf(sf, " %ld\n", period); |
| 11183 | } |
| 11184 | |
| 11185 | /* caller should put the current value in *@periodp before calling */ |
| 11186 | static int __maybe_unused cpu_period_quota_parse(char *buf, |
| 11187 | u64 *periodp, u64 *quotap) |
| 11188 | { |
| 11189 | char tok[21]; /* U64_MAX */ |
| 11190 | |
Konstantin Khlebnikov | 4c47acd | 2019-03-06 20:11:42 +0300 | [diff] [blame] | 11191 | if (sscanf(buf, "%20s %llu", tok, periodp) < 1) |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11192 | return -EINVAL; |
| 11193 | |
| 11194 | *periodp *= NSEC_PER_USEC; |
| 11195 | |
| 11196 | if (sscanf(tok, "%llu", quotap)) |
| 11197 | *quotap *= NSEC_PER_USEC; |
| 11198 | else if (!strcmp(tok, "max")) |
| 11199 | *quotap = RUNTIME_INF; |
| 11200 | else |
| 11201 | return -EINVAL; |
| 11202 | |
| 11203 | return 0; |
| 11204 | } |
| 11205 | |
| 11206 | #ifdef CONFIG_CFS_BANDWIDTH |
| 11207 | static int cpu_max_show(struct seq_file *sf, void *v) |
| 11208 | { |
| 11209 | struct task_group *tg = css_tg(seq_css(sf)); |
| 11210 | |
| 11211 | cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg)); |
| 11212 | return 0; |
| 11213 | } |
| 11214 | |
| 11215 | static ssize_t cpu_max_write(struct kernfs_open_file *of, |
| 11216 | char *buf, size_t nbytes, loff_t off) |
| 11217 | { |
| 11218 | struct task_group *tg = css_tg(of_css(of)); |
| 11219 | u64 period = tg_get_cfs_period(tg); |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 11220 | u64 burst = tg_get_cfs_burst(tg); |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11221 | u64 quota; |
| 11222 | int ret; |
| 11223 | |
| 11224 | ret = cpu_period_quota_parse(buf, &period, "a); |
| 11225 | if (!ret) |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 11226 | ret = tg_set_cfs_bandwidth(tg, period, quota, burst); |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11227 | return ret ?: nbytes; |
| 11228 | } |
| 11229 | #endif |
| 11230 | |
| 11231 | static struct cftype cpu_files[] = { |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11232 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 11233 | { |
| 11234 | .name = "weight", |
| 11235 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11236 | .read_u64 = cpu_weight_read_u64, |
| 11237 | .write_u64 = cpu_weight_write_u64, |
| 11238 | }, |
| 11239 | { |
| 11240 | .name = "weight.nice", |
| 11241 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11242 | .read_s64 = cpu_weight_nice_read_s64, |
| 11243 | .write_s64 = cpu_weight_nice_write_s64, |
| 11244 | }, |
Josh Don | 3040003 | 2021-07-29 19:00:18 -0700 | [diff] [blame] | 11245 | { |
| 11246 | .name = "idle", |
| 11247 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11248 | .read_s64 = cpu_idle_read_s64, |
| 11249 | .write_s64 = cpu_idle_write_s64, |
| 11250 | }, |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11251 | #endif |
| 11252 | #ifdef CONFIG_CFS_BANDWIDTH |
| 11253 | { |
| 11254 | .name = "max", |
| 11255 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11256 | .seq_show = cpu_max_show, |
| 11257 | .write = cpu_max_write, |
| 11258 | }, |
Huaixin Chang | f418371 | 2021-06-21 17:27:58 +0800 | [diff] [blame] | 11259 | { |
| 11260 | .name = "max.burst", |
| 11261 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11262 | .read_u64 = cpu_cfs_burst_read_u64, |
| 11263 | .write_u64 = cpu_cfs_burst_write_u64, |
| 11264 | }, |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11265 | #endif |
Patrick Bellasi | 2480c09 | 2019-08-22 14:28:06 +0100 | [diff] [blame] | 11266 | #ifdef CONFIG_UCLAMP_TASK_GROUP |
| 11267 | { |
| 11268 | .name = "uclamp.min", |
| 11269 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11270 | .seq_show = cpu_uclamp_min_show, |
| 11271 | .write = cpu_uclamp_min_write, |
| 11272 | }, |
| 11273 | { |
| 11274 | .name = "uclamp.max", |
| 11275 | .flags = CFTYPE_NOT_ON_ROOT, |
| 11276 | .seq_show = cpu_uclamp_max_show, |
| 11277 | .write = cpu_uclamp_max_write, |
| 11278 | }, |
| 11279 | #endif |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11280 | { } /* terminate */ |
| 11281 | }; |
| 11282 | |
Tejun Heo | 073219e | 2014-02-08 10:36:58 -0500 | [diff] [blame] | 11283 | struct cgroup_subsys cpu_cgrp_subsys = { |
Tejun Heo | 92fb974 | 2012-11-19 08:13:38 -0800 | [diff] [blame] | 11284 | .css_alloc = cpu_cgroup_css_alloc, |
Konstantin Khlebnikov | 96b7774 | 2017-02-08 14:27:27 +0300 | [diff] [blame] | 11285 | .css_online = cpu_cgroup_css_online, |
Peter Zijlstra | 2f5177f | 2016-03-16 16:22:45 +0100 | [diff] [blame] | 11286 | .css_released = cpu_cgroup_css_released, |
Tejun Heo | 92fb974 | 2012-11-19 08:13:38 -0800 | [diff] [blame] | 11287 | .css_free = cpu_cgroup_css_free, |
Tejun Heo | d41bf8c9 | 2017-10-23 16:18:27 -0700 | [diff] [blame] | 11288 | .css_extra_stat_show = cpu_extra_stat_show, |
Chengming Zhou | df16b71 | 2022-08-18 20:48:03 +0800 | [diff] [blame] | 11289 | #ifdef CONFIG_RT_GROUP_SCHED |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 11290 | .can_attach = cpu_cgroup_can_attach, |
Chengming Zhou | df16b71 | 2022-08-18 20:48:03 +0800 | [diff] [blame] | 11291 | #endif |
Tejun Heo | bb9d97b | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 11292 | .attach = cpu_cgroup_attach, |
Tejun Heo | a1f7164 | 2017-09-25 09:00:18 -0700 | [diff] [blame] | 11293 | .legacy_cftypes = cpu_legacy_files, |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11294 | .dfl_cftypes = cpu_files, |
Tejun Heo | b38e42e | 2016-02-23 10:00:50 -0500 | [diff] [blame] | 11295 | .early_init = true, |
Tejun Heo | 0d59363 | 2017-09-25 09:00:19 -0700 | [diff] [blame] | 11296 | .threaded = true, |
Srivatsa Vaddagiri | 68318b8 | 2007-10-18 23:41:03 -0700 | [diff] [blame] | 11297 | }; |
| 11298 | |
Peter Zijlstra | 052f1dc | 2008-02-13 15:45:40 +0100 | [diff] [blame] | 11299 | #endif /* CONFIG_CGROUP_SCHED */ |
Srivatsa Vaddagiri | d842de8 | 2007-12-02 20:04:49 +0100 | [diff] [blame] | 11300 | |
Paul E. McKenney | b637a32 | 2012-09-19 16:58:38 -0700 | [diff] [blame] | 11301 | void dump_cpu_task(int cpu) |
| 11302 | { |
Zhen Lei | bc1cca9 | 2022-08-04 10:34:20 +0800 | [diff] [blame] | 11303 | if (cpu == smp_processor_id() && in_hardirq()) { |
| 11304 | struct pt_regs *regs; |
| 11305 | |
| 11306 | regs = get_irq_regs(); |
| 11307 | if (regs) { |
| 11308 | show_regs(regs); |
| 11309 | return; |
| 11310 | } |
| 11311 | } |
| 11312 | |
Zhen Lei | e73dfe3 | 2022-08-04 10:34:19 +0800 | [diff] [blame] | 11313 | if (trigger_single_cpu_backtrace(cpu)) |
| 11314 | return; |
| 11315 | |
Paul E. McKenney | b637a32 | 2012-09-19 16:58:38 -0700 | [diff] [blame] | 11316 | pr_info("Task dump for CPU %d:\n", cpu); |
| 11317 | sched_show_task(cpu_curr(cpu)); |
| 11318 | } |
Andi Kleen | ed82b8a | 2015-11-29 20:59:43 -0800 | [diff] [blame] | 11319 | |
| 11320 | /* |
| 11321 | * Nice levels are multiplicative, with a gentle 10% change for every |
| 11322 | * nice level changed. I.e. when a CPU-bound task goes from nice 0 to |
| 11323 | * nice 1, it will get ~10% less CPU time than another CPU-bound task |
| 11324 | * that remained on nice 0. |
| 11325 | * |
| 11326 | * The "10% effect" is relative and cumulative: from _any_ nice level, |
| 11327 | * if you go up 1 level, it's -10% CPU usage, if you go down 1 level |
| 11328 | * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. |
| 11329 | * If a task goes up by ~10% and another task goes down by ~10% then |
| 11330 | * the relative distance between them is ~25%.) |
| 11331 | */ |
| 11332 | const int sched_prio_to_weight[40] = { |
| 11333 | /* -20 */ 88761, 71755, 56483, 46273, 36291, |
| 11334 | /* -15 */ 29154, 23254, 18705, 14949, 11916, |
| 11335 | /* -10 */ 9548, 7620, 6100, 4904, 3906, |
| 11336 | /* -5 */ 3121, 2501, 1991, 1586, 1277, |
| 11337 | /* 0 */ 1024, 820, 655, 526, 423, |
| 11338 | /* 5 */ 335, 272, 215, 172, 137, |
| 11339 | /* 10 */ 110, 87, 70, 56, 45, |
| 11340 | /* 15 */ 36, 29, 23, 18, 15, |
| 11341 | }; |
| 11342 | |
| 11343 | /* |
| 11344 | * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. |
| 11345 | * |
| 11346 | * In cases where the weight does not change often, we can use the |
| 11347 | * precalculated inverse to speed up arithmetics by turning divisions |
| 11348 | * into multiplications: |
| 11349 | */ |
| 11350 | const u32 sched_prio_to_wmult[40] = { |
| 11351 | /* -20 */ 48388, 59856, 76040, 92818, 118348, |
| 11352 | /* -15 */ 147320, 184698, 229616, 287308, 360437, |
| 11353 | /* -10 */ 449829, 563644, 704093, 875809, 1099582, |
| 11354 | /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, |
| 11355 | /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, |
| 11356 | /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, |
| 11357 | /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, |
| 11358 | /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, |
| 11359 | }; |
Ingo Molnar | 14a7405 | 2018-03-03 16:32:24 +0100 | [diff] [blame] | 11360 | |
Phil Auld | 9d24605 | 2020-06-29 15:23:03 -0400 | [diff] [blame] | 11361 | void call_trace_sched_update_nr_running(struct rq *rq, int count) |
| 11362 | { |
| 11363 | trace_sched_update_nr_running_tp(rq, count); |
| 11364 | } |