Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 2 | |
| 3 | #ifdef CONFIG_SCHEDSTATS |
Alexey Dobriyan | b5aadf7 | 2008-10-06 13:23:43 +0400 | [diff] [blame] | 4 | |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 5 | /* |
| 6 | * Expects runqueue lock to be held for atomicity of update |
| 7 | */ |
| 8 | static inline void |
| 9 | rq_sched_info_arrive(struct rq *rq, unsigned long long delta) |
| 10 | { |
| 11 | if (rq) { |
| 12 | rq->rq_sched_info.run_delay += delta; |
Ingo Molnar | 2d72376 | 2007-10-15 17:00:12 +0200 | [diff] [blame] | 13 | rq->rq_sched_info.pcount++; |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 14 | } |
| 15 | } |
| 16 | |
| 17 | /* |
| 18 | * Expects runqueue lock to be held for atomicity of update |
| 19 | */ |
| 20 | static inline void |
| 21 | rq_sched_info_depart(struct rq *rq, unsigned long long delta) |
| 22 | { |
| 23 | if (rq) |
Ken Chen | 9c2c480 | 2008-12-16 23:41:22 -0800 | [diff] [blame] | 24 | rq->rq_cpu_time += delta; |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 25 | } |
Ankita Garg | 46ac22b | 2008-07-01 14:30:06 +0530 | [diff] [blame] | 26 | |
| 27 | static inline void |
| 28 | rq_sched_info_dequeued(struct rq *rq, unsigned long long delta) |
| 29 | { |
| 30 | if (rq) |
| 31 | rq->rq_sched_info.run_delay += delta; |
| 32 | } |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 33 | #define schedstat_enabled() static_branch_unlikely(&sched_schedstats) |
Peter Zijlstra | b85c8b7 | 2018-01-16 20:51:06 +0100 | [diff] [blame] | 34 | #define __schedstat_inc(var) do { var++; } while (0) |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 35 | #define schedstat_inc(var) do { if (schedstat_enabled()) { var++; } } while (0) |
Peter Zijlstra | 2ed41a5 | 2018-01-23 20:34:30 +0100 | [diff] [blame] | 36 | #define __schedstat_add(var, amt) do { var += (amt); } while (0) |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 37 | #define schedstat_add(var, amt) do { if (schedstat_enabled()) { var += (amt); } } while (0) |
| 38 | #define __schedstat_set(var, val) do { var = (val); } while (0) |
| 39 | #define schedstat_set(var, val) do { if (schedstat_enabled()) { var = (val); } } while (0) |
| 40 | #define schedstat_val(var) (var) |
| 41 | #define schedstat_val_or_zero(var) ((schedstat_enabled()) ? (var) : 0) |
Josh Poimboeuf | 9c57259 | 2016-06-03 17:58:40 -0500 | [diff] [blame] | 42 | |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 43 | #else /* !CONFIG_SCHEDSTATS: */ |
| 44 | static inline void rq_sched_info_arrive (struct rq *rq, unsigned long long delta) { } |
| 45 | static inline void rq_sched_info_dequeued(struct rq *rq, unsigned long long delta) { } |
| 46 | static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delta) { } |
| 47 | # define schedstat_enabled() 0 |
| 48 | # define __schedstat_inc(var) do { } while (0) |
| 49 | # define schedstat_inc(var) do { } while (0) |
| 50 | # define __schedstat_add(var, amt) do { } while (0) |
| 51 | # define schedstat_add(var, amt) do { } while (0) |
| 52 | # define __schedstat_set(var, val) do { } while (0) |
| 53 | # define schedstat_set(var, val) do { } while (0) |
| 54 | # define schedstat_val(var) 0 |
| 55 | # define schedstat_val_or_zero(var) 0 |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 56 | #endif /* CONFIG_SCHEDSTATS */ |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 57 | |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 58 | #ifdef CONFIG_PSI |
| 59 | /* |
| 60 | * PSI tracks state that persists across sleeps, such as iowaits and |
| 61 | * memory stalls. As a result, it has to distinguish between sleeps, |
| 62 | * where a task's runnable state changes, and requeues, where a task |
| 63 | * and its state are being moved between CPUs and runqueues. |
| 64 | */ |
| 65 | static inline void psi_enqueue(struct task_struct *p, bool wakeup) |
| 66 | { |
| 67 | int clear = 0, set = TSK_RUNNING; |
| 68 | |
Johannes Weiner | e0c2744 | 2018-11-30 14:09:58 -0800 | [diff] [blame] | 69 | if (static_branch_likely(&psi_disabled)) |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 70 | return; |
| 71 | |
| 72 | if (!wakeup || p->sched_psi_wake_requeue) { |
Yafang Shao | 1066d1b | 2020-03-16 21:28:05 -0400 | [diff] [blame] | 73 | if (p->in_memstall) |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 74 | set |= TSK_MEMSTALL; |
| 75 | if (p->sched_psi_wake_requeue) |
| 76 | p->sched_psi_wake_requeue = 0; |
| 77 | } else { |
| 78 | if (p->in_iowait) |
| 79 | clear |= TSK_IOWAIT; |
| 80 | } |
| 81 | |
| 82 | psi_task_change(p, clear, set); |
| 83 | } |
| 84 | |
| 85 | static inline void psi_dequeue(struct task_struct *p, bool sleep) |
| 86 | { |
| 87 | int clear = TSK_RUNNING, set = 0; |
| 88 | |
Johannes Weiner | e0c2744 | 2018-11-30 14:09:58 -0800 | [diff] [blame] | 89 | if (static_branch_likely(&psi_disabled)) |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 90 | return; |
| 91 | |
| 92 | if (!sleep) { |
Yafang Shao | 1066d1b | 2020-03-16 21:28:05 -0400 | [diff] [blame] | 93 | if (p->in_memstall) |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 94 | clear |= TSK_MEMSTALL; |
| 95 | } else { |
Johannes Weiner | b05e75d | 2020-03-16 15:13:31 -0400 | [diff] [blame] | 96 | /* |
| 97 | * When a task sleeps, schedule() dequeues it before |
| 98 | * switching to the next one. Merge the clearing of |
| 99 | * TSK_RUNNING and TSK_ONCPU to save an unnecessary |
| 100 | * psi_task_change() call in psi_sched_switch(). |
| 101 | */ |
| 102 | clear |= TSK_ONCPU; |
| 103 | |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 104 | if (p->in_iowait) |
| 105 | set |= TSK_IOWAIT; |
| 106 | } |
| 107 | |
| 108 | psi_task_change(p, clear, set); |
| 109 | } |
| 110 | |
| 111 | static inline void psi_ttwu_dequeue(struct task_struct *p) |
| 112 | { |
Johannes Weiner | e0c2744 | 2018-11-30 14:09:58 -0800 | [diff] [blame] | 113 | if (static_branch_likely(&psi_disabled)) |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 114 | return; |
| 115 | /* |
| 116 | * Is the task being migrated during a wakeup? Make sure to |
| 117 | * deregister its sleep-persistent psi states from the old |
| 118 | * queue, and let psi_enqueue() know it has to requeue. |
| 119 | */ |
Yafang Shao | 1066d1b | 2020-03-16 21:28:05 -0400 | [diff] [blame] | 120 | if (unlikely(p->in_iowait || p->in_memstall)) { |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 121 | struct rq_flags rf; |
| 122 | struct rq *rq; |
| 123 | int clear = 0; |
| 124 | |
| 125 | if (p->in_iowait) |
| 126 | clear |= TSK_IOWAIT; |
Yafang Shao | 1066d1b | 2020-03-16 21:28:05 -0400 | [diff] [blame] | 127 | if (p->in_memstall) |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 128 | clear |= TSK_MEMSTALL; |
| 129 | |
| 130 | rq = __task_rq_lock(p, &rf); |
| 131 | psi_task_change(p, clear, 0); |
| 132 | p->sched_psi_wake_requeue = 1; |
| 133 | __task_rq_unlock(rq, &rf); |
| 134 | } |
| 135 | } |
| 136 | |
Johannes Weiner | b05e75d | 2020-03-16 15:13:31 -0400 | [diff] [blame] | 137 | static inline void psi_sched_switch(struct task_struct *prev, |
| 138 | struct task_struct *next, |
| 139 | bool sleep) |
| 140 | { |
| 141 | if (static_branch_likely(&psi_disabled)) |
| 142 | return; |
| 143 | |
Johannes Weiner | 36b238d | 2020-03-16 15:13:32 -0400 | [diff] [blame] | 144 | psi_task_switch(prev, next, sleep); |
Johannes Weiner | b05e75d | 2020-03-16 15:13:31 -0400 | [diff] [blame] | 145 | } |
| 146 | |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 147 | static inline void psi_task_tick(struct rq *rq) |
| 148 | { |
Johannes Weiner | e0c2744 | 2018-11-30 14:09:58 -0800 | [diff] [blame] | 149 | if (static_branch_likely(&psi_disabled)) |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 150 | return; |
| 151 | |
Yafang Shao | 1066d1b | 2020-03-16 21:28:05 -0400 | [diff] [blame] | 152 | if (unlikely(rq->curr->in_memstall)) |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 153 | psi_memstall_tick(rq->curr, cpu_of(rq)); |
| 154 | } |
| 155 | #else /* CONFIG_PSI */ |
| 156 | static inline void psi_enqueue(struct task_struct *p, bool wakeup) {} |
| 157 | static inline void psi_dequeue(struct task_struct *p, bool sleep) {} |
| 158 | static inline void psi_ttwu_dequeue(struct task_struct *p) {} |
Johannes Weiner | b05e75d | 2020-03-16 15:13:31 -0400 | [diff] [blame] | 159 | static inline void psi_sched_switch(struct task_struct *prev, |
| 160 | struct task_struct *next, |
| 161 | bool sleep) {} |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 162 | static inline void psi_task_tick(struct rq *rq) {} |
| 163 | #endif /* CONFIG_PSI */ |
| 164 | |
Naveen N. Rao | f6db834 | 2015-06-25 23:53:37 +0530 | [diff] [blame] | 165 | #ifdef CONFIG_SCHED_INFO |
Ankita Garg | 46ac22b | 2008-07-01 14:30:06 +0530 | [diff] [blame] | 166 | static inline void sched_info_reset_dequeued(struct task_struct *t) |
| 167 | { |
| 168 | t->sched_info.last_queued = 0; |
| 169 | } |
| 170 | |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 171 | /* |
Rakib Mullick | d4a6f3c | 2010-10-24 16:28:47 +0600 | [diff] [blame] | 172 | * We are interested in knowing how long it was from the *first* time a |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 173 | * task was queued to the time that it finally hit a CPU, we call this routine |
| 174 | * from dequeue_task() to account for possible rq->clock skew across CPUs. The |
| 175 | * delta taken on each CPU would annul the skew. |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 176 | */ |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 177 | static inline void sched_info_dequeued(struct rq *rq, struct task_struct *t) |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 178 | { |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 179 | unsigned long long now = rq_clock(rq), delta = 0; |
Ankita Garg | 46ac22b | 2008-07-01 14:30:06 +0530 | [diff] [blame] | 180 | |
Yi Wang | 65d74e9 | 2019-07-05 12:35:07 +0800 | [diff] [blame] | 181 | if (sched_info_on()) { |
Ankita Garg | 46ac22b | 2008-07-01 14:30:06 +0530 | [diff] [blame] | 182 | if (t->sched_info.last_queued) |
| 183 | delta = now - t->sched_info.last_queued; |
Yi Wang | 65d74e9 | 2019-07-05 12:35:07 +0800 | [diff] [blame] | 184 | } |
Ankita Garg | 46ac22b | 2008-07-01 14:30:06 +0530 | [diff] [blame] | 185 | sched_info_reset_dequeued(t); |
| 186 | t->sched_info.run_delay += delta; |
| 187 | |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 188 | rq_sched_info_dequeued(rq, delta); |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 189 | } |
| 190 | |
| 191 | /* |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 192 | * Called when a task finally hits the CPU. We can now calculate how |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 193 | * long it was waiting to run. We also note when it began so that we |
| 194 | * can keep stats on how long its timeslice is. |
| 195 | */ |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 196 | static void sched_info_arrive(struct rq *rq, struct task_struct *t) |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 197 | { |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 198 | unsigned long long now = rq_clock(rq), delta = 0; |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 199 | |
| 200 | if (t->sched_info.last_queued) |
| 201 | delta = now - t->sched_info.last_queued; |
Ankita Garg | 46ac22b | 2008-07-01 14:30:06 +0530 | [diff] [blame] | 202 | sched_info_reset_dequeued(t); |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 203 | t->sched_info.run_delay += delta; |
| 204 | t->sched_info.last_arrival = now; |
Ingo Molnar | 2d72376 | 2007-10-15 17:00:12 +0200 | [diff] [blame] | 205 | t->sched_info.pcount++; |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 206 | |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 207 | rq_sched_info_arrive(rq, delta); |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 208 | } |
| 209 | |
| 210 | /* |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 211 | * This function is only called from enqueue_task(), but also only updates |
| 212 | * the timestamp if it is already not set. It's assumed that |
| 213 | * sched_info_dequeued() will clear that stamp when appropriate. |
| 214 | */ |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 215 | static inline void sched_info_queued(struct rq *rq, struct task_struct *t) |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 216 | { |
Yi Wang | 65d74e9 | 2019-07-05 12:35:07 +0800 | [diff] [blame] | 217 | if (sched_info_on()) { |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 218 | if (!t->sched_info.last_queued) |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 219 | t->sched_info.last_queued = rq_clock(rq); |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 220 | } |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 221 | } |
| 222 | |
| 223 | /* |
Michael S. Tsirkin | 13b62e4 | 2013-09-16 11:30:36 +0300 | [diff] [blame] | 224 | * Called when a process ceases being the active-running process involuntarily |
| 225 | * due, typically, to expiring its time slice (this may also be called when |
| 226 | * switching to the idle task). Now we can calculate how long we ran. |
Bharath Ravi | d4abc23 | 2008-06-16 15:11:01 +0530 | [diff] [blame] | 227 | * Also, if the process is still in the TASK_RUNNING state, call |
| 228 | * sched_info_queued() to mark that it has now again started waiting on |
| 229 | * the runqueue. |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 230 | */ |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 231 | static inline void sched_info_depart(struct rq *rq, struct task_struct *t) |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 232 | { |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 233 | unsigned long long delta = rq_clock(rq) - t->sched_info.last_arrival; |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 234 | |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 235 | rq_sched_info_depart(rq, delta); |
Bharath Ravi | d4abc23 | 2008-06-16 15:11:01 +0530 | [diff] [blame] | 236 | |
| 237 | if (t->state == TASK_RUNNING) |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 238 | sched_info_queued(rq, t); |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 239 | } |
| 240 | |
| 241 | /* |
| 242 | * Called when tasks are switched involuntarily due, typically, to expiring |
| 243 | * their time slice. (This may also be called when switching to or from |
| 244 | * the idle task.) We are only called when prev != next. |
| 245 | */ |
| 246 | static inline void |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 247 | __sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 248 | { |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 249 | /* |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 250 | * prev now departs the CPU. It's not interesting to record |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 251 | * stats about how efficient we were at scheduling the idle |
| 252 | * process, however. |
| 253 | */ |
| 254 | if (prev != rq->idle) |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 255 | sched_info_depart(rq, prev); |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 256 | |
| 257 | if (next != rq->idle) |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 258 | sched_info_arrive(rq, next); |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 259 | } |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 260 | |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 261 | static inline void |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 262 | sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 263 | { |
Yi Wang | 65d74e9 | 2019-07-05 12:35:07 +0800 | [diff] [blame] | 264 | if (sched_info_on()) |
Michael S. Tsirkin | 4314895 | 2013-09-22 17:20:54 +0300 | [diff] [blame] | 265 | __sched_info_switch(rq, prev, next); |
Ingo Molnar | 425e096 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 266 | } |
Ingo Molnar | 97fb7a0 | 2018-03-03 14:01:12 +0100 | [diff] [blame] | 267 | |
| 268 | #else /* !CONFIG_SCHED_INFO: */ |
| 269 | # define sched_info_queued(rq, t) do { } while (0) |
| 270 | # define sched_info_reset_dequeued(t) do { } while (0) |
| 271 | # define sched_info_dequeued(rq, t) do { } while (0) |
| 272 | # define sched_info_depart(rq, t) do { } while (0) |
| 273 | # define sched_info_arrive(rq, next) do { } while (0) |
| 274 | # define sched_info_switch(rq, t, next) do { } while (0) |
Naveen N. Rao | f6db834 | 2015-06-25 23:53:37 +0530 | [diff] [blame] | 275 | #endif /* CONFIG_SCHED_INFO */ |