blob: ff9435dee1df218a96d05ac4fc1130f0bca204e1 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Ingo Molnar325ea102018-03-03 12:20:47 +01002/*
3 * Simple CPU accounting cgroup controller
4 */
Frederic Weisbecker73fbec62012-06-16 15:57:37 +02005#include "sched.h"
Frederic Weisbecker73fbec62012-06-16 15:57:37 +02006
7#ifdef CONFIG_IRQ_TIME_ACCOUNTING
8
9/*
10 * There are no locks covering percpu hardirq/softirq time.
Frederic Weisbeckerbf9fae92012-09-08 15:23:11 +020011 * They are only modified in vtime_account, on corresponding CPU
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020012 * with interrupts disabled. So, writes are safe.
13 * They are read and saved off onto struct rq in update_rq_clock().
14 * This may result in other CPU reading this CPU's irq time and can
Frederic Weisbeckerbf9fae92012-09-08 15:23:11 +020015 * race with irq/vtime_account on this CPU. We would either get old
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020016 * or new value with a side effect of accounting a slice of irq time to wrong
17 * task when irq is in progress while we read rq->clock. That is a worthy
18 * compromise in place of having locks on each irq in account_system_time.
19 */
Frederic Weisbecker19d23dbf2016-09-26 02:29:20 +020020DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020021
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020022static int sched_clock_irqtime;
23
24void enable_sched_clock_irqtime(void)
25{
26 sched_clock_irqtime = 1;
27}
28
29void disable_sched_clock_irqtime(void)
30{
31 sched_clock_irqtime = 0;
32}
33
Frederic Weisbecker25e2d8c2017-04-25 16:10:48 +020034static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
35 enum cpu_usage_stat idx)
36{
37 u64 *cpustat = kcpustat_this_cpu->cpustat;
38
39 u64_stats_update_begin(&irqtime->sync);
40 cpustat[idx] += delta;
41 irqtime->total += delta;
42 irqtime->tick_delta += delta;
43 u64_stats_update_end(&irqtime->sync);
44}
45
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020046/*
47 * Called before incrementing preempt_count on {soft,}irq_enter
48 * and before decrementing preempt_count on {soft,}irq_exit.
49 */
Frederic Weisbecker3e1df4f52012-10-06 05:23:22 +020050void irqtime_account_irq(struct task_struct *curr)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020051{
Frederic Weisbecker19d23dbf2016-09-26 02:29:20 +020052 struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020053 s64 delta;
54 int cpu;
55
56 if (!sched_clock_irqtime)
57 return;
58
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020059 cpu = smp_processor_id();
Frederic Weisbecker19d23dbf2016-09-26 02:29:20 +020060 delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
61 irqtime->irq_start_time += delta;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020062
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020063 /*
64 * We do not account for softirq time from ksoftirqd here.
65 * We want to continue accounting softirq time to ksoftirqd thread
66 * in that case, so as not to confuse scheduler with a special task
67 * that do not consume any time, but still wants to run.
68 */
Frederic Weisbecker25e2d8c2017-04-25 16:10:48 +020069 if (hardirq_count())
70 irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
71 else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
72 irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020073}
Frederic Weisbecker3e1df4f52012-10-06 05:23:22 +020074EXPORT_SYMBOL_GPL(irqtime_account_irq);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020075
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +010076static u64 irqtime_tick_accounted(u64 maxtime)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020077{
Frederic Weisbeckera499a5a2017-01-31 04:09:32 +010078 struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +010079 u64 delta;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020080
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +010081 delta = min(irqtime->tick_delta, maxtime);
82 irqtime->tick_delta -= delta;
Frederic Weisbecker2810f612016-09-26 02:29:18 +020083
Frederic Weisbeckera499a5a2017-01-31 04:09:32 +010084 return delta;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020085}
86
87#else /* CONFIG_IRQ_TIME_ACCOUNTING */
88
89#define sched_clock_irqtime (0)
90
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +010091static u64 irqtime_tick_accounted(u64 dummy)
Rik van Riel57430212016-07-13 16:50:01 +020092{
93 return 0;
94}
95
Frederic Weisbecker73fbec62012-06-16 15:57:37 +020096#endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
97
98static inline void task_group_account_field(struct task_struct *p, int index,
99 u64 tmp)
100{
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200101 /*
102 * Since all updates are sure to touch the root cgroup, we
103 * get ourselves ahead and touch it first. If the root cgroup
104 * is the only cgroup, then nothing else should be necessary.
105 *
106 */
Christoph Lametera4f61cc2013-08-07 15:38:24 +0000107 __this_cpu_add(kernel_cpustat.cpustat[index], tmp);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200108
Tejun Heod2cc5ed2017-09-25 08:12:04 -0700109 cgroup_account_cputime_field(p, index, tmp);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200110}
111
112/*
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100113 * Account user CPU time to a process.
114 * @p: the process that the CPU time gets accounted to
115 * @cputime: the CPU time spent in user space since the last update
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200116 */
Frederic Weisbecker23244a52017-01-31 04:09:37 +0100117void account_user_time(struct task_struct *p, u64 cputime)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200118{
119 int index;
120
121 /* Add user time to process. */
Frederic Weisbecker23244a52017-01-31 04:09:37 +0100122 p->utime += cputime;
123 account_group_user_time(p, cputime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200124
Dongsheng Yangd0ea0262014-01-27 22:00:45 -0500125 index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200126
127 /* Add user time to cpustat. */
Frederic Weisbecker23244a52017-01-31 04:09:37 +0100128 task_group_account_field(p, index, cputime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200129
130 /* Account for user time used */
Frederic Weisbecker6fac4822012-11-13 14:20:55 +0100131 acct_account_cputime(p);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200132}
133
134/*
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100135 * Account guest CPU time to a process.
136 * @p: the process that the CPU time gets accounted to
137 * @cputime: the CPU time spent in virtual machine since the last update
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200138 */
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100139void account_guest_time(struct task_struct *p, u64 cputime)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200140{
141 u64 *cpustat = kcpustat_this_cpu->cpustat;
142
143 /* Add guest time to process. */
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100144 p->utime += cputime;
145 account_group_user_time(p, cputime);
146 p->gtime += cputime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200147
148 /* Add guest time to cpustat. */
Dongsheng Yangd0ea0262014-01-27 22:00:45 -0500149 if (task_nice(p) > 0) {
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100150 cpustat[CPUTIME_NICE] += cputime;
151 cpustat[CPUTIME_GUEST_NICE] += cputime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200152 } else {
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100153 cpustat[CPUTIME_USER] += cputime;
154 cpustat[CPUTIME_GUEST] += cputime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200155 }
156}
157
158/*
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100159 * Account system CPU time to a process and desired cpustat field
160 * @p: the process that the CPU time gets accounted to
161 * @cputime: the CPU time spent in kernel space since the last update
Stanislaw Gruszka40565b52016-11-15 03:06:51 +0100162 * @index: pointer to cpustat field that has to be updated
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200163 */
Frederic Weisbeckerc31cc6a2017-01-05 18:11:43 +0100164void account_system_index_time(struct task_struct *p,
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100165 u64 cputime, enum cpu_usage_stat index)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200166{
167 /* Add system time to process. */
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100168 p->stime += cputime;
169 account_group_system_time(p, cputime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200170
171 /* Add system time to cpustat. */
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100172 task_group_account_field(p, index, cputime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200173
174 /* Account for system time used */
Frederic Weisbecker6fac4822012-11-13 14:20:55 +0100175 acct_account_cputime(p);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200176}
177
178/*
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100179 * Account system CPU time to a process.
180 * @p: the process that the CPU time gets accounted to
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200181 * @hardirq_offset: the offset to subtract from hardirq_count()
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100182 * @cputime: the CPU time spent in kernel space since the last update
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200183 */
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100184void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200185{
186 int index;
187
188 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
Stanislaw Gruszka40565b52016-11-15 03:06:51 +0100189 account_guest_time(p, cputime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200190 return;
191 }
192
193 if (hardirq_count() - hardirq_offset)
194 index = CPUTIME_IRQ;
195 else if (in_serving_softirq())
196 index = CPUTIME_SOFTIRQ;
197 else
198 index = CPUTIME_SYSTEM;
199
Frederic Weisbeckerc31cc6a2017-01-05 18:11:43 +0100200 account_system_index_time(p, cputime, index);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200201}
202
203/*
204 * Account for involuntary wait time.
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100205 * @cputime: the CPU time spent in involuntary wait
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200206 */
Frederic Weisbeckerbe9095e2017-01-31 04:09:38 +0100207void account_steal_time(u64 cputime)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200208{
209 u64 *cpustat = kcpustat_this_cpu->cpustat;
210
Frederic Weisbeckerbe9095e2017-01-31 04:09:38 +0100211 cpustat[CPUTIME_STEAL] += cputime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200212}
213
214/*
215 * Account for idle time.
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100216 * @cputime: the CPU time spent in idle wait
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200217 */
Frederic Weisbecker18b43a92017-01-31 04:09:39 +0100218void account_idle_time(u64 cputime)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200219{
220 u64 *cpustat = kcpustat_this_cpu->cpustat;
221 struct rq *rq = this_rq();
222
223 if (atomic_read(&rq->nr_iowait) > 0)
Frederic Weisbecker18b43a92017-01-31 04:09:39 +0100224 cpustat[CPUTIME_IOWAIT] += cputime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200225 else
Frederic Weisbecker18b43a92017-01-31 04:09:39 +0100226 cpustat[CPUTIME_IDLE] += cputime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200227}
228
Wanpeng Li03cbc732016-08-17 10:05:46 +0800229/*
230 * When a guest is interrupted for a longer amount of time, missed clock
231 * ticks are not redelivered later. Due to that, this function may on
232 * occasion account more time than the calling functions think elapsed.
233 */
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100234static __always_inline u64 steal_account_process_time(u64 maxtime)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200235{
236#ifdef CONFIG_PARAVIRT
237 if (static_key_false(&paravirt_steal_enabled)) {
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100238 u64 steal;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200239
240 steal = paravirt_steal_clock(smp_processor_id());
241 steal -= this_rq()->prev_steal_time;
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100242 steal = min(steal, maxtime);
243 account_steal_time(steal);
244 this_rq()->prev_steal_time += steal;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200245
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100246 return steal;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200247 }
248#endif
Wanpeng Li807e5b82016-06-13 18:32:46 +0800249 return 0;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200250}
251
Frederic Weisbeckera634f932012-11-21 15:55:59 +0100252/*
Rik van Riel57430212016-07-13 16:50:01 +0200253 * Account how much elapsed time was spent in steal, irq, or softirq time.
254 */
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100255static inline u64 account_other_time(u64 max)
Rik van Riel57430212016-07-13 16:50:01 +0200256{
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100257 u64 accounted;
Rik van Riel57430212016-07-13 16:50:01 +0200258
Frederic Weisbecker2c11dba2017-11-06 16:01:27 +0100259 lockdep_assert_irqs_disabled();
Frederic Weisbecker2810f612016-09-26 02:29:18 +0200260
Rik van Riel57430212016-07-13 16:50:01 +0200261 accounted = steal_account_process_time(max);
262
263 if (accounted < max)
Frederic Weisbeckera499a5a2017-01-31 04:09:32 +0100264 accounted += irqtime_tick_accounted(max - accounted);
Rik van Riel57430212016-07-13 16:50:01 +0200265
266 return accounted;
267}
268
Stanislaw Gruszkaa1eb1412016-08-17 11:30:44 +0200269#ifdef CONFIG_64BIT
270static inline u64 read_sum_exec_runtime(struct task_struct *t)
271{
272 return t->se.sum_exec_runtime;
273}
274#else
275static u64 read_sum_exec_runtime(struct task_struct *t)
276{
277 u64 ns;
278 struct rq_flags rf;
279 struct rq *rq;
280
281 rq = task_rq_lock(t, &rf);
282 ns = t->se.sum_exec_runtime;
283 task_rq_unlock(rq, t, &rf);
284
285 return ns;
286}
287#endif
288
Rik van Riel57430212016-07-13 16:50:01 +0200289/*
Frederic Weisbeckera634f932012-11-21 15:55:59 +0100290 * Accumulate raw cputime values of dead tasks (sig->[us]time) and live
291 * tasks (sum on group iteration) belonging to @tsk's group.
292 */
293void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
294{
295 struct signal_struct *sig = tsk->signal;
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100296 u64 utime, stime;
Frederic Weisbeckera634f932012-11-21 15:55:59 +0100297 struct task_struct *t;
Rik van Riele78c3492014-08-16 13:40:10 -0400298 unsigned int seq, nextseq;
Rik van Riel9c368b52014-09-12 09:12:15 -0400299 unsigned long flags;
Frederic Weisbeckera634f932012-11-21 15:55:59 +0100300
Stanislaw Gruszkaa1eb1412016-08-17 11:30:44 +0200301 /*
302 * Update current task runtime to account pending time since last
303 * scheduler action or thread_group_cputime() call. This thread group
304 * might have other running tasks on different CPUs, but updating
305 * their runtime can affect syscall performance, so we skip account
306 * those pending times and rely only on values updated on tick or
307 * other scheduler action.
308 */
309 if (same_thread_group(current, tsk))
310 (void) task_sched_runtime(current);
311
Frederic Weisbeckera634f932012-11-21 15:55:59 +0100312 rcu_read_lock();
Rik van Riele78c3492014-08-16 13:40:10 -0400313 /* Attempt a lockless read on the first round. */
314 nextseq = 0;
315 do {
316 seq = nextseq;
Rik van Riel9c368b52014-09-12 09:12:15 -0400317 flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq);
Rik van Riele78c3492014-08-16 13:40:10 -0400318 times->utime = sig->utime;
319 times->stime = sig->stime;
320 times->sum_exec_runtime = sig->sum_sched_runtime;
321
322 for_each_thread(tsk, t) {
323 task_cputime(t, &utime, &stime);
324 times->utime += utime;
325 times->stime += stime;
Stanislaw Gruszkaa1eb1412016-08-17 11:30:44 +0200326 times->sum_exec_runtime += read_sum_exec_runtime(t);
Rik van Riele78c3492014-08-16 13:40:10 -0400327 }
328 /* If lockless access failed, take the lock. */
329 nextseq = 1;
330 } while (need_seqretry(&sig->stats_lock, seq));
Rik van Riel9c368b52014-09-12 09:12:15 -0400331 done_seqretry_irqrestore(&sig->stats_lock, seq, flags);
Frederic Weisbeckera634f932012-11-21 15:55:59 +0100332 rcu_read_unlock();
333}
334
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200335#ifdef CONFIG_IRQ_TIME_ACCOUNTING
336/*
337 * Account a tick to a process and cpustat
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100338 * @p: the process that the CPU time gets accounted to
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200339 * @user_tick: is the tick from userspace
340 * @rq: the pointer to rq
341 *
342 * Tick demultiplexing follows the order
343 * - pending hardirq update
344 * - pending softirq update
345 * - user_time
346 * - idle_time
347 * - system time
348 * - check for guest_time
349 * - else account as system_time
350 *
351 * Check for hardirq is done both for system and user time as there is
352 * no timer going off while we are on hardirq and hence we may never get an
353 * opportunity to update it solely in system time.
354 * p->stime and friends are only updated on system time and not on irq
355 * softirq as those do not count in task exec_runtime any more.
356 */
357static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
Alex Shi9dec1b62020-01-02 18:07:52 +0800358 int ticks)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200359{
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100360 u64 other, cputime = TICK_NSEC * ticks;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200361
Rik van Riel57430212016-07-13 16:50:01 +0200362 /*
363 * When returning from idle, many ticks can get accounted at
364 * once, including some ticks of steal, irq, and softirq time.
365 * Subtract those ticks from the amount of time accounted to
366 * idle, or potentially user or system time. Due to rounding,
367 * other time can exceed ticks occasionally.
368 */
Wanpeng Li03cbc732016-08-17 10:05:46 +0800369 other = account_other_time(ULONG_MAX);
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100370 if (other >= cputime)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200371 return;
Frederic Weisbecker23244a52017-01-31 04:09:37 +0100372
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100373 cputime -= other;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200374
Rik van Riel57430212016-07-13 16:50:01 +0200375 if (this_cpu_ksoftirqd() == p) {
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200376 /*
377 * ksoftirqd time do not get accounted in cpu_softirq_time.
378 * So, we have to handle it separately here.
379 * Also, p->stime needs to be updated for ksoftirqd.
380 */
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100381 account_system_index_time(p, cputime, CPUTIME_SOFTIRQ);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200382 } else if (user_tick) {
Stanislaw Gruszka40565b52016-11-15 03:06:51 +0100383 account_user_time(p, cputime);
Alex Shi9dec1b62020-01-02 18:07:52 +0800384 } else if (p == this_rq()->idle) {
Frederic Weisbecker18b43a92017-01-31 04:09:39 +0100385 account_idle_time(cputime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200386 } else if (p->flags & PF_VCPU) { /* System time or guest time */
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100387 account_guest_time(p, cputime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200388 } else {
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100389 account_system_index_time(p, cputime, CPUTIME_SYSTEM);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200390 }
391}
392
393static void irqtime_account_idle_ticks(int ticks)
394{
Alex Shi9dec1b62020-01-02 18:07:52 +0800395 irqtime_account_process_tick(current, 0, ticks);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200396}
397#else /* CONFIG_IRQ_TIME_ACCOUNTING */
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100398static inline void irqtime_account_idle_ticks(int ticks) { }
Frederic Weisbecker3f4724e2012-07-16 18:00:34 +0200399static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
Alex Shi9dec1b62020-01-02 18:07:52 +0800400 int nr_ticks) { }
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200401#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
402
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200403/*
404 * Use precise platform statistics if available:
405 */
Frederic Weisbecker8d495472019-10-03 18:17:45 +0200406#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
407
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100408# ifndef __ARCH_HAS_VTIME_TASK_SWITCH
Frederic Weisbecker8d495472019-10-03 18:17:45 +0200409void vtime_task_switch(struct task_struct *prev)
Frederic Weisbeckere3942ba2012-11-14 00:24:25 +0100410{
411 if (is_idle_task(prev))
412 vtime_account_idle(prev);
413 else
Frederic Weisbeckerf83eeb12019-10-03 18:17:44 +0200414 vtime_account_kernel(prev);
Frederic Weisbeckere3942ba2012-11-14 00:24:25 +0100415
Frederic Weisbeckerc8d7dab2017-01-05 18:11:50 +0100416 vtime_flush(prev);
Frederic Weisbeckere3942ba2012-11-14 00:24:25 +0100417 arch_vtime_task_switch(prev);
418}
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100419# endif
Frederic Weisbecker0cfdf9a2016-07-13 16:50:03 +0200420
Frederic Weisbeckera7e1a9e2012-09-08 16:14:02 +0200421/*
422 * Archs that account the whole time spent in the idle task
423 * (outside irq) as idle time can rely on this and just implement
Frederic Weisbeckerf83eeb12019-10-03 18:17:44 +0200424 * vtime_account_kernel() and vtime_account_idle(). Archs that
Frederic Weisbeckera7e1a9e2012-09-08 16:14:02 +0200425 * have other meaning of the idle time (s390 only includes the
426 * time spent by the CPU when it's in low power mode) must override
427 * vtime_account().
428 */
429#ifndef __ARCH_HAS_VTIME_ACCOUNT
Frederic Weisbecker0cfdf9a2016-07-13 16:50:03 +0200430void vtime_account_irq_enter(struct task_struct *tsk)
Frederic Weisbeckera7e1a9e2012-09-08 16:14:02 +0200431{
Frederic Weisbecker0cfdf9a2016-07-13 16:50:03 +0200432 if (!in_interrupt() && is_idle_task(tsk))
433 vtime_account_idle(tsk);
434 else
Frederic Weisbeckerf83eeb12019-10-03 18:17:44 +0200435 vtime_account_kernel(tsk);
Frederic Weisbeckera7e1a9e2012-09-08 16:14:02 +0200436}
Frederic Weisbecker0cfdf9a2016-07-13 16:50:03 +0200437EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
Frederic Weisbeckera7e1a9e2012-09-08 16:14:02 +0200438#endif /* __ARCH_HAS_VTIME_ACCOUNT */
439
Tejun Heo8157a7f2017-09-25 14:27:54 -0700440void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
441 u64 *ut, u64 *st)
442{
443 *ut = curr->utime;
444 *st = curr->stime;
445}
446
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100447void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200448{
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100449 *ut = p->utime;
450 *st = p->stime;
451}
Andrey Smetanin9eec50b2015-09-16 12:29:50 +0300452EXPORT_SYMBOL_GPL(task_cputime_adjusted);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200453
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100454void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100455{
456 struct task_cputime cputime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200457
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100458 thread_group_cputime(p, &cputime);
459
460 *ut = cputime.utime;
461 *st = cputime.stime;
462}
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100463
464#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE: */
465
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100466/*
Ingo Molnar97fb7a02018-03-03 14:01:12 +0100467 * Account a single tick of CPU time.
468 * @p: the process that the CPU time gets accounted to
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100469 * @user_tick: indicates if the tick is a user or a system tick
470 */
471void account_process_tick(struct task_struct *p, int user_tick)
472{
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100473 u64 cputime, steal;
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100474
Frederic Weisbeckere44fcb42019-10-16 04:56:54 +0200475 if (vtime_accounting_enabled_this_cpu())
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100476 return;
477
478 if (sched_clock_irqtime) {
Alex Shi9dec1b62020-01-02 18:07:52 +0800479 irqtime_account_process_tick(p, user_tick, 1);
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100480 return;
481 }
482
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100483 cputime = TICK_NSEC;
Wanpeng Li03cbc732016-08-17 10:05:46 +0800484 steal = steal_account_process_time(ULONG_MAX);
Rik van Riel57430212016-07-13 16:50:01 +0200485
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100486 if (steal >= cputime)
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100487 return;
488
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100489 cputime -= steal;
Rik van Riel57430212016-07-13 16:50:01 +0200490
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100491 if (user_tick)
Stanislaw Gruszka40565b52016-11-15 03:06:51 +0100492 account_user_time(p, cputime);
Alex Shi9dec1b62020-01-02 18:07:52 +0800493 else if ((p != this_rq()->idle) || (irq_count() != HARDIRQ_OFFSET))
Frederic Weisbeckerfb8b0492017-01-31 04:09:40 +0100494 account_system_time(p, HARDIRQ_OFFSET, cputime);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200495 else
Frederic Weisbecker18b43a92017-01-31 04:09:39 +0100496 account_idle_time(cputime);
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100497}
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200498
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100499/*
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100500 * Account multiple ticks of idle time.
501 * @ticks: number of stolen ticks
502 */
503void account_idle_ticks(unsigned long ticks)
504{
Frederic Weisbecker18b43a92017-01-31 04:09:39 +0100505 u64 cputime, steal;
Frederic Weisbecker26f2c752016-08-11 14:58:24 +0200506
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100507 if (sched_clock_irqtime) {
508 irqtime_account_idle_ticks(ticks);
509 return;
510 }
511
Frederic Weisbecker18b43a92017-01-31 04:09:39 +0100512 cputime = ticks * TICK_NSEC;
Frederic Weisbecker2b1f9672017-01-31 04:09:41 +0100513 steal = steal_account_process_time(ULONG_MAX);
Wanpeng Lif9bcf1e2016-08-11 13:36:35 +0800514
515 if (steal >= cputime)
516 return;
517
518 cputime -= steal;
519 account_idle_time(cputime);
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100520}
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200521
Frederic Weisbeckerd9a3c982013-02-20 18:54:55 +0100522/*
Stanislaw Gruszka55eaa7c2013-04-30 17:14:42 +0200523 * Perform (stime * rtime) / total, but avoid multiplication overflow by
Ingo Molnardfcb2452018-12-03 10:05:56 +0100524 * losing precision when the numbers are big.
Frederic Weisbeckerd9a3c982013-02-20 18:54:55 +0100525 */
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100526static u64 scale_stime(u64 stime, u64 rtime, u64 total)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200527{
Stanislaw Gruszka55eaa7c2013-04-30 17:14:42 +0200528 u64 scaled;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200529
Stanislaw Gruszka55eaa7c2013-04-30 17:14:42 +0200530 for (;;) {
531 /* Make sure "rtime" is the bigger of stime/rtime */
Stanislaw Gruszka84f9f3a2013-05-02 15:34:33 +0200532 if (stime > rtime)
533 swap(rtime, stime);
Stanislaw Gruszka55eaa7c2013-04-30 17:14:42 +0200534
535 /* Make sure 'total' fits in 32 bits */
536 if (total >> 32)
537 goto drop_precision;
538
539 /* Does rtime (and thus stime) fit in 32 bits? */
540 if (!(rtime >> 32))
541 break;
542
543 /* Can we just balance rtime/stime rather than dropping bits? */
544 if (stime >> 31)
545 goto drop_precision;
546
547 /* We can grow stime and shrink rtime and try to make them both fit */
548 stime <<= 1;
549 rtime >>= 1;
550 continue;
551
552drop_precision:
553 /* We drop from rtime, it has more bits than stime */
554 rtime >>= 1;
555 total >>= 1;
Frederic Weisbeckerd9a3c982013-02-20 18:54:55 +0100556 }
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200557
Stanislaw Gruszka55eaa7c2013-04-30 17:14:42 +0200558 /*
559 * Make sure gcc understands that this is a 32x32->64 multiply,
560 * followed by a 64/32->64 divide.
561 */
562 scaled = div_u64((u64) (u32) stime * (u64) (u32) rtime, (u32)total);
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100563 return scaled;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200564}
565
Frederic Weisbeckerfa092052012-11-28 17:00:57 +0100566/*
Peter Zijlstra9d7fb042015-06-30 11:30:54 +0200567 * Adjust tick based cputime random precision against scheduler runtime
568 * accounting.
Rik van Riel347abad2014-09-30 15:59:47 -0400569 *
Peter Zijlstra9d7fb042015-06-30 11:30:54 +0200570 * Tick based cputime accounting depend on random scheduling timeslices of a
571 * task to be interrupted or not by the timer. Depending on these
572 * circumstances, the number of these interrupts may be over or
573 * under-optimistic, matching the real user and system cputime with a variable
574 * precision.
575 *
576 * Fix this by scaling these tick based values against the total runtime
577 * accounted by the CFS scheduler.
578 *
579 * This code provides the following guarantees:
580 *
581 * stime + utime == rtime
582 * stime_i+1 >= stime_i, utime_i+1 >= utime_i
583 *
584 * Assuming that rtime_i+1 >= rtime_i.
Frederic Weisbeckerfa092052012-11-28 17:00:57 +0100585 */
Tejun Heocfb766d2017-09-25 08:12:04 -0700586void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
587 u64 *ut, u64 *st)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200588{
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100589 u64 rtime, stime, utime;
Peter Zijlstra9d7fb042015-06-30 11:30:54 +0200590 unsigned long flags;
Frederic Weisbeckerfa092052012-11-28 17:00:57 +0100591
Peter Zijlstra9d7fb042015-06-30 11:30:54 +0200592 /* Serialize concurrent callers such that we can honour our guarantees */
593 raw_spin_lock_irqsave(&prev->lock, flags);
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100594 rtime = curr->sum_exec_runtime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200595
Stanislaw Gruszka772c8082013-04-30 11:35:05 +0200596 /*
Peter Zijlstra9d7fb042015-06-30 11:30:54 +0200597 * This is possible under two circumstances:
598 * - rtime isn't monotonic after all (a bug);
599 * - we got reordered by the lock.
600 *
601 * In both cases this acts as a filter such that the rest of the code
602 * can assume it is monotonic regardless of anything else.
Stanislaw Gruszka772c8082013-04-30 11:35:05 +0200603 */
604 if (prev->stime + prev->utime >= rtime)
605 goto out;
606
Stanislaw Gruszka5a8e01f2013-09-04 15:16:03 +0200607 stime = curr->stime;
608 utime = curr->utime;
609
Peter Zijlstra173be9a2016-08-15 18:38:42 +0200610 /*
Ingo Molnar3b9c08a2017-07-04 11:53:40 +0200611 * If either stime or utime are 0, assume all runtime is userspace.
612 * Once a task gets some ticks, the monotonicy code at 'update:'
613 * will ensure things converge to the observed ratio.
Peter Zijlstra173be9a2016-08-15 18:38:42 +0200614 */
Ingo Molnar3b9c08a2017-07-04 11:53:40 +0200615 if (stime == 0) {
616 utime = rtime;
617 goto update;
Frederic Weisbeckerd9a3c982013-02-20 18:54:55 +0100618 }
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200619
Ingo Molnar3b9c08a2017-07-04 11:53:40 +0200620 if (utime == 0) {
621 stime = rtime;
622 goto update;
623 }
624
625 stime = scale_stime(stime, rtime, stime + utime);
626
627update:
Peter Zijlstra9d7fb042015-06-30 11:30:54 +0200628 /*
629 * Make sure stime doesn't go backwards; this preserves monotonicity
630 * for utime because rtime is monotonic.
631 *
632 * utime_i+1 = rtime_i+1 - stime_i
633 * = rtime_i+1 - (rtime_i - utime_i)
634 * = (rtime_i+1 - rtime_i) + utime_i
635 * >= utime_i
636 */
637 if (stime < prev->stime)
638 stime = prev->stime;
639 utime = rtime - stime;
640
641 /*
642 * Make sure utime doesn't go backwards; this still preserves
643 * monotonicity for stime, analogous argument to above.
644 */
645 if (utime < prev->utime) {
646 utime = prev->utime;
647 stime = rtime - utime;
648 }
649
Peter Zijlstra9d7fb042015-06-30 11:30:54 +0200650 prev->stime = stime;
651 prev->utime = utime;
Stanislaw Gruszka772c8082013-04-30 11:35:05 +0200652out:
Frederic Weisbeckerd37f761d2012-11-22 00:58:35 +0100653 *ut = prev->utime;
654 *st = prev->stime;
Peter Zijlstra9d7fb042015-06-30 11:30:54 +0200655 raw_spin_unlock_irqrestore(&prev->lock, flags);
Frederic Weisbeckerd37f761d2012-11-22 00:58:35 +0100656}
657
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100658void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
Frederic Weisbeckerd37f761d2012-11-22 00:58:35 +0100659{
660 struct task_cputime cputime = {
Frederic Weisbeckerd37f761d2012-11-22 00:58:35 +0100661 .sum_exec_runtime = p->se.sum_exec_runtime,
662 };
663
Frederic Weisbecker6fac4822012-11-13 14:20:55 +0100664 task_cputime(p, &cputime.utime, &cputime.stime);
Frederic Weisbeckerd37f761d2012-11-22 00:58:35 +0100665 cputime_adjust(&cputime, &p->prev_cputime, ut, st);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200666}
Andrey Smetanin9eec50b2015-09-16 12:29:50 +0300667EXPORT_SYMBOL_GPL(task_cputime_adjusted);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200668
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100669void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200670{
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200671 struct task_cputime cputime;
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200672
673 thread_group_cputime(p, &cputime);
Frederic Weisbeckerd37f761d2012-11-22 00:58:35 +0100674 cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
Frederic Weisbecker73fbec62012-06-16 15:57:37 +0200675}
Frederic Weisbecker9fbc42e2013-02-25 17:25:39 +0100676#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200677
678#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200679static u64 vtime_delta(struct vtime *vtime)
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200680{
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200681 unsigned long long clock;
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200682
Wanpeng Li0e4097c2017-07-09 00:40:28 -0700683 clock = sched_clock();
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200684 if (clock < vtime->starttime)
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100685 return 0;
686
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200687 return clock - vtime->starttime;
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100688}
689
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200690static u64 get_vtime_delta(struct vtime *vtime)
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100691{
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200692 u64 delta = vtime_delta(vtime);
693 u64 other;
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100694
Wanpeng Li03cbc732016-08-17 10:05:46 +0800695 /*
696 * Unlike tick based timing, vtime based timing never has lost
697 * ticks, and no need for steal time accounting to make up for
698 * lost ticks. Vtime accounts a rounded version of actual
699 * elapsed time. Limit account_other_time to prevent rounding
700 * errors from causing elapsed vtime to go negative.
701 */
Rik van Rielb58c3582016-07-13 16:50:02 +0200702 other = account_other_time(delta);
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200703 WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200704 vtime->starttime += delta;
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200705
Rik van Rielb58c3582016-07-13 16:50:02 +0200706 return delta - other;
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200707}
708
Frederic Weisbeckerf83eeb12019-10-03 18:17:44 +0200709static void vtime_account_system(struct task_struct *tsk,
710 struct vtime *vtime)
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100711{
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200712 vtime->stime += get_vtime_delta(vtime);
713 if (vtime->stime >= TICK_NSEC) {
714 account_system_time(tsk, irq_count(), vtime->stime);
715 vtime->stime = 0;
716 }
717}
718
719static void vtime_account_guest(struct task_struct *tsk,
720 struct vtime *vtime)
721{
722 vtime->gtime += get_vtime_delta(vtime);
723 if (vtime->gtime >= TICK_NSEC) {
724 account_guest_time(tsk, vtime->gtime);
725 vtime->gtime = 0;
726 }
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100727}
728
Frederic Weisbecker8d495472019-10-03 18:17:45 +0200729static void __vtime_account_kernel(struct task_struct *tsk,
730 struct vtime *vtime)
731{
732 /* We might have scheduled out from guest path */
Frederic Weisbeckere6d5bf32019-10-16 04:56:49 +0200733 if (vtime->state == VTIME_GUEST)
Frederic Weisbecker8d495472019-10-03 18:17:45 +0200734 vtime_account_guest(tsk, vtime);
735 else
736 vtime_account_system(tsk, vtime);
737}
738
Frederic Weisbeckerf83eeb12019-10-03 18:17:44 +0200739void vtime_account_kernel(struct task_struct *tsk)
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200740{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200741 struct vtime *vtime = &tsk->vtime;
742
743 if (!vtime_delta(vtime))
Rik van Rielff9a9b42016-02-10 20:08:27 -0500744 return;
745
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200746 write_seqcount_begin(&vtime->seqcount);
Frederic Weisbecker8d495472019-10-03 18:17:45 +0200747 __vtime_account_kernel(tsk, vtime);
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200748 write_seqcount_end(&vtime->seqcount);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100749}
750
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100751void vtime_user_enter(struct task_struct *tsk)
752{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200753 struct vtime *vtime = &tsk->vtime;
754
755 write_seqcount_begin(&vtime->seqcount);
Frederic Weisbeckerf83eeb12019-10-03 18:17:44 +0200756 vtime_account_system(tsk, vtime);
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200757 vtime->state = VTIME_USER;
758 write_seqcount_end(&vtime->seqcount);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100759}
760
Frederic Weisbecker1c3eda02017-06-29 19:15:07 +0200761void vtime_user_exit(struct task_struct *tsk)
762{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200763 struct vtime *vtime = &tsk->vtime;
764
765 write_seqcount_begin(&vtime->seqcount);
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200766 vtime->utime += get_vtime_delta(vtime);
767 if (vtime->utime >= TICK_NSEC) {
768 account_user_time(tsk, vtime->utime);
769 vtime->utime = 0;
770 }
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200771 vtime->state = VTIME_SYS;
772 write_seqcount_end(&vtime->seqcount);
Frederic Weisbecker1c3eda02017-06-29 19:15:07 +0200773}
774
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100775void vtime_guest_enter(struct task_struct *tsk)
776{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200777 struct vtime *vtime = &tsk->vtime;
Frederic Weisbecker5b206d42013-07-12 19:05:14 +0200778 /*
779 * The flags must be updated under the lock with
Frederic Weisbecker60a9ce52017-06-29 19:15:09 +0200780 * the vtime_starttime flush and update.
Frederic Weisbecker5b206d42013-07-12 19:05:14 +0200781 * That enforces a right ordering and update sequence
782 * synchronization against the reader (task_gtime())
783 * that can thus safely catch up with a tickless delta.
784 */
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200785 write_seqcount_begin(&vtime->seqcount);
Frederic Weisbeckerf83eeb12019-10-03 18:17:44 +0200786 vtime_account_system(tsk, vtime);
Frederic Weisbecker68e7a4d2019-09-25 23:42:42 +0200787 tsk->flags |= PF_VCPU;
Frederic Weisbeckere6d5bf32019-10-16 04:56:49 +0200788 vtime->state = VTIME_GUEST;
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200789 write_seqcount_end(&vtime->seqcount);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100790}
Frederic Weisbecker48d6a812013-07-10 02:44:35 +0200791EXPORT_SYMBOL_GPL(vtime_guest_enter);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100792
793void vtime_guest_exit(struct task_struct *tsk)
794{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200795 struct vtime *vtime = &tsk->vtime;
796
797 write_seqcount_begin(&vtime->seqcount);
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200798 vtime_account_guest(tsk, vtime);
Frederic Weisbecker68e7a4d2019-09-25 23:42:42 +0200799 tsk->flags &= ~PF_VCPU;
Frederic Weisbeckere6d5bf32019-10-16 04:56:49 +0200800 vtime->state = VTIME_SYS;
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200801 write_seqcount_end(&vtime->seqcount);
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200802}
Frederic Weisbecker48d6a812013-07-10 02:44:35 +0200803EXPORT_SYMBOL_GPL(vtime_guest_exit);
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200804
805void vtime_account_idle(struct task_struct *tsk)
806{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200807 account_idle_time(get_vtime_delta(&tsk->vtime));
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +0200808}
Frederic Weisbecker3f4724e2012-07-16 18:00:34 +0200809
Frederic Weisbecker8d495472019-10-03 18:17:45 +0200810void vtime_task_switch_generic(struct task_struct *prev)
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100811{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200812 struct vtime *vtime = &prev->vtime;
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100813
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200814 write_seqcount_begin(&vtime->seqcount);
Frederic Weisbecker14faf6f2019-10-16 04:56:48 +0200815 if (vtime->state == VTIME_IDLE)
Frederic Weisbecker8d495472019-10-03 18:17:45 +0200816 vtime_account_idle(prev);
817 else
818 __vtime_account_kernel(prev, vtime);
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200819 vtime->state = VTIME_INACTIVE;
Frederic Weisbecker802f4a82019-10-16 04:56:47 +0200820 vtime->cpu = -1;
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200821 write_seqcount_end(&vtime->seqcount);
822
823 vtime = &current->vtime;
824
825 write_seqcount_begin(&vtime->seqcount);
Frederic Weisbecker14faf6f2019-10-16 04:56:48 +0200826 if (is_idle_task(current))
827 vtime->state = VTIME_IDLE;
Frederic Weisbeckere6d5bf32019-10-16 04:56:49 +0200828 else if (current->flags & PF_VCPU)
829 vtime->state = VTIME_GUEST;
Frederic Weisbecker14faf6f2019-10-16 04:56:48 +0200830 else
831 vtime->state = VTIME_SYS;
Wanpeng Li0e4097c2017-07-09 00:40:28 -0700832 vtime->starttime = sched_clock();
Frederic Weisbecker802f4a82019-10-16 04:56:47 +0200833 vtime->cpu = smp_processor_id();
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200834 write_seqcount_end(&vtime->seqcount);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100835}
836
Frederic Weisbecker45eacc62013-05-15 22:16:32 +0200837void vtime_init_idle(struct task_struct *t, int cpu)
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100838{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200839 struct vtime *vtime = &t->vtime;
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100840 unsigned long flags;
841
Frederic Weisbeckerb7ce2272015-11-19 16:47:34 +0100842 local_irq_save(flags);
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200843 write_seqcount_begin(&vtime->seqcount);
Frederic Weisbecker14faf6f2019-10-16 04:56:48 +0200844 vtime->state = VTIME_IDLE;
Wanpeng Li0e4097c2017-07-09 00:40:28 -0700845 vtime->starttime = sched_clock();
Frederic Weisbecker802f4a82019-10-16 04:56:47 +0200846 vtime->cpu = cpu;
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200847 write_seqcount_end(&vtime->seqcount);
Frederic Weisbeckerb7ce2272015-11-19 16:47:34 +0100848 local_irq_restore(flags);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100849}
850
Frederic Weisbecker16a6d9b2017-01-31 04:09:21 +0100851u64 task_gtime(struct task_struct *t)
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100852{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200853 struct vtime *vtime = &t->vtime;
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100854 unsigned int seq;
Frederic Weisbecker16a6d9b2017-01-31 04:09:21 +0100855 u64 gtime;
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100856
Frederic Weisbeckere5925392015-11-19 16:47:33 +0100857 if (!vtime_accounting_enabled())
Hiroshi Shimamoto25411172015-11-19 16:47:28 +0100858 return t->gtime;
859
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100860 do {
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200861 seq = read_seqcount_begin(&vtime->seqcount);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100862
863 gtime = t->gtime;
Frederic Weisbeckere6d5bf32019-10-16 04:56:49 +0200864 if (vtime->state == VTIME_GUEST)
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200865 gtime += vtime->gtime + vtime_delta(vtime);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100866
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200867 } while (read_seqcount_retry(&vtime->seqcount, seq));
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100868
869 return gtime;
870}
871
872/*
873 * Fetch cputime raw values from fields of task_struct and
874 * add up the pending nohz execution time since the last
875 * cputime snapshot.
876 */
Frederic Weisbecker5613fda2017-01-31 04:09:23 +0100877void task_cputime(struct task_struct *t, u64 *utime, u64 *stime)
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100878{
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200879 struct vtime *vtime = &t->vtime;
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100880 unsigned int seq;
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200881 u64 delta;
Stanislaw Gruszka353c50e2016-11-15 03:06:52 +0100882
883 if (!vtime_accounting_enabled()) {
884 *utime = t->utime;
885 *stime = t->stime;
886 return;
887 }
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100888
889 do {
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200890 seq = read_seqcount_begin(&vtime->seqcount);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100891
Stanislaw Gruszka353c50e2016-11-15 03:06:52 +0100892 *utime = t->utime;
893 *stime = t->stime;
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100894
Frederic Weisbecker14faf6f2019-10-16 04:56:48 +0200895 /* Task is sleeping or idle, nothing to add */
896 if (vtime->state < VTIME_SYS)
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100897 continue;
898
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200899 delta = vtime_delta(vtime);
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100900
901 /*
Frederic Weisbeckere6d5bf32019-10-16 04:56:49 +0200902 * Task runs either in user (including guest) or kernel space,
903 * add pending nohz time to the right place.
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100904 */
Frederic Weisbeckere6d5bf32019-10-16 04:56:49 +0200905 if (vtime->state == VTIME_SYS)
Wanpeng Li2a42eb92017-06-29 19:15:11 +0200906 *stime += vtime->stime + delta;
Frederic Weisbeckere6d5bf32019-10-16 04:56:49 +0200907 else
908 *utime += vtime->utime + delta;
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +0200909 } while (read_seqcount_retry(&vtime->seqcount, seq));
Frederic Weisbecker6a616712012-12-16 20:00:34 +0100910}
Frederic Weisbecker64eea632019-10-25 04:03:03 +0200911
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100912static int vtime_state_fetch(struct vtime *vtime, int cpu)
Frederic Weisbecker74722bb2019-11-21 03:44:26 +0100913{
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100914 int state = READ_ONCE(vtime->state);
915
Frederic Weisbecker74722bb2019-11-21 03:44:26 +0100916 /*
917 * We raced against a context switch, fetch the
918 * kcpustat task again.
919 */
920 if (vtime->cpu != cpu && vtime->cpu != -1)
921 return -EAGAIN;
922
923 /*
924 * Two possible things here:
925 * 1) We are seeing the scheduling out task (prev) or any past one.
926 * 2) We are seeing the scheduling in task (next) but it hasn't
927 * passed though vtime_task_switch() yet so the pending
928 * cputime of the prev task may not be flushed yet.
929 *
930 * Case 1) is ok but 2) is not. So wait for a safe VTIME state.
931 */
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100932 if (state == VTIME_INACTIVE)
Frederic Weisbecker74722bb2019-11-21 03:44:26 +0100933 return -EAGAIN;
934
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100935 return state;
Frederic Weisbecker74722bb2019-11-21 03:44:26 +0100936}
937
Frederic Weisbecker5a1c9552019-11-21 03:44:25 +0100938static u64 kcpustat_user_vtime(struct vtime *vtime)
939{
940 if (vtime->state == VTIME_USER)
941 return vtime->utime + vtime_delta(vtime);
942 else if (vtime->state == VTIME_GUEST)
943 return vtime->gtime + vtime_delta(vtime);
944 return 0;
945}
946
Frederic Weisbecker64eea632019-10-25 04:03:03 +0200947static int kcpustat_field_vtime(u64 *cpustat,
Frederic Weisbecker5a1c9552019-11-21 03:44:25 +0100948 struct task_struct *tsk,
Frederic Weisbecker64eea632019-10-25 04:03:03 +0200949 enum cpu_usage_stat usage,
950 int cpu, u64 *val)
951{
Frederic Weisbecker5a1c9552019-11-21 03:44:25 +0100952 struct vtime *vtime = &tsk->vtime;
Frederic Weisbecker64eea632019-10-25 04:03:03 +0200953 unsigned int seq;
Frederic Weisbecker64eea632019-10-25 04:03:03 +0200954
955 do {
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100956 int state;
957
Frederic Weisbecker64eea632019-10-25 04:03:03 +0200958 seq = read_seqcount_begin(&vtime->seqcount);
959
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100960 state = vtime_state_fetch(vtime, cpu);
961 if (state < 0)
962 return state;
Frederic Weisbecker64eea632019-10-25 04:03:03 +0200963
964 *val = cpustat[usage];
965
Frederic Weisbecker5a1c9552019-11-21 03:44:25 +0100966 /*
967 * Nice VS unnice cputime accounting may be inaccurate if
968 * the nice value has changed since the last vtime update.
969 * But proper fix would involve interrupting target on nice
970 * updates which is a no go on nohz_full (although the scheduler
971 * may still interrupt the target if rescheduling is needed...)
972 */
973 switch (usage) {
974 case CPUTIME_SYSTEM:
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100975 if (state == VTIME_SYS)
Frederic Weisbecker5a1c9552019-11-21 03:44:25 +0100976 *val += vtime->stime + vtime_delta(vtime);
977 break;
978 case CPUTIME_USER:
979 if (task_nice(tsk) <= 0)
980 *val += kcpustat_user_vtime(vtime);
981 break;
982 case CPUTIME_NICE:
983 if (task_nice(tsk) > 0)
984 *val += kcpustat_user_vtime(vtime);
985 break;
986 case CPUTIME_GUEST:
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100987 if (state == VTIME_GUEST && task_nice(tsk) <= 0)
Frederic Weisbecker5a1c9552019-11-21 03:44:25 +0100988 *val += vtime->gtime + vtime_delta(vtime);
989 break;
990 case CPUTIME_GUEST_NICE:
Chris Wilsonf1dfdab2020-01-23 19:08:49 +0100991 if (state == VTIME_GUEST && task_nice(tsk) > 0)
Frederic Weisbecker5a1c9552019-11-21 03:44:25 +0100992 *val += vtime->gtime + vtime_delta(vtime);
993 break;
994 default:
995 break;
996 }
Frederic Weisbecker64eea632019-10-25 04:03:03 +0200997 } while (read_seqcount_retry(&vtime->seqcount, seq));
998
999 return 0;
1000}
1001
1002u64 kcpustat_field(struct kernel_cpustat *kcpustat,
1003 enum cpu_usage_stat usage, int cpu)
1004{
1005 u64 *cpustat = kcpustat->cpustat;
Borislav Petkove0d648f2020-03-27 22:43:34 +01001006 u64 val = cpustat[usage];
Frederic Weisbecker64eea632019-10-25 04:03:03 +02001007 struct rq *rq;
Frederic Weisbecker64eea632019-10-25 04:03:03 +02001008 int err;
1009
1010 if (!vtime_accounting_enabled_cpu(cpu))
Borislav Petkove0d648f2020-03-27 22:43:34 +01001011 return val;
Frederic Weisbecker64eea632019-10-25 04:03:03 +02001012
Frederic Weisbecker64eea632019-10-25 04:03:03 +02001013 rq = cpu_rq(cpu);
1014
1015 for (;;) {
1016 struct task_struct *curr;
Frederic Weisbecker64eea632019-10-25 04:03:03 +02001017
1018 rcu_read_lock();
1019 curr = rcu_dereference(rq->curr);
1020 if (WARN_ON_ONCE(!curr)) {
1021 rcu_read_unlock();
1022 return cpustat[usage];
1023 }
1024
Frederic Weisbecker5a1c9552019-11-21 03:44:25 +01001025 err = kcpustat_field_vtime(cpustat, curr, usage, cpu, &val);
Frederic Weisbecker64eea632019-10-25 04:03:03 +02001026 rcu_read_unlock();
1027
1028 if (!err)
1029 return val;
1030
1031 cpu_relax();
1032 }
1033}
1034EXPORT_SYMBOL_GPL(kcpustat_field);
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001035
1036static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
1037 const struct kernel_cpustat *src,
1038 struct task_struct *tsk, int cpu)
1039{
1040 struct vtime *vtime = &tsk->vtime;
1041 unsigned int seq;
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001042
1043 do {
1044 u64 *cpustat;
1045 u64 delta;
Chris Wilsonf1dfdab2020-01-23 19:08:49 +01001046 int state;
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001047
1048 seq = read_seqcount_begin(&vtime->seqcount);
1049
Chris Wilsonf1dfdab2020-01-23 19:08:49 +01001050 state = vtime_state_fetch(vtime, cpu);
1051 if (state < 0)
1052 return state;
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001053
1054 *dst = *src;
1055 cpustat = dst->cpustat;
1056
1057 /* Task is sleeping, dead or idle, nothing to add */
Chris Wilsonf1dfdab2020-01-23 19:08:49 +01001058 if (state < VTIME_SYS)
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001059 continue;
1060
1061 delta = vtime_delta(vtime);
1062
1063 /*
1064 * Task runs either in user (including guest) or kernel space,
1065 * add pending nohz time to the right place.
1066 */
Chris Wilsonf1dfdab2020-01-23 19:08:49 +01001067 if (state == VTIME_SYS) {
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001068 cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;
Chris Wilsonf1dfdab2020-01-23 19:08:49 +01001069 } else if (state == VTIME_USER) {
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001070 if (task_nice(tsk) > 0)
1071 cpustat[CPUTIME_NICE] += vtime->utime + delta;
1072 else
1073 cpustat[CPUTIME_USER] += vtime->utime + delta;
1074 } else {
Chris Wilsonf1dfdab2020-01-23 19:08:49 +01001075 WARN_ON_ONCE(state != VTIME_GUEST);
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001076 if (task_nice(tsk) > 0) {
1077 cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;
1078 cpustat[CPUTIME_NICE] += vtime->gtime + delta;
1079 } else {
1080 cpustat[CPUTIME_GUEST] += vtime->gtime + delta;
1081 cpustat[CPUTIME_USER] += vtime->gtime + delta;
1082 }
1083 }
1084 } while (read_seqcount_retry(&vtime->seqcount, seq));
1085
Chris Wilsonf1dfdab2020-01-23 19:08:49 +01001086 return 0;
Frederic Weisbecker74722bb2019-11-21 03:44:26 +01001087}
1088
1089void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
1090{
1091 const struct kernel_cpustat *src = &kcpustat_cpu(cpu);
1092 struct rq *rq;
1093 int err;
1094
1095 if (!vtime_accounting_enabled_cpu(cpu)) {
1096 *dst = *src;
1097 return;
1098 }
1099
1100 rq = cpu_rq(cpu);
1101
1102 for (;;) {
1103 struct task_struct *curr;
1104
1105 rcu_read_lock();
1106 curr = rcu_dereference(rq->curr);
1107 if (WARN_ON_ONCE(!curr)) {
1108 rcu_read_unlock();
1109 *dst = *src;
1110 return;
1111 }
1112
1113 err = kcpustat_cpu_fetch_vtime(dst, src, curr, cpu);
1114 rcu_read_unlock();
1115
1116 if (!err)
1117 return;
1118
1119 cpu_relax();
1120 }
1121}
1122EXPORT_SYMBOL_GPL(kcpustat_cpu_fetch);
1123
Frederic Weisbeckerabf917c2012-07-25 07:56:04 +02001124#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */