blob: 024540f97f74c3e94205826f33d3968ea765f626 [file] [log] [blame]
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -08001/*
2 * linux/kernel/time/tick-sched.c
3 *
4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
7 *
8 * No idle tick implementation for low and high resolution timers
9 *
10 * Started by: Thomas Gleixner and Ingo Molnar
11 *
Pavel Machekb10db7f2008-01-30 13:30:00 +010012 * Distribute under GPLv2.
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080013 */
14#include <linux/cpu.h>
15#include <linux/err.h>
16#include <linux/hrtimer.h>
17#include <linux/interrupt.h>
18#include <linux/kernel_stat.h>
19#include <linux/percpu.h>
20#include <linux/profile.h>
21#include <linux/sched.h>
venkatesh.pallipadi@intel.com8083e4a2008-08-04 11:59:11 -070022#include <linux/module.h>
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080023
David S. Miller9e203bc2007-02-24 22:10:13 -080024#include <asm/irq_regs.h>
25
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080026#include "tick-internal.h"
27
28/*
29 * Per cpu nohz control structure
30 */
31static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
32
33/*
34 * The time, when the last jiffy update happened. Protected by xtime_lock.
35 */
36static ktime_t last_jiffies_update;
37
Ingo Molnar289f4802007-02-16 01:28:15 -080038struct tick_sched *tick_get_tick_sched(int cpu)
39{
40 return &per_cpu(tick_cpu_sched, cpu);
41}
42
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080043/*
44 * Must be called with interrupts disabled !
45 */
46static void tick_do_update_jiffies64(ktime_t now)
47{
48 unsigned long ticks = 0;
49 ktime_t delta;
50
Ingo Molnar7a14ce12008-05-12 15:43:53 +020051 /*
52 * Do a quick check without holding xtime_lock:
53 */
54 delta = ktime_sub(now, last_jiffies_update);
55 if (delta.tv64 < tick_period.tv64)
56 return;
57
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080058 /* Reevalute with xtime_lock held */
59 write_seqlock(&xtime_lock);
60
61 delta = ktime_sub(now, last_jiffies_update);
62 if (delta.tv64 >= tick_period.tv64) {
63
64 delta = ktime_sub(delta, tick_period);
65 last_jiffies_update = ktime_add(last_jiffies_update,
66 tick_period);
67
68 /* Slow path for long timeouts */
69 if (unlikely(delta.tv64 >= tick_period.tv64)) {
70 s64 incr = ktime_to_ns(tick_period);
71
72 ticks = ktime_divns(delta, incr);
73
74 last_jiffies_update = ktime_add_ns(last_jiffies_update,
75 incr * ticks);
76 }
77 do_timer(++ticks);
Thomas Gleixner49d670f2008-09-22 18:56:01 +020078
79 /* Keep the tick_next_period variable up to date */
80 tick_next_period = ktime_add(last_jiffies_update, tick_period);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080081 }
82 write_sequnlock(&xtime_lock);
83}
84
85/*
86 * Initialize and return retrieve the jiffies update.
87 */
88static ktime_t tick_init_jiffy_update(void)
89{
90 ktime_t period;
91
92 write_seqlock(&xtime_lock);
93 /* Did we start the jiffies update yet ? */
94 if (last_jiffies_update.tv64 == 0)
95 last_jiffies_update = tick_next_period;
96 period = last_jiffies_update;
97 write_sequnlock(&xtime_lock);
98 return period;
99}
100
101/*
102 * NOHZ - aka dynamic tick functionality
103 */
104#ifdef CONFIG_NO_HZ
105/*
106 * NO HZ enabled ?
107 */
Paul E. McKenney9d2ad242012-06-24 10:15:02 -0700108int tick_nohz_enabled __read_mostly = 1;
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800109
110/*
111 * Enable / Disable tickless mode
112 */
113static int __init setup_tick_nohz(char *str)
114{
115 if (!strcmp(str, "off"))
116 tick_nohz_enabled = 0;
117 else if (!strcmp(str, "on"))
118 tick_nohz_enabled = 1;
119 else
120 return 0;
121 return 1;
122}
123
124__setup("nohz=", setup_tick_nohz);
125
126/**
127 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
128 *
129 * Called from interrupt entry when the CPU was idle
130 *
131 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
132 * must be updated. Otherwise an interrupt handler could use a stale jiffy
133 * value. We do this unconditionally on any cpu, as we don't know whether the
134 * cpu, which has the update task assigned is in a long sleep.
135 */
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200136static void tick_nohz_update_jiffies(ktime_t now)
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800137{
138 int cpu = smp_processor_id();
139 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
140 unsigned long flags;
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800141
Thomas Gleixner5df7fa12008-02-01 17:45:14 +0100142 ts->idle_waketime = now;
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800143
144 local_irq_save(flags);
145 tick_do_update_jiffies64(now);
146 local_irq_restore(flags);
Ingo Molnar02ff3752008-05-12 15:43:53 +0200147
148 touch_softlockup_watchdog();
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800149}
150
Arjan van de Ven595aac42010-05-09 08:22:45 -0700151/*
152 * Updates the per cpu time idle statistics counters
153 */
Arjan van de Ven8d63bf92010-05-09 08:24:03 -0700154static void
Peter Zijlstra8c215bd2010-07-01 09:07:17 +0200155update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
Arjan van de Ven595aac42010-05-09 08:22:45 -0700156{
157 ktime_t delta;
158
Arjan van de Ven595aac42010-05-09 08:22:45 -0700159 if (ts->idle_active) {
160 delta = ktime_sub(now, ts->idle_entrytime);
Peter Zijlstra8c215bd2010-07-01 09:07:17 +0200161 if (nr_iowait_cpu(cpu) > 0)
Arjan van de Ven0224cf42010-05-09 08:25:23 -0700162 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
Michal Hocko6beea0c2011-08-24 09:37:48 +0200163 else
164 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
Arjan van de Ven8c7b09f2010-05-09 08:23:23 -0700165 ts->idle_entrytime = now;
Arjan van de Ven595aac42010-05-09 08:22:45 -0700166 }
Arjan van de Ven8d63bf92010-05-09 08:24:03 -0700167
Arjan van de Vene0e37c22010-05-09 08:24:39 -0700168 if (last_update_time)
Arjan van de Ven8d63bf92010-05-09 08:24:03 -0700169 *last_update_time = ktime_to_us(now);
170
Arjan van de Ven595aac42010-05-09 08:22:45 -0700171}
172
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200173static void tick_nohz_stop_idle(int cpu, ktime_t now)
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100174{
175 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
176
Peter Zijlstra8c215bd2010-07-01 09:07:17 +0200177 update_ts_time_stats(cpu, ts, now, NULL);
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200178 ts->idle_active = 0;
Peter Zijlstra56c74262008-09-01 16:44:23 +0200179
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200180 sched_clock_idle_wakeup_event(0);
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100181}
182
Peter Zijlstra8c215bd2010-07-01 09:07:17 +0200183static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts)
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100184{
Michal Hocko430ee882011-12-01 17:00:22 +0100185 ktime_t now = ktime_get();
Arjan van de Ven595aac42010-05-09 08:22:45 -0700186
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100187 ts->idle_entrytime = now;
188 ts->idle_active = 1;
Peter Zijlstra56c74262008-09-01 16:44:23 +0200189 sched_clock_idle_sleep_event();
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100190 return now;
191}
192
Arjan van de Venb1f724c2010-05-09 08:22:08 -0700193/**
194 * get_cpu_idle_time_us - get the total idle time of a cpu
195 * @cpu: CPU number to query
Michal Hocko09a1d34f2011-08-24 09:39:30 +0200196 * @last_update_time: variable to store update time in. Do not update
197 * counters if NULL.
Arjan van de Venb1f724c2010-05-09 08:22:08 -0700198 *
199 * Return the cummulative idle time (since boot) for a given
Michal Hocko6beea0c2011-08-24 09:37:48 +0200200 * CPU, in microseconds.
Arjan van de Venb1f724c2010-05-09 08:22:08 -0700201 *
202 * This time is measured via accounting rather than sampling,
203 * and is as accurate as ktime_get() is.
204 *
205 * This function returns -1 if NOHZ is not enabled.
206 */
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100207u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
208{
209 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
Michal Hocko09a1d34f2011-08-24 09:39:30 +0200210 ktime_t now, idle;
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100211
venkatesh.pallipadi@intel.com8083e4a2008-08-04 11:59:11 -0700212 if (!tick_nohz_enabled)
213 return -1;
214
Michal Hocko09a1d34f2011-08-24 09:39:30 +0200215 now = ktime_get();
216 if (last_update_time) {
217 update_ts_time_stats(cpu, ts, now, last_update_time);
218 idle = ts->idle_sleeptime;
219 } else {
220 if (ts->idle_active && !nr_iowait_cpu(cpu)) {
221 ktime_t delta = ktime_sub(now, ts->idle_entrytime);
venkatesh.pallipadi@intel.com8083e4a2008-08-04 11:59:11 -0700222
Michal Hocko09a1d34f2011-08-24 09:39:30 +0200223 idle = ktime_add(ts->idle_sleeptime, delta);
224 } else {
225 idle = ts->idle_sleeptime;
226 }
227 }
228
229 return ktime_to_us(idle);
230
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100231}
venkatesh.pallipadi@intel.com8083e4a2008-08-04 11:59:11 -0700232EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100233
Michal Hocko6beea0c2011-08-24 09:37:48 +0200234/**
Arjan van de Ven0224cf42010-05-09 08:25:23 -0700235 * get_cpu_iowait_time_us - get the total iowait time of a cpu
236 * @cpu: CPU number to query
Michal Hocko09a1d34f2011-08-24 09:39:30 +0200237 * @last_update_time: variable to store update time in. Do not update
238 * counters if NULL.
Arjan van de Ven0224cf42010-05-09 08:25:23 -0700239 *
240 * Return the cummulative iowait time (since boot) for a given
241 * CPU, in microseconds.
242 *
243 * This time is measured via accounting rather than sampling,
244 * and is as accurate as ktime_get() is.
245 *
246 * This function returns -1 if NOHZ is not enabled.
247 */
248u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
249{
250 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
Michal Hocko09a1d34f2011-08-24 09:39:30 +0200251 ktime_t now, iowait;
Arjan van de Ven0224cf42010-05-09 08:25:23 -0700252
253 if (!tick_nohz_enabled)
254 return -1;
255
Michal Hocko09a1d34f2011-08-24 09:39:30 +0200256 now = ktime_get();
257 if (last_update_time) {
258 update_ts_time_stats(cpu, ts, now, last_update_time);
259 iowait = ts->iowait_sleeptime;
260 } else {
261 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
262 ktime_t delta = ktime_sub(now, ts->idle_entrytime);
Arjan van de Ven0224cf42010-05-09 08:25:23 -0700263
Michal Hocko09a1d34f2011-08-24 09:39:30 +0200264 iowait = ktime_add(ts->iowait_sleeptime, delta);
265 } else {
266 iowait = ts->iowait_sleeptime;
267 }
268 }
269
270 return ktime_to_us(iowait);
Arjan van de Ven0224cf42010-05-09 08:25:23 -0700271}
272EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
273
Frederic Weisbecker84bf1bc2011-08-01 01:25:38 +0200274static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
275 ktime_t now, int cpu)
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800276{
Frederic Weisbecker280f0672011-10-07 18:22:06 +0200277 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
Frederic Weisbecker84bf1bc2011-08-01 01:25:38 +0200278 ktime_t last_update, expires, ret = { .tv64 = 0 };
Paul E. McKenneyaa9b16302012-05-10 16:41:44 -0700279 unsigned long rcu_delta_jiffies;
Len Brown4f86d3a2007-10-03 18:58:00 -0400280 struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
Jon Hunter98962462009-08-18 12:45:10 -0500281 u64 time_delta;
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800282
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800283 /* Read jiffies and the time when jiffies were updated last */
284 do {
285 seq = read_seqbegin(&xtime_lock);
286 last_update = last_jiffies_update;
287 last_jiffies = jiffies;
Thomas Gleixner27185012009-11-12 22:12:06 +0100288 time_delta = timekeeping_max_deferment();
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800289 } while (read_seqretry(&xtime_lock, seq));
290
Paul E. McKenneyaa9b16302012-05-10 16:41:44 -0700291 if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) ||
Peter Zijlstra396e8942010-07-09 15:12:27 +0200292 arch_needs_cpu(cpu)) {
Martin Schwidefsky3c5d92a2009-09-29 14:25:16 +0200293 next_jiffies = last_jiffies + 1;
Ingo Molnar6ba9b342007-02-19 18:11:56 +0000294 delta_jiffies = 1;
Martin Schwidefsky3c5d92a2009-09-29 14:25:16 +0200295 } else {
296 /* Get the next timer wheel timer */
297 next_jiffies = get_next_timer_interrupt(last_jiffies);
298 delta_jiffies = next_jiffies - last_jiffies;
Paul E. McKenneyaa9b16302012-05-10 16:41:44 -0700299 if (rcu_delta_jiffies < delta_jiffies) {
300 next_jiffies = last_jiffies + rcu_delta_jiffies;
301 delta_jiffies = rcu_delta_jiffies;
302 }
Martin Schwidefsky3c5d92a2009-09-29 14:25:16 +0200303 }
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800304 /*
305 * Do not stop the tick, if we are only one off
306 * or if the cpu is required for rcu
307 */
Ingo Molnar6ba9b342007-02-19 18:11:56 +0000308 if (!ts->tick_stopped && delta_jiffies == 1)
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800309 goto out;
310
311 /* Schedule the tick, if we are at least one jiffie off */
312 if ((long)delta_jiffies >= 1) {
313
Woodruff, Richard00147442008-12-01 14:18:11 -0800314 /*
Woodruff, Richard00147442008-12-01 14:18:11 -0800315 * If this cpu is the one which updates jiffies, then
316 * give up the assignment and let it be taken by the
317 * cpu which runs the tick timer next, which might be
318 * this cpu as well. If we don't drop this here the
319 * jiffies might be stale and do_timer() never
Thomas Gleixner27185012009-11-12 22:12:06 +0100320 * invoked. Keep track of the fact that it was the one
321 * which had the do_timer() duty last. If this cpu is
322 * the one which had the do_timer() duty last, we
323 * limit the sleep time to the timekeeping
324 * max_deferement value which we retrieved
325 * above. Otherwise we can sleep as long as we want.
Woodruff, Richard00147442008-12-01 14:18:11 -0800326 */
Thomas Gleixner27185012009-11-12 22:12:06 +0100327 if (cpu == tick_do_timer_cpu) {
Woodruff, Richard00147442008-12-01 14:18:11 -0800328 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
Thomas Gleixner27185012009-11-12 22:12:06 +0100329 ts->do_timer_last = 1;
330 } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
331 time_delta = KTIME_MAX;
332 ts->do_timer_last = 0;
333 } else if (!ts->do_timer_last) {
334 time_delta = KTIME_MAX;
335 }
336
337 /*
Jon Hunter98962462009-08-18 12:45:10 -0500338 * calculate the expiry time for the next timer wheel
339 * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals
340 * that there is no timer pending or at least extremely
341 * far into the future (12 days for HZ=1000). In this
342 * case we set the expiry to the end of time.
343 */
344 if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) {
345 /*
346 * Calculate the time delta for the next timer event.
347 * If the time delta exceeds the maximum time delta
348 * permitted by the current clocksource then adjust
349 * the time delta accordingly to ensure the
350 * clocksource does not wrap.
351 */
352 time_delta = min_t(u64, time_delta,
353 tick_period.tv64 * delta_jiffies);
Jon Hunter98962462009-08-18 12:45:10 -0500354 }
Woodruff, Richard00147442008-12-01 14:18:11 -0800355
Thomas Gleixner27185012009-11-12 22:12:06 +0100356 if (time_delta < KTIME_MAX)
357 expires = ktime_add_ns(last_update, time_delta);
358 else
359 expires.tv64 = KTIME_MAX;
Woodruff, Richard00147442008-12-01 14:18:11 -0800360
Woodruff, Richard00147442008-12-01 14:18:11 -0800361 /* Skip reprogram of event if its not changed */
362 if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
363 goto out;
364
Frederic Weisbecker84bf1bc2011-08-01 01:25:38 +0200365 ret = expires;
366
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800367 /*
368 * nohz_stop_sched_tick can be called several times before
369 * the nohz_restart_sched_tick is called. This happens when
370 * interrupts arrive which do not cause a reschedule. In the
371 * first call we save the current tick time, so we can restart
372 * the scheduler tick in nohz_restart_sched_tick.
373 */
374 if (!ts->tick_stopped) {
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -0700375 select_nohz_load_balancer(1);
Peter Zijlstra5167e8d2012-06-22 15:52:09 +0200376 calc_load_enter_idle();
Siddha, Suresh B46cb4b72007-05-08 00:32:51 -0700377
Frederic Weisbeckerf5d411c2011-07-31 17:44:12 +0200378 ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800379 ts->tick_stopped = 1;
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800380 }
Thomas Gleixnerd3ed7822007-05-08 00:30:03 -0700381
Thomas Gleixnereaad0842007-05-29 23:47:39 +0200382 /*
Jon Hunter98962462009-08-18 12:45:10 -0500383 * If the expiration time == KTIME_MAX, then
384 * in this case we simply stop the tick timer.
Thomas Gleixnereaad0842007-05-29 23:47:39 +0200385 */
Jon Hunter98962462009-08-18 12:45:10 -0500386 if (unlikely(expires.tv64 == KTIME_MAX)) {
Thomas Gleixnereaad0842007-05-29 23:47:39 +0200387 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
388 hrtimer_cancel(&ts->sched_timer);
389 goto out;
390 }
391
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800392 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
393 hrtimer_start(&ts->sched_timer, expires,
Arun R Bharadwaj5c333862009-04-16 12:14:37 +0530394 HRTIMER_MODE_ABS_PINNED);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800395 /* Check, if the timer was already in the past */
396 if (hrtimer_active(&ts->sched_timer))
397 goto out;
Pavel Machek4c9dc642008-01-30 13:30:00 +0100398 } else if (!tick_program_event(expires, 0))
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800399 goto out;
400 /*
401 * We are past the event already. So we crossed a
402 * jiffie boundary. Update jiffies and raise the
403 * softirq.
404 */
405 tick_do_update_jiffies64(ktime_get());
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800406 }
407 raise_softirq_irqoff(TIMER_SOFTIRQ);
408out:
409 ts->next_jiffies = next_jiffies;
410 ts->last_jiffies = last_jiffies;
Len Brown4f86d3a2007-10-03 18:58:00 -0400411 ts->sleep_length = ktime_sub(dev->next_event, now);
Frederic Weisbecker84bf1bc2011-08-01 01:25:38 +0200412
413 return ret;
Frederic Weisbecker280f0672011-10-07 18:22:06 +0200414}
415
Frederic Weisbecker5b399392011-08-01 00:06:10 +0200416static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
417{
418 /*
419 * If this cpu is offline and it is the one which updates
420 * jiffies, then give up the assignment and let it be taken by
421 * the cpu which runs the tick timer next. If we don't drop
422 * this here the jiffies might be stale and do_timer() never
423 * invoked.
424 */
425 if (unlikely(!cpu_online(cpu))) {
426 if (cpu == tick_do_timer_cpu)
427 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
428 }
429
430 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
431 return false;
432
433 if (need_resched())
434 return false;
435
436 if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
437 static int ratelimit;
438
439 if (ratelimit < 10) {
440 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
441 (unsigned int) local_softirq_pending());
442 ratelimit++;
443 }
444 return false;
445 }
446
447 return true;
448}
449
Frederic Weisbecker19f5f732011-07-27 17:29:28 +0200450static void __tick_nohz_idle_enter(struct tick_sched *ts)
451{
Frederic Weisbecker84bf1bc2011-08-01 01:25:38 +0200452 ktime_t now, expires;
Frederic Weisbecker5b399392011-08-01 00:06:10 +0200453 int cpu = smp_processor_id();
Frederic Weisbecker19f5f732011-07-27 17:29:28 +0200454
Frederic Weisbecker5b399392011-08-01 00:06:10 +0200455 now = tick_nohz_start_idle(cpu, ts);
Frederic Weisbecker2ac0d982011-07-28 04:00:47 +0200456
Frederic Weisbecker5b399392011-08-01 00:06:10 +0200457 if (can_stop_idle_tick(cpu, ts)) {
458 int was_stopped = ts->tick_stopped;
459
460 ts->idle_calls++;
Frederic Weisbecker84bf1bc2011-08-01 01:25:38 +0200461
462 expires = tick_nohz_stop_sched_tick(ts, now, cpu);
463 if (expires.tv64 > 0LL) {
464 ts->idle_sleeps++;
465 ts->idle_expires = expires;
466 }
Frederic Weisbecker5b399392011-08-01 00:06:10 +0200467
468 if (!was_stopped && ts->tick_stopped)
469 ts->idle_jiffies = ts->last_jiffies;
470 }
Frederic Weisbecker280f0672011-10-07 18:22:06 +0200471}
472
473/**
474 * tick_nohz_idle_enter - stop the idle tick from the idle task
475 *
476 * When the next event is more than a tick into the future, stop the idle tick
477 * Called when we start the idle loop.
Frederic Weisbecker2bbb6812011-10-08 16:01:00 +0200478 *
Frederic Weisbecker1268fbc2011-11-17 18:48:14 +0100479 * The arch is responsible of calling:
Frederic Weisbecker2bbb6812011-10-08 16:01:00 +0200480 *
481 * - rcu_idle_enter() after its last use of RCU before the CPU is put
482 * to sleep.
483 * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
Frederic Weisbecker280f0672011-10-07 18:22:06 +0200484 */
Frederic Weisbecker1268fbc2011-11-17 18:48:14 +0100485void tick_nohz_idle_enter(void)
Frederic Weisbecker280f0672011-10-07 18:22:06 +0200486{
487 struct tick_sched *ts;
488
Frederic Weisbecker1268fbc2011-11-17 18:48:14 +0100489 WARN_ON_ONCE(irqs_disabled());
490
Linus Torvalds0db49b72012-01-06 08:33:28 -0800491 /*
492 * Update the idle state in the scheduler domain hierarchy
493 * when tick_nohz_stop_sched_tick() is called from the idle loop.
494 * State will be updated to busy during the first busy tick after
495 * exiting idle.
496 */
497 set_cpu_sd_state_idle();
498
Frederic Weisbecker1268fbc2011-11-17 18:48:14 +0100499 local_irq_disable();
500
Frederic Weisbecker280f0672011-10-07 18:22:06 +0200501 ts = &__get_cpu_var(tick_cpu_sched);
502 /*
503 * set ts->inidle unconditionally. even if the system did not
504 * switch to nohz mode the cpu frequency governers rely on the
505 * update of the idle time accounting in tick_nohz_start_idle().
506 */
507 ts->inidle = 1;
Frederic Weisbecker19f5f732011-07-27 17:29:28 +0200508 __tick_nohz_idle_enter(ts);
Frederic Weisbecker1268fbc2011-11-17 18:48:14 +0100509
510 local_irq_enable();
Frederic Weisbecker280f0672011-10-07 18:22:06 +0200511}
512
513/**
514 * tick_nohz_irq_exit - update next tick event from interrupt exit
515 *
516 * When an interrupt fires while we are idle and it doesn't cause
517 * a reschedule, it may still add, modify or delete a timer, enqueue
518 * an RCU callback, etc...
519 * So we need to re-calculate and reprogram the next tick event.
520 */
521void tick_nohz_irq_exit(void)
522{
523 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
524
525 if (!ts->inidle)
526 return;
527
Frederic Weisbecker19f5f732011-07-27 17:29:28 +0200528 __tick_nohz_idle_enter(ts);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800529}
530
531/**
Len Brown4f86d3a2007-10-03 18:58:00 -0400532 * tick_nohz_get_sleep_length - return the length of the current sleep
533 *
534 * Called from power state control code with interrupts disabled
535 */
536ktime_t tick_nohz_get_sleep_length(void)
537{
538 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
539
540 return ts->sleep_length;
541}
542
Thomas Gleixnerc34bec52008-10-17 10:04:34 +0200543static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
544{
545 hrtimer_cancel(&ts->sched_timer);
Frederic Weisbeckerf5d411c2011-07-31 17:44:12 +0200546 hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
Thomas Gleixnerc34bec52008-10-17 10:04:34 +0200547
548 while (1) {
549 /* Forward the time to expire in the future */
550 hrtimer_forward(&ts->sched_timer, now, tick_period);
551
552 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
Thomas Gleixner268a3dc2008-10-22 09:48:06 +0200553 hrtimer_start_expires(&ts->sched_timer,
Arun R Bharadwaj5c333862009-04-16 12:14:37 +0530554 HRTIMER_MODE_ABS_PINNED);
Thomas Gleixnerc34bec52008-10-17 10:04:34 +0200555 /* Check, if the timer was already in the past */
556 if (hrtimer_active(&ts->sched_timer))
557 break;
558 } else {
Thomas Gleixner268a3dc2008-10-22 09:48:06 +0200559 if (!tick_program_event(
560 hrtimer_get_expires(&ts->sched_timer), 0))
Thomas Gleixnerc34bec52008-10-17 10:04:34 +0200561 break;
562 }
Neal Cardwell6f103922012-03-27 15:09:37 -0400563 /* Reread time and update jiffies */
Thomas Gleixnerc34bec52008-10-17 10:04:34 +0200564 now = ktime_get();
Neal Cardwell6f103922012-03-27 15:09:37 -0400565 tick_do_update_jiffies64(now);
Thomas Gleixnerc34bec52008-10-17 10:04:34 +0200566 }
567}
568
Frederic Weisbecker19f5f732011-07-27 17:29:28 +0200569static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800570{
Venki Pallipadi6378ddb52008-01-30 13:30:04 +0100571 /* Update jiffies first */
Siddha, Suresh B46cb4b72007-05-08 00:32:51 -0700572 select_nohz_load_balancer(0);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800573 tick_do_update_jiffies64(now);
Peter Zijlstra5aaa0b72012-05-17 17:15:29 +0200574 update_cpu_load_nohz();
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800575
Frederic Weisbecker2ac0d982011-07-28 04:00:47 +0200576 touch_softlockup_watchdog();
577 /*
578 * Cancel the scheduled timer and restore the tick
579 */
580 ts->tick_stopped = 0;
581 ts->idle_exittime = now;
582
583 tick_nohz_restart(ts, now);
584}
585
586static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
587{
Martin Schwidefsky79741dd2008-12-31 15:11:38 +0100588#ifndef CONFIG_VIRT_CPU_ACCOUNTING
Frederic Weisbecker2ac0d982011-07-28 04:00:47 +0200589 unsigned long ticks;
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800590 /*
591 * We stopped the tick in idle. Update process times would miss the
592 * time we slept as update_process_times does only a 1 tick
593 * accounting. Enforce that this is accounted to idle !
594 */
595 ticks = jiffies - ts->idle_jiffies;
596 /*
597 * We might be one off. Do not randomly account a huge number of ticks!
598 */
Martin Schwidefsky79741dd2008-12-31 15:11:38 +0100599 if (ticks && ticks < LONG_MAX)
600 account_idle_ticks(ticks);
601#endif
Frederic Weisbecker19f5f732011-07-27 17:29:28 +0200602}
603
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800604/**
605 * tick_nohz_idle_exit - restart the idle tick from the idle task
606 *
607 * Restart the idle tick when the CPU is woken up from idle
608 * This also exit the RCU extended quiescent state. The CPU
609 * can use RCU again after this function is called.
610 */
611void tick_nohz_idle_exit(void)
612{
613 int cpu = smp_processor_id();
614 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800615 ktime_t now;
616
617 local_irq_disable();
618
619 WARN_ON_ONCE(!ts->inidle);
620
621 ts->inidle = 0;
622
623 if (ts->idle_active || ts->tick_stopped)
624 now = ktime_get();
625
626 if (ts->idle_active)
627 tick_nohz_stop_idle(cpu, now);
628
Frederic Weisbecker2ac0d982011-07-28 04:00:47 +0200629 if (ts->tick_stopped) {
Frederic Weisbecker19f5f732011-07-27 17:29:28 +0200630 tick_nohz_restart_sched_tick(ts, now);
Frederic Weisbecker2ac0d982011-07-28 04:00:47 +0200631 tick_nohz_account_idle_ticks(ts);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800632 }
633
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800634 local_irq_enable();
635}
636
637static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
638{
639 hrtimer_forward(&ts->sched_timer, now, tick_period);
Arjan van de Vencc584b22008-09-01 15:02:30 -0700640 return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800641}
642
643/*
644 * The nohz low res interrupt handler
645 */
646static void tick_nohz_handler(struct clock_event_device *dev)
647{
648 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
649 struct pt_regs *regs = get_irq_regs();
Thomas Gleixnerd3ed7822007-05-08 00:30:03 -0700650 int cpu = smp_processor_id();
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800651 ktime_t now = ktime_get();
652
653 dev->next_event.tv64 = KTIME_MAX;
654
Thomas Gleixnerd3ed7822007-05-08 00:30:03 -0700655 /*
656 * Check if the do_timer duty was dropped. We don't care about
657 * concurrency: This happens only when the cpu in charge went
658 * into a long sleep. If two cpus happen to assign themself to
659 * this duty, then the jiffies update is still serialized by
660 * xtime_lock.
661 */
Thomas Gleixner64414022008-09-22 18:46:37 +0200662 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
Thomas Gleixnerd3ed7822007-05-08 00:30:03 -0700663 tick_do_timer_cpu = cpu;
664
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800665 /* Check, if the jiffies need an update */
Thomas Gleixnerd3ed7822007-05-08 00:30:03 -0700666 if (tick_do_timer_cpu == cpu)
667 tick_do_update_jiffies64(now);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800668
669 /*
670 * When we are idle and the tick is stopped, we have to touch
671 * the watchdog as we might not schedule for a really long
672 * time. This happens on complete idle SMP systems while
673 * waiting on the login prompt. We also increment the "start
674 * of idle" jiffy stamp so the idle accounting adjustment we
675 * do when we go busy again does not account too much ticks.
676 */
677 if (ts->tick_stopped) {
678 touch_softlockup_watchdog();
679 ts->idle_jiffies++;
680 }
681
682 update_process_times(user_mode(regs));
683 profile_tick(CPU_PROFILING);
684
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800685 while (tick_nohz_reprogram(ts, now)) {
686 now = ktime_get();
687 tick_do_update_jiffies64(now);
688 }
689}
690
691/**
692 * tick_nohz_switch_to_nohz - switch to nohz mode
693 */
694static void tick_nohz_switch_to_nohz(void)
695{
696 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
697 ktime_t next;
698
699 if (!tick_nohz_enabled)
700 return;
701
702 local_irq_disable();
703 if (tick_switch_to_oneshot(tick_nohz_handler)) {
704 local_irq_enable();
705 return;
706 }
707
708 ts->nohz_mode = NOHZ_MODE_LOWRES;
709
710 /*
711 * Recycle the hrtimer in ts, so we can share the
712 * hrtimer_forward with the highres code.
713 */
714 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
715 /* Get the next period */
716 next = tick_init_jiffy_update();
717
718 for (;;) {
Arjan van de Vencc584b22008-09-01 15:02:30 -0700719 hrtimer_set_expires(&ts->sched_timer, next);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800720 if (!tick_program_event(next, 0))
721 break;
722 next = ktime_add(next, tick_period);
723 }
724 local_irq_enable();
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800725}
726
Thomas Gleixnerfb02fbc2008-10-17 10:01:23 +0200727/*
728 * When NOHZ is enabled and the tick is stopped, we need to kick the
729 * tick timer from irq_enter() so that the jiffies update is kept
730 * alive during long running softirqs. That's ugly as hell, but
731 * correctness is key even if we need to fix the offending softirq in
732 * the first place.
733 *
734 * Note, this is different to tick_nohz_restart. We just kick the
735 * timer and do not touch the other magic bits which need to be done
736 * when idle is left.
737 */
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200738static void tick_nohz_kick_tick(int cpu, ktime_t now)
Thomas Gleixnerfb02fbc2008-10-17 10:01:23 +0200739{
Thomas Gleixnerae992862008-11-10 13:20:23 +0100740#if 0
741 /* Switch back to 2.6.27 behaviour */
742
Thomas Gleixnerfb02fbc2008-10-17 10:01:23 +0200743 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200744 ktime_t delta;
Thomas Gleixnerfb02fbc2008-10-17 10:01:23 +0200745
Thomas Gleixnerc4bd8222008-10-21 20:17:35 +0200746 /*
747 * Do not touch the tick device, when the next expiry is either
748 * already reached or less/equal than the tick period.
749 */
Thomas Gleixner268a3dc2008-10-22 09:48:06 +0200750 delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
Thomas Gleixnerc4bd8222008-10-21 20:17:35 +0200751 if (delta.tv64 <= tick_period.tv64)
752 return;
753
754 tick_nohz_restart(ts, now);
Thomas Gleixnerae992862008-11-10 13:20:23 +0100755#endif
Thomas Gleixnerfb02fbc2008-10-17 10:01:23 +0200756}
757
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200758static inline void tick_check_nohz(int cpu)
759{
760 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
761 ktime_t now;
762
763 if (!ts->idle_active && !ts->tick_stopped)
764 return;
765 now = ktime_get();
766 if (ts->idle_active)
767 tick_nohz_stop_idle(cpu, now);
768 if (ts->tick_stopped) {
769 tick_nohz_update_jiffies(now);
770 tick_nohz_kick_tick(cpu, now);
771 }
772}
773
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800774#else
775
776static inline void tick_nohz_switch_to_nohz(void) { }
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200777static inline void tick_check_nohz(int cpu) { }
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800778
779#endif /* NO_HZ */
780
781/*
Thomas Gleixner719254fa2008-10-17 09:59:47 +0200782 * Called from irq_enter to notify about the possible interruption of idle()
783 */
784void tick_check_idle(int cpu)
785{
Thomas Gleixnerfb02fbc2008-10-17 10:01:23 +0200786 tick_check_oneshot_broadcast(cpu);
Martin Schwidefskyeed3b9c2009-09-29 14:25:15 +0200787 tick_check_nohz(cpu);
Thomas Gleixner719254fa2008-10-17 09:59:47 +0200788}
789
790/*
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800791 * High resolution timer specific code
792 */
793#ifdef CONFIG_HIGH_RES_TIMERS
794/*
Pavel Machek4c9dc642008-01-30 13:30:00 +0100795 * We rearm the timer until we get disabled by the idle code.
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800796 * Called with interrupts disabled and timer->base->cpu_base->lock held.
797 */
798static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
799{
800 struct tick_sched *ts =
801 container_of(timer, struct tick_sched, sched_timer);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800802 struct pt_regs *regs = get_irq_regs();
803 ktime_t now = ktime_get();
Thomas Gleixnerd3ed7822007-05-08 00:30:03 -0700804 int cpu = smp_processor_id();
805
806#ifdef CONFIG_NO_HZ
807 /*
808 * Check if the do_timer duty was dropped. We don't care about
809 * concurrency: This happens only when the cpu in charge went
810 * into a long sleep. If two cpus happen to assign themself to
811 * this duty, then the jiffies update is still serialized by
812 * xtime_lock.
813 */
Thomas Gleixner64414022008-09-22 18:46:37 +0200814 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
Thomas Gleixnerd3ed7822007-05-08 00:30:03 -0700815 tick_do_timer_cpu = cpu;
816#endif
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800817
818 /* Check, if the jiffies need an update */
Thomas Gleixnerd3ed7822007-05-08 00:30:03 -0700819 if (tick_do_timer_cpu == cpu)
820 tick_do_update_jiffies64(now);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800821
822 /*
823 * Do not call, when we are not in irq context and have
824 * no valid regs pointer
825 */
826 if (regs) {
827 /*
828 * When we are idle and the tick is stopped, we have to touch
829 * the watchdog as we might not schedule for a really long
830 * time. This happens on complete idle SMP systems while
831 * waiting on the login prompt. We also increment the "start of
832 * idle" jiffy stamp so the idle accounting adjustment we do
833 * when we go busy again does not account too much ticks.
834 */
835 if (ts->tick_stopped) {
836 touch_softlockup_watchdog();
Frederic Weisbecker2ac0d982011-07-28 04:00:47 +0200837 if (idle_cpu(cpu))
838 ts->idle_jiffies++;
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800839 }
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800840 update_process_times(user_mode(regs));
841 profile_tick(CPU_PROFILING);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800842 }
843
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800844 hrtimer_forward(timer, now, tick_period);
845
846 return HRTIMER_RESTART;
847}
848
Mike Galbraith5307c952012-05-08 12:20:58 +0200849static int sched_skew_tick;
850
Thomas Gleixner62cf20b2012-05-25 14:08:57 +0200851static int __init skew_tick(char *str)
852{
853 get_option(&str, &sched_skew_tick);
854
855 return 0;
856}
857early_param("skew_tick", skew_tick);
858
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800859/**
860 * tick_setup_sched_timer - setup the tick emulation timer
861 */
862void tick_setup_sched_timer(void)
863{
864 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
865 ktime_t now = ktime_get();
866
867 /*
868 * Emulate tick processing via per-CPU hrtimers:
869 */
870 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
871 ts->sched_timer.function = tick_sched_timer;
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800872
john stultz37045402007-07-21 04:37:35 -0700873 /* Get the next period (per cpu) */
Arjan van de Vencc584b22008-09-01 15:02:30 -0700874 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800875
Mike Galbraith5307c952012-05-08 12:20:58 +0200876 /* Offset the tick to avert xtime_lock contention. */
877 if (sched_skew_tick) {
878 u64 offset = ktime_to_ns(tick_period) >> 1;
879 do_div(offset, num_possible_cpus());
880 offset *= smp_processor_id();
881 hrtimer_add_expires_ns(&ts->sched_timer, offset);
882 }
883
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800884 for (;;) {
885 hrtimer_forward(&ts->sched_timer, now, tick_period);
Arun R Bharadwaj5c333862009-04-16 12:14:37 +0530886 hrtimer_start_expires(&ts->sched_timer,
887 HRTIMER_MODE_ABS_PINNED);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800888 /* Check, if the timer was already in the past */
889 if (hrtimer_active(&ts->sched_timer))
890 break;
891 now = ktime_get();
892 }
893
894#ifdef CONFIG_NO_HZ
Heiko Carstens29c158e2011-08-23 13:20:46 +0200895 if (tick_nohz_enabled)
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800896 ts->nohz_mode = NOHZ_MODE_HIGHRES;
897#endif
898}
Miao Xie3c4fbe52008-08-20 16:37:38 -0700899#endif /* HIGH_RES_TIMERS */
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800900
Miao Xie3c4fbe52008-08-20 16:37:38 -0700901#if defined CONFIG_NO_HZ || defined CONFIG_HIGH_RES_TIMERS
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800902void tick_cancel_sched_timer(int cpu)
903{
904 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
905
Miao Xie3c4fbe52008-08-20 16:37:38 -0700906# ifdef CONFIG_HIGH_RES_TIMERS
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800907 if (ts->sched_timer.base)
908 hrtimer_cancel(&ts->sched_timer);
Miao Xie3c4fbe52008-08-20 16:37:38 -0700909# endif
Karsten Wiesea7901762008-03-04 14:59:55 -0800910
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800911 ts->nohz_mode = NOHZ_MODE_INACTIVE;
912}
Miao Xie3c4fbe52008-08-20 16:37:38 -0700913#endif
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800914
915/**
916 * Async notification about clocksource changes
917 */
918void tick_clock_notify(void)
919{
920 int cpu;
921
922 for_each_possible_cpu(cpu)
923 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
924}
925
926/*
927 * Async notification about clock event changes
928 */
929void tick_oneshot_notify(void)
930{
931 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
932
933 set_bit(0, &ts->check_clocks);
934}
935
936/**
937 * Check, if a change happened, which makes oneshot possible.
938 *
939 * Called cyclic from the hrtimer softirq (driven by the timer
940 * softirq) allow_nohz signals, that we can switch into low-res nohz
941 * mode, because high resolution timers are disabled (either compile
942 * or runtime).
943 */
944int tick_check_oneshot_change(int allow_nohz)
945{
946 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
947
948 if (!test_and_clear_bit(0, &ts->check_clocks))
949 return 0;
950
951 if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
952 return 0;
953
Li Zefancf4fc6c2008-02-08 04:19:24 -0800954 if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800955 return 0;
956
957 if (!allow_nohz)
958 return 1;
959
960 tick_nohz_switch_to_nohz();
961 return 0;
962}