blob: ee305c8d4e18eb038a54f0f9aea8eb2f353b1f44 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/timer.c
3 *
john stultz85240702007-05-08 00:27:59 -07004 * Kernel internal timers, basic process system calls
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * Copyright (C) 1991, 1992 Linus Torvalds
7 *
8 * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better.
9 *
10 * 1997-09-10 Updated NTP code according to technical memorandum Jan '96
11 * "A Kernel Model for Precision Timekeeping" by Dave Mills
12 * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
13 * serialize accesses to xtime/lost_ticks).
14 * Copyright (C) 1998 Andrea Arcangeli
15 * 1999-03-10 Improved NTP compatibility by Ulrich Windl
16 * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love
17 * 2000-10-05 Implemented scalable SMP per-CPU timer handling.
18 * Copyright (C) 2000, 2001, 2002 Ingo Molnar
19 * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar
20 */
21
22#include <linux/kernel_stat.h>
23#include <linux/module.h>
24#include <linux/interrupt.h>
25#include <linux/percpu.h>
26#include <linux/init.h>
27#include <linux/mm.h>
28#include <linux/swap.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070029#include <linux/pid_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/notifier.h>
31#include <linux/thread_info.h>
32#include <linux/time.h>
33#include <linux/jiffies.h>
34#include <linux/posix-timers.h>
35#include <linux/cpu.h>
36#include <linux/syscalls.h>
Adrian Bunk97a41e22006-01-08 01:02:17 -080037#include <linux/delay.h>
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080038#include <linux/tick.h>
Ingo Molnar82f67cd2007-02-16 01:28:13 -080039#include <linux/kallsyms.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020040#include <linux/perf_event.h>
Arun R Bharadwajeea08f32009-04-16 12:16:41 +053041#include <linux/sched.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090042#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
44#include <asm/uaccess.h>
45#include <asm/unistd.h>
46#include <asm/div64.h>
47#include <asm/timex.h>
48#include <asm/io.h>
49
Xiao Guangrong2b022e32009-08-10 10:48:59 +080050#define CREATE_TRACE_POINTS
51#include <trace/events/timer.h>
52
Thomas Gleixnerecea8d12005-10-30 15:03:00 -080053u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
54
55EXPORT_SYMBOL(jiffies_64);
56
Linus Torvalds1da177e2005-04-16 15:20:36 -070057/*
58 * per-CPU timer vector definitions:
59 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070060#define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)
61#define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)
62#define TVN_SIZE (1 << TVN_BITS)
63#define TVR_SIZE (1 << TVR_BITS)
64#define TVN_MASK (TVN_SIZE - 1)
65#define TVR_MASK (TVR_SIZE - 1)
66
Pavel Macheka6fa8e52008-01-30 13:30:00 +010067struct tvec {
Linus Torvalds1da177e2005-04-16 15:20:36 -070068 struct list_head vec[TVN_SIZE];
Pavel Macheka6fa8e52008-01-30 13:30:00 +010069};
Linus Torvalds1da177e2005-04-16 15:20:36 -070070
Pavel Macheka6fa8e52008-01-30 13:30:00 +010071struct tvec_root {
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 struct list_head vec[TVR_SIZE];
Pavel Macheka6fa8e52008-01-30 13:30:00 +010073};
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
Pavel Macheka6fa8e52008-01-30 13:30:00 +010075struct tvec_base {
Oleg Nesterov3691c512006-03-31 02:30:30 -080076 spinlock_t lock;
77 struct timer_list *running_timer;
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 unsigned long timer_jiffies;
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +020079 unsigned long next_timer;
Pavel Macheka6fa8e52008-01-30 13:30:00 +010080 struct tvec_root tv1;
81 struct tvec tv2;
82 struct tvec tv3;
83 struct tvec tv4;
84 struct tvec tv5;
Venki Pallipadi6e453a62007-05-08 00:27:44 -070085} ____cacheline_aligned;
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
Pavel Macheka6fa8e52008-01-30 13:30:00 +010087struct tvec_base boot_tvec_bases;
Oleg Nesterov3691c512006-03-31 02:30:30 -080088EXPORT_SYMBOL(boot_tvec_bases);
Pavel Macheka6fa8e52008-01-30 13:30:00 +010089static DEFINE_PER_CPU(struct tvec_base *, tvec_bases) = &boot_tvec_bases;
Linus Torvalds1da177e2005-04-16 15:20:36 -070090
Venki Pallipadi6e453a62007-05-08 00:27:44 -070091/*
Pavel Macheka6fa8e52008-01-30 13:30:00 +010092 * Note that all tvec_bases are 2 byte aligned and lower bit of
Venki Pallipadi6e453a62007-05-08 00:27:44 -070093 * base in timer_list is guaranteed to be zero. Use the LSB for
94 * the new flag to indicate whether the timer is deferrable
95 */
96#define TBASE_DEFERRABLE_FLAG (0x1)
97
98/* Functions below help us manage 'deferrable' flag */
Pavel Macheka6fa8e52008-01-30 13:30:00 +010099static inline unsigned int tbase_get_deferrable(struct tvec_base *base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700100{
akpm@linux-foundation.orge9910842007-05-10 03:16:01 -0700101 return ((unsigned int)(unsigned long)base & TBASE_DEFERRABLE_FLAG);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700102}
103
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100104static inline struct tvec_base *tbase_get_base(struct tvec_base *base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700105{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100106 return ((struct tvec_base *)((unsigned long)base & ~TBASE_DEFERRABLE_FLAG));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700107}
108
109static inline void timer_set_deferrable(struct timer_list *timer)
110{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100111 timer->base = ((struct tvec_base *)((unsigned long)(timer->base) |
Thomas Gleixner68194572007-07-19 01:49:16 -0700112 TBASE_DEFERRABLE_FLAG));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700113}
114
115static inline void
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100116timer_set_base(struct timer_list *timer, struct tvec_base *new_base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700117{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100118 timer->base = (struct tvec_base *)((unsigned long)(new_base) |
Thomas Gleixner68194572007-07-19 01:49:16 -0700119 tbase_get_deferrable(timer->base));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700120}
121
Alan Stern9c133c42008-11-06 08:42:48 +0100122static unsigned long round_jiffies_common(unsigned long j, int cpu,
123 bool force_up)
124{
125 int rem;
126 unsigned long original = j;
127
128 /*
129 * We don't want all cpus firing their timers at once hitting the
130 * same lock or cachelines, so we skew each extra cpu with an extra
131 * 3 jiffies. This 3 jiffies came originally from the mm/ code which
132 * already did this.
133 * The skew is done by adding 3*cpunr, then round, then subtract this
134 * extra offset again.
135 */
136 j += cpu * 3;
137
138 rem = j % HZ;
139
140 /*
141 * If the target jiffie is just after a whole second (which can happen
142 * due to delays of the timer irq, long irq off times etc etc) then
143 * we should round down to the whole second, not up. Use 1/4th second
144 * as cutoff for this rounding as an extreme upper bound for this.
145 * But never round down if @force_up is set.
146 */
147 if (rem < HZ/4 && !force_up) /* round down */
148 j = j - rem;
149 else /* round up */
150 j = j - rem + HZ;
151
152 /* now that we have rounded, subtract the extra skew again */
153 j -= cpu * 3;
154
155 if (j <= jiffies) /* rounding ate our timeout entirely; */
156 return original;
157 return j;
158}
159
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800160/**
161 * __round_jiffies - function to round jiffies to a full second
162 * @j: the time in (absolute) jiffies that should be rounded
163 * @cpu: the processor number on which the timeout will happen
164 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800165 * __round_jiffies() rounds an absolute time in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800166 * up or down to (approximately) full seconds. This is useful for timers
167 * for which the exact time they fire does not matter too much, as long as
168 * they fire approximately every X seconds.
169 *
170 * By rounding these timers to whole seconds, all such timers will fire
171 * at the same time, rather than at various times spread out. The goal
172 * of this is to have the CPU wake up less, which saves power.
173 *
174 * The exact rounding is skewed for each processor to avoid all
175 * processors firing at the exact same time, which could lead
176 * to lock contention or spurious cache line bouncing.
177 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800178 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800179 */
180unsigned long __round_jiffies(unsigned long j, int cpu)
181{
Alan Stern9c133c42008-11-06 08:42:48 +0100182 return round_jiffies_common(j, cpu, false);
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800183}
184EXPORT_SYMBOL_GPL(__round_jiffies);
185
186/**
187 * __round_jiffies_relative - function to round jiffies to a full second
188 * @j: the time in (relative) jiffies that should be rounded
189 * @cpu: the processor number on which the timeout will happen
190 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800191 * __round_jiffies_relative() rounds a time delta in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800192 * up or down to (approximately) full seconds. This is useful for timers
193 * for which the exact time they fire does not matter too much, as long as
194 * they fire approximately every X seconds.
195 *
196 * By rounding these timers to whole seconds, all such timers will fire
197 * at the same time, rather than at various times spread out. The goal
198 * of this is to have the CPU wake up less, which saves power.
199 *
200 * The exact rounding is skewed for each processor to avoid all
201 * processors firing at the exact same time, which could lead
202 * to lock contention or spurious cache line bouncing.
203 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800204 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800205 */
206unsigned long __round_jiffies_relative(unsigned long j, int cpu)
207{
Alan Stern9c133c42008-11-06 08:42:48 +0100208 unsigned long j0 = jiffies;
209
210 /* Use j0 because jiffies might change while we run */
211 return round_jiffies_common(j + j0, cpu, false) - j0;
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800212}
213EXPORT_SYMBOL_GPL(__round_jiffies_relative);
214
215/**
216 * round_jiffies - function to round jiffies to a full second
217 * @j: the time in (absolute) jiffies that should be rounded
218 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800219 * round_jiffies() rounds an absolute time in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800220 * up or down to (approximately) full seconds. This is useful for timers
221 * for which the exact time they fire does not matter too much, as long as
222 * they fire approximately every X seconds.
223 *
224 * By rounding these timers to whole seconds, all such timers will fire
225 * at the same time, rather than at various times spread out. The goal
226 * of this is to have the CPU wake up less, which saves power.
227 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800228 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800229 */
230unsigned long round_jiffies(unsigned long j)
231{
Alan Stern9c133c42008-11-06 08:42:48 +0100232 return round_jiffies_common(j, raw_smp_processor_id(), false);
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800233}
234EXPORT_SYMBOL_GPL(round_jiffies);
235
236/**
237 * round_jiffies_relative - function to round jiffies to a full second
238 * @j: the time in (relative) jiffies that should be rounded
239 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800240 * round_jiffies_relative() rounds a time delta in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800241 * up or down to (approximately) full seconds. This is useful for timers
242 * for which the exact time they fire does not matter too much, as long as
243 * they fire approximately every X seconds.
244 *
245 * By rounding these timers to whole seconds, all such timers will fire
246 * at the same time, rather than at various times spread out. The goal
247 * of this is to have the CPU wake up less, which saves power.
248 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800249 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800250 */
251unsigned long round_jiffies_relative(unsigned long j)
252{
253 return __round_jiffies_relative(j, raw_smp_processor_id());
254}
255EXPORT_SYMBOL_GPL(round_jiffies_relative);
256
Alan Stern9c133c42008-11-06 08:42:48 +0100257/**
258 * __round_jiffies_up - function to round jiffies up to a full second
259 * @j: the time in (absolute) jiffies that should be rounded
260 * @cpu: the processor number on which the timeout will happen
261 *
262 * This is the same as __round_jiffies() except that it will never
263 * round down. This is useful for timeouts for which the exact time
264 * of firing does not matter too much, as long as they don't fire too
265 * early.
266 */
267unsigned long __round_jiffies_up(unsigned long j, int cpu)
268{
269 return round_jiffies_common(j, cpu, true);
270}
271EXPORT_SYMBOL_GPL(__round_jiffies_up);
272
273/**
274 * __round_jiffies_up_relative - function to round jiffies up to a full second
275 * @j: the time in (relative) jiffies that should be rounded
276 * @cpu: the processor number on which the timeout will happen
277 *
278 * This is the same as __round_jiffies_relative() except that it will never
279 * round down. This is useful for timeouts for which the exact time
280 * of firing does not matter too much, as long as they don't fire too
281 * early.
282 */
283unsigned long __round_jiffies_up_relative(unsigned long j, int cpu)
284{
285 unsigned long j0 = jiffies;
286
287 /* Use j0 because jiffies might change while we run */
288 return round_jiffies_common(j + j0, cpu, true) - j0;
289}
290EXPORT_SYMBOL_GPL(__round_jiffies_up_relative);
291
292/**
293 * round_jiffies_up - function to round jiffies up to a full second
294 * @j: the time in (absolute) jiffies that should be rounded
295 *
296 * This is the same as round_jiffies() except that it will never
297 * round down. This is useful for timeouts for which the exact time
298 * of firing does not matter too much, as long as they don't fire too
299 * early.
300 */
301unsigned long round_jiffies_up(unsigned long j)
302{
303 return round_jiffies_common(j, raw_smp_processor_id(), true);
304}
305EXPORT_SYMBOL_GPL(round_jiffies_up);
306
307/**
308 * round_jiffies_up_relative - function to round jiffies up to a full second
309 * @j: the time in (relative) jiffies that should be rounded
310 *
311 * This is the same as round_jiffies_relative() except that it will never
312 * round down. This is useful for timeouts for which the exact time
313 * of firing does not matter too much, as long as they don't fire too
314 * early.
315 */
316unsigned long round_jiffies_up_relative(unsigned long j)
317{
318 return __round_jiffies_up_relative(j, raw_smp_processor_id());
319}
320EXPORT_SYMBOL_GPL(round_jiffies_up_relative);
321
Arjan van de Ven3bbb9ec2010-03-11 14:04:36 -0800322/**
323 * set_timer_slack - set the allowed slack for a timer
324 * @slack_hz: the amount of time (in jiffies) allowed for rounding
325 *
326 * Set the amount of time, in jiffies, that a certain timer has
327 * in terms of slack. By setting this value, the timer subsystem
328 * will schedule the actual timer somewhere between
329 * the time mod_timer() asks for, and that time plus the slack.
330 *
331 * By setting the slack to -1, a percentage of the delay is used
332 * instead.
333 */
334void set_timer_slack(struct timer_list *timer, int slack_hz)
335{
336 timer->slack = slack_hz;
337}
338EXPORT_SYMBOL_GPL(set_timer_slack);
339
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800340
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100341static inline void set_running_timer(struct tvec_base *base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 struct timer_list *timer)
343{
344#ifdef CONFIG_SMP
Oleg Nesterov3691c512006-03-31 02:30:30 -0800345 base->running_timer = timer;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346#endif
347}
348
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100349static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350{
351 unsigned long expires = timer->expires;
352 unsigned long idx = expires - base->timer_jiffies;
353 struct list_head *vec;
354
355 if (idx < TVR_SIZE) {
356 int i = expires & TVR_MASK;
357 vec = base->tv1.vec + i;
358 } else if (idx < 1 << (TVR_BITS + TVN_BITS)) {
359 int i = (expires >> TVR_BITS) & TVN_MASK;
360 vec = base->tv2.vec + i;
361 } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) {
362 int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;
363 vec = base->tv3.vec + i;
364 } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) {
365 int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK;
366 vec = base->tv4.vec + i;
367 } else if ((signed long) idx < 0) {
368 /*
369 * Can happen if you add a timer with expires == jiffies,
370 * or you set a timer to go off in the past
371 */
372 vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);
373 } else {
374 int i;
375 /* If the timeout is larger than 0xffffffff on 64-bit
376 * architectures then we use the maximum timeout:
377 */
378 if (idx > 0xffffffffUL) {
379 idx = 0xffffffffUL;
380 expires = idx + base->timer_jiffies;
381 }
382 i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;
383 vec = base->tv5.vec + i;
384 }
385 /*
386 * Timers are FIFO:
387 */
388 list_add_tail(&timer->entry, vec);
389}
390
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800391#ifdef CONFIG_TIMER_STATS
392void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr)
393{
394 if (timer->start_site)
395 return;
396
397 timer->start_site = addr;
398 memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);
399 timer->start_pid = current->pid;
400}
Venki Pallipadic5c061b82007-07-15 23:40:30 -0700401
402static void timer_stats_account_timer(struct timer_list *timer)
403{
404 unsigned int flag = 0;
405
Heiko Carstens507e1232009-06-23 17:38:15 +0200406 if (likely(!timer->start_site))
407 return;
Venki Pallipadic5c061b82007-07-15 23:40:30 -0700408 if (unlikely(tbase_get_deferrable(timer->base)))
409 flag |= TIMER_STATS_FLAG_DEFERRABLE;
410
411 timer_stats_update_stats(timer, timer->start_pid, timer->start_site,
412 timer->function, timer->start_comm, flag);
413}
414
415#else
416static void timer_stats_account_timer(struct timer_list *timer) {}
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800417#endif
418
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700419#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
420
421static struct debug_obj_descr timer_debug_descr;
422
423/*
424 * fixup_init is called when:
425 * - an active object is initialized
426 */
427static int timer_fixup_init(void *addr, enum debug_obj_state state)
428{
429 struct timer_list *timer = addr;
430
431 switch (state) {
432 case ODEBUG_STATE_ACTIVE:
433 del_timer_sync(timer);
434 debug_object_init(timer, &timer_debug_descr);
435 return 1;
436 default:
437 return 0;
438 }
439}
440
441/*
442 * fixup_activate is called when:
443 * - an active object is activated
444 * - an unknown object is activated (might be a statically initialized object)
445 */
446static int timer_fixup_activate(void *addr, enum debug_obj_state state)
447{
448 struct timer_list *timer = addr;
449
450 switch (state) {
451
452 case ODEBUG_STATE_NOTAVAILABLE:
453 /*
454 * This is not really a fixup. The timer was
455 * statically initialized. We just make sure that it
456 * is tracked in the object tracker.
457 */
458 if (timer->entry.next == NULL &&
459 timer->entry.prev == TIMER_ENTRY_STATIC) {
460 debug_object_init(timer, &timer_debug_descr);
461 debug_object_activate(timer, &timer_debug_descr);
462 return 0;
463 } else {
464 WARN_ON_ONCE(1);
465 }
466 return 0;
467
468 case ODEBUG_STATE_ACTIVE:
469 WARN_ON(1);
470
471 default:
472 return 0;
473 }
474}
475
476/*
477 * fixup_free is called when:
478 * - an active object is freed
479 */
480static int timer_fixup_free(void *addr, enum debug_obj_state state)
481{
482 struct timer_list *timer = addr;
483
484 switch (state) {
485 case ODEBUG_STATE_ACTIVE:
486 del_timer_sync(timer);
487 debug_object_free(timer, &timer_debug_descr);
488 return 1;
489 default:
490 return 0;
491 }
492}
493
494static struct debug_obj_descr timer_debug_descr = {
495 .name = "timer_list",
496 .fixup_init = timer_fixup_init,
497 .fixup_activate = timer_fixup_activate,
498 .fixup_free = timer_fixup_free,
499};
500
501static inline void debug_timer_init(struct timer_list *timer)
502{
503 debug_object_init(timer, &timer_debug_descr);
504}
505
506static inline void debug_timer_activate(struct timer_list *timer)
507{
508 debug_object_activate(timer, &timer_debug_descr);
509}
510
511static inline void debug_timer_deactivate(struct timer_list *timer)
512{
513 debug_object_deactivate(timer, &timer_debug_descr);
514}
515
516static inline void debug_timer_free(struct timer_list *timer)
517{
518 debug_object_free(timer, &timer_debug_descr);
519}
520
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100521static void __init_timer(struct timer_list *timer,
522 const char *name,
523 struct lock_class_key *key);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700524
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100525void init_timer_on_stack_key(struct timer_list *timer,
526 const char *name,
527 struct lock_class_key *key)
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700528{
529 debug_object_init_on_stack(timer, &timer_debug_descr);
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100530 __init_timer(timer, name, key);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700531}
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100532EXPORT_SYMBOL_GPL(init_timer_on_stack_key);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700533
534void destroy_timer_on_stack(struct timer_list *timer)
535{
536 debug_object_free(timer, &timer_debug_descr);
537}
538EXPORT_SYMBOL_GPL(destroy_timer_on_stack);
539
540#else
541static inline void debug_timer_init(struct timer_list *timer) { }
542static inline void debug_timer_activate(struct timer_list *timer) { }
543static inline void debug_timer_deactivate(struct timer_list *timer) { }
544#endif
545
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800546static inline void debug_init(struct timer_list *timer)
547{
548 debug_timer_init(timer);
549 trace_timer_init(timer);
550}
551
552static inline void
553debug_activate(struct timer_list *timer, unsigned long expires)
554{
555 debug_timer_activate(timer);
556 trace_timer_start(timer, expires);
557}
558
559static inline void debug_deactivate(struct timer_list *timer)
560{
561 debug_timer_deactivate(timer);
562 trace_timer_cancel(timer);
563}
564
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100565static void __init_timer(struct timer_list *timer,
566 const char *name,
567 struct lock_class_key *key)
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700568{
569 timer->entry.next = NULL;
570 timer->base = __raw_get_cpu_var(tvec_bases);
Arjan van de Ven3bbb9ec2010-03-11 14:04:36 -0800571 timer->slack = -1;
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700572#ifdef CONFIG_TIMER_STATS
573 timer->start_site = NULL;
574 timer->start_pid = -1;
575 memset(timer->start_comm, 0, TASK_COMM_LEN);
576#endif
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100577 lockdep_init_map(&timer->lockdep_map, name, key, 0);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700578}
579
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700580/**
Randy Dunlap633fe792009-04-01 17:47:23 -0700581 * init_timer_key - initialize a timer
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700582 * @timer: the timer to be initialized
Randy Dunlap633fe792009-04-01 17:47:23 -0700583 * @name: name of the timer
584 * @key: lockdep class key of the fake lock used for tracking timer
585 * sync lock dependencies
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700586 *
Randy Dunlap633fe792009-04-01 17:47:23 -0700587 * init_timer_key() must be done to a timer prior calling *any* of the
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700588 * other timer functions.
589 */
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100590void init_timer_key(struct timer_list *timer,
591 const char *name,
592 struct lock_class_key *key)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700593{
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800594 debug_init(timer);
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100595 __init_timer(timer, name, key);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700596}
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100597EXPORT_SYMBOL(init_timer_key);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700598
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100599void init_timer_deferrable_key(struct timer_list *timer,
600 const char *name,
601 struct lock_class_key *key)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700602{
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100603 init_timer_key(timer, name, key);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700604 timer_set_deferrable(timer);
605}
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100606EXPORT_SYMBOL(init_timer_deferrable_key);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700607
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700608static inline void detach_timer(struct timer_list *timer,
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800609 int clear_pending)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700610{
611 struct list_head *entry = &timer->entry;
612
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800613 debug_deactivate(timer);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700614
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700615 __list_del(entry->prev, entry->next);
616 if (clear_pending)
617 entry->next = NULL;
618 entry->prev = LIST_POISON2;
619}
620
621/*
Oleg Nesterov3691c512006-03-31 02:30:30 -0800622 * We are using hashed locking: holding per_cpu(tvec_bases).lock
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700623 * means that all timers which are tied to this base via timer->base are
624 * locked, and the base itself is locked too.
625 *
626 * So __run_timers/migrate_timers can safely modify all timers which could
627 * be found on ->tvX lists.
628 *
629 * When the timer's base is locked, and the timer removed from list, it is
630 * possible to set timer->base = NULL and drop the lock: the timer remains
631 * locked.
632 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100633static struct tvec_base *lock_timer_base(struct timer_list *timer,
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700634 unsigned long *flags)
Josh Triplett89e7e3742006-09-29 01:59:36 -0700635 __acquires(timer->base->lock)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700636{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100637 struct tvec_base *base;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700638
639 for (;;) {
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100640 struct tvec_base *prelock_base = timer->base;
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700641 base = tbase_get_base(prelock_base);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700642 if (likely(base != NULL)) {
643 spin_lock_irqsave(&base->lock, *flags);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700644 if (likely(prelock_base == timer->base))
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700645 return base;
646 /* The timer has migrated to another CPU */
647 spin_unlock_irqrestore(&base->lock, *flags);
648 }
649 cpu_relax();
650 }
651}
652
Ingo Molnar74019222009-02-18 12:23:29 +0100653static inline int
Arun R Bharadwaj597d0272009-04-16 12:13:26 +0530654__mod_timer(struct timer_list *timer, unsigned long expires,
655 bool pending_only, int pinned)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100657 struct tvec_base *base, *new_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 unsigned long flags;
Arun R Bharadwajeea08f32009-04-16 12:16:41 +0530659 int ret = 0 , cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800661 timer_stats_timer_set_start_info(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 BUG_ON(!timer->function);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700664 base = lock_timer_base(timer, &flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700666 if (timer_pending(timer)) {
667 detach_timer(timer, 0);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200668 if (timer->expires == base->next_timer &&
669 !tbase_get_deferrable(timer->base))
670 base->next_timer = base->timer_jiffies;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 ret = 1;
Ingo Molnar74019222009-02-18 12:23:29 +0100672 } else {
673 if (pending_only)
674 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 }
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700676
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800677 debug_activate(timer, expires);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700678
Arun R Bharadwajeea08f32009-04-16 12:16:41 +0530679 cpu = smp_processor_id();
680
681#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
682 if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu)) {
683 int preferred_cpu = get_nohz_load_balancer();
684
685 if (preferred_cpu >= 0)
686 cpu = preferred_cpu;
687 }
688#endif
689 new_base = per_cpu(tvec_bases, cpu);
690
Oleg Nesterov3691c512006-03-31 02:30:30 -0800691 if (base != new_base) {
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700692 /*
693 * We are trying to schedule the timer on the local CPU.
694 * However we can't change timer's base while it is running,
695 * otherwise del_timer_sync() can't detect that the timer's
696 * handler yet has not finished. This also guarantees that
697 * the timer is serialized wrt itself.
698 */
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800699 if (likely(base->running_timer != timer)) {
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700700 /* See the comment in lock_timer_base() */
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700701 timer_set_base(timer, NULL);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700702 spin_unlock(&base->lock);
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800703 base = new_base;
704 spin_lock(&base->lock);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700705 timer_set_base(timer, base);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700706 }
707 }
708
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 timer->expires = expires;
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200710 if (time_before(timer->expires, base->next_timer) &&
711 !tbase_get_deferrable(timer->base))
712 base->next_timer = timer->expires;
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800713 internal_add_timer(base, timer);
Ingo Molnar74019222009-02-18 12:23:29 +0100714
715out_unlock:
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800716 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717
718 return ret;
719}
720
Ingo Molnar74019222009-02-18 12:23:29 +0100721/**
722 * mod_timer_pending - modify a pending timer's timeout
723 * @timer: the pending timer to be modified
724 * @expires: new timeout in jiffies
725 *
726 * mod_timer_pending() is the same for pending timers as mod_timer(),
727 * but will not re-activate and modify already deleted timers.
728 *
729 * It is useful for unserialized use of timers.
730 */
731int mod_timer_pending(struct timer_list *timer, unsigned long expires)
732{
Arun R Bharadwaj597d0272009-04-16 12:13:26 +0530733 return __mod_timer(timer, expires, true, TIMER_NOT_PINNED);
Ingo Molnar74019222009-02-18 12:23:29 +0100734}
735EXPORT_SYMBOL(mod_timer_pending);
736
Arjan van de Ven3bbb9ec2010-03-11 14:04:36 -0800737/*
738 * Decide where to put the timer while taking the slack into account
739 *
740 * Algorithm:
741 * 1) calculate the maximum (absolute) time
742 * 2) calculate the highest bit where the expires and new max are different
743 * 3) use this bit to make a mask
744 * 4) use the bitmask to round down the maximum time, so that all last
745 * bits are zeros
746 */
747static inline
748unsigned long apply_slack(struct timer_list *timer, unsigned long expires)
749{
750 unsigned long expires_limit, mask;
751 int bit;
752
Jeff Chuaf00e0472010-05-24 07:16:24 +0800753 expires_limit = expires;
Arjan van de Ven3bbb9ec2010-03-11 14:04:36 -0800754
Thomas Gleixner8e63d772010-05-25 20:43:30 +0200755 if (timer->slack >= 0) {
Jeff Chuaf00e0472010-05-24 07:16:24 +0800756 expires_limit = expires + timer->slack;
Thomas Gleixner8e63d772010-05-25 20:43:30 +0200757 } else {
Thomas Gleixner2abfb9e2010-05-26 16:07:13 +0200758 unsigned long now = jiffies;
Arjan van de Ven3bbb9ec2010-03-11 14:04:36 -0800759
Thomas Gleixner8e63d772010-05-25 20:43:30 +0200760 /* No slack, if already expired else auto slack 0.4% */
761 if (time_after(expires, now))
762 expires_limit = expires + (expires - now)/256;
763 }
Arjan van de Ven3bbb9ec2010-03-11 14:04:36 -0800764 mask = expires ^ expires_limit;
Arjan van de Ven3bbb9ec2010-03-11 14:04:36 -0800765 if (mask == 0)
766 return expires;
767
768 bit = find_last_bit(&mask, BITS_PER_LONG);
769
770 mask = (1 << bit) - 1;
771
772 expires_limit = expires_limit & ~(mask);
773
774 return expires_limit;
775}
776
Ingo Molnar74019222009-02-18 12:23:29 +0100777/**
778 * mod_timer - modify a timer's timeout
779 * @timer: the timer to be modified
780 * @expires: new timeout in jiffies
781 *
782 * mod_timer() is a more efficient way to update the expire field of an
783 * active timer (if the timer is inactive it will be activated)
784 *
785 * mod_timer(timer, expires) is equivalent to:
786 *
787 * del_timer(timer); timer->expires = expires; add_timer(timer);
788 *
789 * Note that if there are multiple unserialized concurrent users of the
790 * same timer, then mod_timer() is the only safe way to modify the timeout,
791 * since add_timer() cannot modify an already running timer.
792 *
793 * The function returns whether it has modified a pending timer or not.
794 * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an
795 * active timer returns 1.)
796 */
797int mod_timer(struct timer_list *timer, unsigned long expires)
798{
799 /*
800 * This is a common optimization triggered by the
801 * networking code - if the timer is re-modified
802 * to be the same thing then just return:
803 */
Pavel Roskin48411582009-07-18 16:46:02 -0400804 if (timer_pending(timer) && timer->expires == expires)
Ingo Molnar74019222009-02-18 12:23:29 +0100805 return 1;
806
Arjan van de Ven3bbb9ec2010-03-11 14:04:36 -0800807 expires = apply_slack(timer, expires);
808
Arun R Bharadwaj597d0272009-04-16 12:13:26 +0530809 return __mod_timer(timer, expires, false, TIMER_NOT_PINNED);
Ingo Molnar74019222009-02-18 12:23:29 +0100810}
811EXPORT_SYMBOL(mod_timer);
812
813/**
Arun R Bharadwaj597d0272009-04-16 12:13:26 +0530814 * mod_timer_pinned - modify a timer's timeout
815 * @timer: the timer to be modified
816 * @expires: new timeout in jiffies
817 *
818 * mod_timer_pinned() is a way to update the expire field of an
819 * active timer (if the timer is inactive it will be activated)
820 * and not allow the timer to be migrated to a different CPU.
821 *
822 * mod_timer_pinned(timer, expires) is equivalent to:
823 *
824 * del_timer(timer); timer->expires = expires; add_timer(timer);
825 */
826int mod_timer_pinned(struct timer_list *timer, unsigned long expires)
827{
828 if (timer->expires == expires && timer_pending(timer))
829 return 1;
830
831 return __mod_timer(timer, expires, false, TIMER_PINNED);
832}
833EXPORT_SYMBOL(mod_timer_pinned);
834
835/**
Ingo Molnar74019222009-02-18 12:23:29 +0100836 * add_timer - start a timer
837 * @timer: the timer to be added
838 *
839 * The kernel will do a ->function(->data) callback from the
840 * timer interrupt at the ->expires point in the future. The
841 * current time is 'jiffies'.
842 *
843 * The timer's ->expires, ->function (and if the handler uses it, ->data)
844 * fields must be set prior calling this function.
845 *
846 * Timers with an ->expires field in the past will be executed in the next
847 * timer tick.
848 */
849void add_timer(struct timer_list *timer)
850{
851 BUG_ON(timer_pending(timer));
852 mod_timer(timer, timer->expires);
853}
854EXPORT_SYMBOL(add_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700856/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 * add_timer_on - start a timer on a particular CPU
858 * @timer: the timer to be added
859 * @cpu: the CPU to start it on
860 *
861 * This is not very scalable on SMP. Double adds are not possible.
862 */
863void add_timer_on(struct timer_list *timer, int cpu)
864{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100865 struct tvec_base *base = per_cpu(tvec_bases, cpu);
Thomas Gleixner68194572007-07-19 01:49:16 -0700866 unsigned long flags;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700867
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800868 timer_stats_timer_set_start_info(timer);
Thomas Gleixner68194572007-07-19 01:49:16 -0700869 BUG_ON(timer_pending(timer) || !timer->function);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800870 spin_lock_irqsave(&base->lock, flags);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700871 timer_set_base(timer, base);
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800872 debug_activate(timer, timer->expires);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200873 if (time_before(timer->expires, base->next_timer) &&
874 !tbase_get_deferrable(timer->base))
875 base->next_timer = timer->expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 internal_add_timer(base, timer);
Thomas Gleixner06d83082008-03-22 09:20:24 +0100877 /*
878 * Check whether the other CPU is idle and needs to be
879 * triggered to reevaluate the timer wheel when nohz is
880 * active. We are protected against the other CPU fiddling
881 * with the timer by holding the timer base lock. This also
882 * makes sure that a CPU on the way to idle can not evaluate
883 * the timer wheel.
884 */
885 wake_up_idle_cpu(cpu);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800886 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887}
Andi Kleena9862e02009-05-19 22:49:07 +0200888EXPORT_SYMBOL_GPL(add_timer_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700890/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891 * del_timer - deactive a timer.
892 * @timer: the timer to be deactivated
893 *
894 * del_timer() deactivates a timer - this works on both active and inactive
895 * timers.
896 *
897 * The function returns whether it has deactivated a pending timer or not.
898 * (ie. del_timer() of an inactive timer returns 0, del_timer() of an
899 * active timer returns 1.)
900 */
901int del_timer(struct timer_list *timer)
902{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100903 struct tvec_base *base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904 unsigned long flags;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700905 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800907 timer_stats_timer_clear_start_info(timer);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700908 if (timer_pending(timer)) {
909 base = lock_timer_base(timer, &flags);
910 if (timer_pending(timer)) {
911 detach_timer(timer, 1);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200912 if (timer->expires == base->next_timer &&
913 !tbase_get_deferrable(timer->base))
914 base->next_timer = base->timer_jiffies;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700915 ret = 1;
916 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700920 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922EXPORT_SYMBOL(del_timer);
923
924#ifdef CONFIG_SMP
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700925/**
926 * try_to_del_timer_sync - Try to deactivate a timer
927 * @timer: timer do del
928 *
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700929 * This function tries to deactivate a timer. Upon successful (ret >= 0)
930 * exit the timer is not queued and the handler is not running on any CPU.
931 *
932 * It must not be called from interrupt contexts.
933 */
934int try_to_del_timer_sync(struct timer_list *timer)
935{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100936 struct tvec_base *base;
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700937 unsigned long flags;
938 int ret = -1;
939
940 base = lock_timer_base(timer, &flags);
941
942 if (base->running_timer == timer)
943 goto out;
944
Andrew Morton829b6c12010-03-11 14:04:30 -0800945 timer_stats_timer_clear_start_info(timer);
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700946 ret = 0;
947 if (timer_pending(timer)) {
948 detach_timer(timer, 1);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200949 if (timer->expires == base->next_timer &&
950 !tbase_get_deferrable(timer->base))
951 base->next_timer = base->timer_jiffies;
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700952 ret = 1;
953 }
954out:
955 spin_unlock_irqrestore(&base->lock, flags);
956
957 return ret;
958}
David Howellse19dff12007-04-26 15:46:56 -0700959EXPORT_SYMBOL(try_to_del_timer_sync);
960
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700961/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 * del_timer_sync - deactivate a timer and wait for the handler to finish.
963 * @timer: the timer to be deactivated
964 *
965 * This function only differs from del_timer() on SMP: besides deactivating
966 * the timer it also makes sure the handler has finished executing on other
967 * CPUs.
968 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800969 * Synchronization rules: Callers must prevent restarting of the timer,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970 * otherwise this function is meaningless. It must not be called from
971 * interrupt contexts. The caller must not hold locks which would prevent
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700972 * completion of the timer's handler. The timer's handler must not call
973 * add_timer_on(). Upon exit the timer is not queued and the handler is
974 * not running on any CPU.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 *
976 * The function returns whether it has deactivated a pending timer or not.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 */
978int del_timer_sync(struct timer_list *timer)
979{
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100980#ifdef CONFIG_LOCKDEP
981 unsigned long flags;
982
983 local_irq_save(flags);
984 lock_map_acquire(&timer->lockdep_map);
985 lock_map_release(&timer->lockdep_map);
986 local_irq_restore(flags);
987#endif
988
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700989 for (;;) {
990 int ret = try_to_del_timer_sync(timer);
991 if (ret >= 0)
992 return ret;
Andrew Mortona0009652006-07-14 00:24:06 -0700993 cpu_relax();
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700994 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995}
996EXPORT_SYMBOL(del_timer_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997#endif
998
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100999static int cascade(struct tvec_base *base, struct tvec *tv, int index)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000{
1001 /* cascade all the timers from tv up one level */
Porpoise3439dd82006-06-23 02:05:56 -07001002 struct timer_list *timer, *tmp;
1003 struct list_head tv_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004
Porpoise3439dd82006-06-23 02:05:56 -07001005 list_replace_init(tv->vec + index, &tv_list);
1006
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 /*
Porpoise3439dd82006-06-23 02:05:56 -07001008 * We are removing _all_ timers from the list, so we
1009 * don't have to detach them individually.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010 */
Porpoise3439dd82006-06-23 02:05:56 -07001011 list_for_each_entry_safe(timer, tmp, &tv_list, entry) {
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001012 BUG_ON(tbase_get_base(timer->base) != base);
Porpoise3439dd82006-06-23 02:05:56 -07001013 internal_add_timer(base, timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
1016 return index;
1017}
1018
Thomas Gleixner576da122010-03-12 21:10:29 +01001019static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long),
1020 unsigned long data)
1021{
1022 int preempt_count = preempt_count();
1023
1024#ifdef CONFIG_LOCKDEP
1025 /*
1026 * It is permissible to free the timer from inside the
1027 * function that is called from it, this we need to take into
1028 * account for lockdep too. To avoid bogus "held lock freed"
1029 * warnings as well as problems when looking into
1030 * timer->lockdep_map, make a copy and use that here.
1031 */
1032 struct lockdep_map lockdep_map = timer->lockdep_map;
1033#endif
1034 /*
1035 * Couple the lock chain with the lock chain at
1036 * del_timer_sync() by acquiring the lock_map around the fn()
1037 * call here and in del_timer_sync().
1038 */
1039 lock_map_acquire(&lockdep_map);
1040
1041 trace_timer_expire_entry(timer);
1042 fn(data);
1043 trace_timer_expire_exit(timer);
1044
1045 lock_map_release(&lockdep_map);
1046
1047 if (preempt_count != preempt_count()) {
Thomas Gleixner802702e2010-03-12 20:13:23 +01001048 WARN_ONCE(1, "timer: %pF preempt leak: %08x -> %08x\n",
1049 fn, preempt_count, preempt_count());
1050 /*
1051 * Restore the preempt count. That gives us a decent
1052 * chance to survive and extract information. If the
1053 * callback kept a lock held, bad luck, but not worse
1054 * than the BUG() we had.
1055 */
1056 preempt_count() = preempt_count;
Thomas Gleixner576da122010-03-12 21:10:29 +01001057 }
1058}
1059
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -07001060#define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK)
1061
1062/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 * __run_timers - run all expired timers (if any) on this CPU.
1064 * @base: the timer vector to be processed.
1065 *
1066 * This function cascades all vectors and executes all expired timer
1067 * vectors.
1068 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001069static inline void __run_timers(struct tvec_base *base)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070{
1071 struct timer_list *timer;
1072
Oleg Nesterov3691c512006-03-31 02:30:30 -08001073 spin_lock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 while (time_after_eq(jiffies, base->timer_jiffies)) {
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07001075 struct list_head work_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076 struct list_head *head = &work_list;
Thomas Gleixner68194572007-07-19 01:49:16 -07001077 int index = base->timer_jiffies & TVR_MASK;
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07001078
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079 /*
1080 * Cascade timers:
1081 */
1082 if (!index &&
1083 (!cascade(base, &base->tv2, INDEX(0))) &&
1084 (!cascade(base, &base->tv3, INDEX(1))) &&
1085 !cascade(base, &base->tv4, INDEX(2)))
1086 cascade(base, &base->tv5, INDEX(3));
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07001087 ++base->timer_jiffies;
1088 list_replace_init(base->tv1.vec + index, &work_list);
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001089 while (!list_empty(head)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090 void (*fn)(unsigned long);
1091 unsigned long data;
1092
Pavel Emelianovb5e61812007-05-08 00:30:19 -07001093 timer = list_first_entry(head, struct timer_list,entry);
Thomas Gleixner68194572007-07-19 01:49:16 -07001094 fn = timer->function;
1095 data = timer->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096
Ingo Molnar82f67cd2007-02-16 01:28:13 -08001097 timer_stats_account_timer(timer);
1098
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 set_running_timer(base, timer);
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001100 detach_timer(timer, 1);
Johannes Berg6f2b9b92009-01-29 16:03:20 +01001101
Oleg Nesterov3691c512006-03-31 02:30:30 -08001102 spin_unlock_irq(&base->lock);
Thomas Gleixner576da122010-03-12 21:10:29 +01001103 call_timer_fn(timer, fn, data);
Oleg Nesterov3691c512006-03-31 02:30:30 -08001104 spin_lock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105 }
1106 }
1107 set_running_timer(base, NULL);
Oleg Nesterov3691c512006-03-31 02:30:30 -08001108 spin_unlock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109}
1110
Russell Kingee9c5782008-04-20 13:59:33 +01001111#ifdef CONFIG_NO_HZ
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112/*
1113 * Find out when the next timer event is due to happen. This
Randy Dunlap90cba642009-08-25 14:35:41 -07001114 * is used on S/390 to stop all activity when a CPU is idle.
1115 * This function needs to be called with interrupts disabled.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001117static unsigned long __next_timer_interrupt(struct tvec_base *base)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118{
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001119 unsigned long timer_jiffies = base->timer_jiffies;
Thomas Gleixnereaad0842007-05-29 23:47:39 +02001120 unsigned long expires = timer_jiffies + NEXT_TIMER_MAX_DELTA;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001121 int index, slot, array, found = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122 struct timer_list *nte;
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001123 struct tvec *varray[4];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124
1125 /* Look for timer events in tv1. */
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001126 index = slot = timer_jiffies & TVR_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 do {
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001128 list_for_each_entry(nte, base->tv1.vec + slot, entry) {
Thomas Gleixner68194572007-07-19 01:49:16 -07001129 if (tbase_get_deferrable(nte->base))
1130 continue;
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001131
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001132 found = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133 expires = nte->expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001134 /* Look at the cascade bucket(s)? */
1135 if (!index || slot < index)
1136 goto cascade;
1137 return expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 }
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001139 slot = (slot + 1) & TVR_MASK;
1140 } while (slot != index);
1141
1142cascade:
1143 /* Calculate the next cascade event */
1144 if (index)
1145 timer_jiffies += TVR_SIZE - index;
1146 timer_jiffies >>= TVR_BITS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147
1148 /* Check tv2-tv5. */
1149 varray[0] = &base->tv2;
1150 varray[1] = &base->tv3;
1151 varray[2] = &base->tv4;
1152 varray[3] = &base->tv5;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001153
1154 for (array = 0; array < 4; array++) {
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001155 struct tvec *varp = varray[array];
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001156
1157 index = slot = timer_jiffies & TVN_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001158 do {
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001159 list_for_each_entry(nte, varp->vec + slot, entry) {
Jon Huntera0419882009-05-01 13:10:23 -07001160 if (tbase_get_deferrable(nte->base))
1161 continue;
1162
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001163 found = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164 if (time_before(nte->expires, expires))
1165 expires = nte->expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001166 }
1167 /*
1168 * Do we still search for the first timer or are
1169 * we looking up the cascade buckets ?
1170 */
1171 if (found) {
1172 /* Look at the cascade bucket(s)? */
1173 if (!index || slot < index)
1174 break;
1175 return expires;
1176 }
1177 slot = (slot + 1) & TVN_MASK;
1178 } while (slot != index);
1179
1180 if (index)
1181 timer_jiffies += TVN_SIZE - index;
1182 timer_jiffies >>= TVN_BITS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001183 }
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001184 return expires;
1185}
1186
1187/*
1188 * Check, if the next hrtimer event is before the next timer wheel
1189 * event:
1190 */
1191static unsigned long cmp_next_hrtimer_event(unsigned long now,
1192 unsigned long expires)
1193{
1194 ktime_t hr_delta = hrtimer_get_next_event();
1195 struct timespec tsdelta;
Thomas Gleixner9501b6c2007-03-25 14:31:17 +02001196 unsigned long delta;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001197
1198 if (hr_delta.tv64 == KTIME_MAX)
1199 return expires;
1200
Thomas Gleixner9501b6c2007-03-25 14:31:17 +02001201 /*
1202 * Expired timer available, let it expire in the next tick
1203 */
1204 if (hr_delta.tv64 <= 0)
1205 return now + 1;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001206
1207 tsdelta = ktime_to_timespec(hr_delta);
Thomas Gleixner9501b6c2007-03-25 14:31:17 +02001208 delta = timespec_to_jiffies(&tsdelta);
Thomas Gleixnereaad0842007-05-29 23:47:39 +02001209
1210 /*
1211 * Limit the delta to the max value, which is checked in
1212 * tick_nohz_stop_sched_tick():
1213 */
1214 if (delta > NEXT_TIMER_MAX_DELTA)
1215 delta = NEXT_TIMER_MAX_DELTA;
1216
Thomas Gleixner9501b6c2007-03-25 14:31:17 +02001217 /*
1218 * Take rounding errors in to account and make sure, that it
1219 * expires in the next tick. Otherwise we go into an endless
1220 * ping pong due to tick_nohz_stop_sched_tick() retriggering
1221 * the timer softirq
1222 */
1223 if (delta < 1)
1224 delta = 1;
1225 now += delta;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001226 if (time_before(now, expires))
1227 return now;
1228 return expires;
1229}
1230
1231/**
Li Zefan8dce39c2007-11-05 14:51:10 -08001232 * get_next_timer_interrupt - return the jiffy of the next pending timer
Randy Dunlap05fb6bf2007-02-28 20:12:13 -08001233 * @now: current time (in jiffies)
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001234 */
Thomas Gleixnerfd064b92007-02-16 01:27:47 -08001235unsigned long get_next_timer_interrupt(unsigned long now)
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001236{
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001237 struct tvec_base *base = __get_cpu_var(tvec_bases);
Thomas Gleixnerfd064b92007-02-16 01:27:47 -08001238 unsigned long expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001239
1240 spin_lock(&base->lock);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +02001241 if (time_before_eq(base->next_timer, base->timer_jiffies))
1242 base->next_timer = __next_timer_interrupt(base);
1243 expires = base->next_timer;
Oleg Nesterov3691c512006-03-31 02:30:30 -08001244 spin_unlock(&base->lock);
Tony Lindgren69239742006-03-06 15:42:45 -08001245
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001246 if (time_before_eq(expires, now))
1247 return now;
Zachary Amsden0662b712006-05-20 15:00:24 -07001248
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001249 return cmp_next_hrtimer_event(now, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250}
1251#endif
1252
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253/*
Daniel Walker5b4db0c2007-10-18 03:06:11 -07001254 * Called from the timer interrupt handler to charge one tick to the current
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255 * process. user_tick is 1 if the tick is user time, 0 for system.
1256 */
1257void update_process_times(int user_tick)
1258{
1259 struct task_struct *p = current;
1260 int cpu = smp_processor_id();
1261
1262 /* Note: this timer irq context must be accounted for as well. */
Paul Mackerrasfa13a5a2007-11-09 22:39:38 +01001263 account_process_tick(p, user_tick);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264 run_local_timers();
Paul E. McKenneya1572292009-08-22 13:56:51 -07001265 rcu_check_callbacks(cpu, user_tick);
Peter Zijlstrab845b512008-08-08 21:47:09 +02001266 printk_tick();
Peter Zijlstrafe432202010-01-18 09:08:26 +01001267 perf_event_do_pending();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 scheduler_tick();
Thomas Gleixner68194572007-07-19 01:49:16 -07001269 run_posix_cpu_timers(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270}
1271
1272/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273 * This function runs timers and the timer-tq in bottom half context.
1274 */
1275static void run_timer_softirq(struct softirq_action *h)
1276{
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001277 struct tvec_base *base = __get_cpu_var(tvec_bases);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001278
Peter Zijlstrad3d74452008-01-25 21:08:31 +01001279 hrtimer_run_pending();
Ingo Molnar82f67cd2007-02-16 01:28:13 -08001280
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281 if (time_after_eq(jiffies, base->timer_jiffies))
1282 __run_timers(base);
1283}
1284
1285/*
1286 * Called by the local, per-CPU timer interrupt on SMP.
1287 */
1288void run_local_timers(void)
1289{
Peter Zijlstrad3d74452008-01-25 21:08:31 +01001290 hrtimer_run_queues();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 raise_softirq(TIMER_SOFTIRQ);
Ingo Molnar6687a972006-03-24 03:18:41 -08001292 softlockup_tick();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293}
1294
1295/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001296 * The 64-bit jiffies value is not atomic - you MUST NOT read it
1297 * without sampling the sequence number in xtime_lock.
1298 * jiffies is defined in the linker script...
1299 */
1300
Atsushi Nemoto3171a032006-09-29 02:00:32 -07001301void do_timer(unsigned long ticks)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001302{
Atsushi Nemoto3171a032006-09-29 02:00:32 -07001303 jiffies_64 += ticks;
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02001304 update_wall_time();
1305 calc_global_load();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306}
1307
1308#ifdef __ARCH_WANT_SYS_ALARM
1309
1310/*
1311 * For backwards compatibility? This can be done in libc so Alpha
1312 * and all newer ports shouldn't need it.
1313 */
Heiko Carstens58fd3aa2009-01-14 14:14:03 +01001314SYSCALL_DEFINE1(alarm, unsigned int, seconds)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315{
Thomas Gleixnerc08b8a42006-03-25 03:06:33 -08001316 return alarm_setitimer(seconds);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317}
1318
1319#endif
1320
1321#ifndef __alpha__
1322
1323/*
1324 * The Alpha uses getxpid, getxuid, and getxgid instead. Maybe this
1325 * should be moved into arch/i386 instead?
1326 */
1327
1328/**
1329 * sys_getpid - return the thread group id of the current process
1330 *
1331 * Note, despite the name, this returns the tgid not the pid. The tgid and
1332 * the pid are identical unless CLONE_THREAD was specified on clone() in
1333 * which case the tgid is the same in all threads of the same group.
1334 *
1335 * This is SMP safe as current->tgid does not change.
1336 */
Heiko Carstens58fd3aa2009-01-14 14:14:03 +01001337SYSCALL_DEFINE0(getpid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338{
Pavel Emelyanovb4888932007-10-18 23:40:14 -07001339 return task_tgid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340}
1341
1342/*
Kirill Korotaev6997a6f2006-08-13 23:24:23 -07001343 * Accessing ->real_parent is not SMP-safe, it could
1344 * change from under us. However, we can use a stale
1345 * value of ->real_parent under rcu_read_lock(), see
1346 * release_task()->call_rcu(delayed_put_task_struct).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 */
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001348SYSCALL_DEFINE0(getppid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349{
1350 int pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351
Kirill Korotaev6997a6f2006-08-13 23:24:23 -07001352 rcu_read_lock();
Pavel Emelyanov6c5f3e72008-02-08 04:19:20 -08001353 pid = task_tgid_vnr(current->real_parent);
Kirill Korotaev6997a6f2006-08-13 23:24:23 -07001354 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355
Linus Torvalds1da177e2005-04-16 15:20:36 -07001356 return pid;
1357}
1358
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001359SYSCALL_DEFINE0(getuid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360{
1361 /* Only we change this so SMP safe */
David Howells76aac0e2008-11-14 10:39:12 +11001362 return current_uid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363}
1364
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001365SYSCALL_DEFINE0(geteuid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366{
1367 /* Only we change this so SMP safe */
David Howells76aac0e2008-11-14 10:39:12 +11001368 return current_euid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369}
1370
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001371SYSCALL_DEFINE0(getgid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001372{
1373 /* Only we change this so SMP safe */
David Howells76aac0e2008-11-14 10:39:12 +11001374 return current_gid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001375}
1376
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001377SYSCALL_DEFINE0(getegid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378{
1379 /* Only we change this so SMP safe */
David Howells76aac0e2008-11-14 10:39:12 +11001380 return current_egid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381}
1382
1383#endif
1384
1385static void process_timeout(unsigned long __data)
1386{
Ingo Molnar36c8b582006-07-03 00:25:41 -07001387 wake_up_process((struct task_struct *)__data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388}
1389
1390/**
1391 * schedule_timeout - sleep until timeout
1392 * @timeout: timeout value in jiffies
1393 *
1394 * Make the current task sleep until @timeout jiffies have
1395 * elapsed. The routine will return immediately unless
1396 * the current task state has been set (see set_current_state()).
1397 *
1398 * You can set the task state as follows -
1399 *
1400 * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
1401 * pass before the routine returns. The routine will return 0
1402 *
1403 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1404 * delivered to the current task. In this case the remaining time
1405 * in jiffies will be returned, or 0 if the timer expired in time
1406 *
1407 * The current task state is guaranteed to be TASK_RUNNING when this
1408 * routine returns.
1409 *
1410 * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
1411 * the CPU away without a bound on the timeout. In this case the return
1412 * value will be %MAX_SCHEDULE_TIMEOUT.
1413 *
1414 * In all cases the return value is guaranteed to be non-negative.
1415 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08001416signed long __sched schedule_timeout(signed long timeout)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417{
1418 struct timer_list timer;
1419 unsigned long expire;
1420
1421 switch (timeout)
1422 {
1423 case MAX_SCHEDULE_TIMEOUT:
1424 /*
1425 * These two special cases are useful to be comfortable
1426 * in the caller. Nothing more. We could take
1427 * MAX_SCHEDULE_TIMEOUT from one of the negative value
1428 * but I' d like to return a valid offset (>=0) to allow
1429 * the caller to do everything it want with the retval.
1430 */
1431 schedule();
1432 goto out;
1433 default:
1434 /*
1435 * Another bit of PARANOID. Note that the retval will be
1436 * 0 since no piece of kernel is supposed to do a check
1437 * for a negative retval of schedule_timeout() (since it
1438 * should never happens anyway). You just have the printk()
1439 * that will tell you if something is gone wrong and where.
1440 */
Andrew Morton5b149bc2006-12-22 01:10:14 -08001441 if (timeout < 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442 printk(KERN_ERR "schedule_timeout: wrong timeout "
Andrew Morton5b149bc2006-12-22 01:10:14 -08001443 "value %lx\n", timeout);
1444 dump_stack();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445 current->state = TASK_RUNNING;
1446 goto out;
1447 }
1448 }
1449
1450 expire = timeout + jiffies;
1451
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -07001452 setup_timer_on_stack(&timer, process_timeout, (unsigned long)current);
Arun R Bharadwaj597d0272009-04-16 12:13:26 +05301453 __mod_timer(&timer, expire, false, TIMER_NOT_PINNED);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454 schedule();
1455 del_singleshot_timer_sync(&timer);
1456
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -07001457 /* Remove the timer from the object tracker */
1458 destroy_timer_on_stack(&timer);
1459
Linus Torvalds1da177e2005-04-16 15:20:36 -07001460 timeout = expire - jiffies;
1461
1462 out:
1463 return timeout < 0 ? 0 : timeout;
1464}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465EXPORT_SYMBOL(schedule_timeout);
1466
Andrew Morton8a1c1752005-09-13 01:25:15 -07001467/*
1468 * We can use __set_current_state() here because schedule_timeout() calls
1469 * schedule() unconditionally.
1470 */
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001471signed long __sched schedule_timeout_interruptible(signed long timeout)
1472{
Andrew Mortona5a0d522005-10-30 15:01:42 -08001473 __set_current_state(TASK_INTERRUPTIBLE);
1474 return schedule_timeout(timeout);
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001475}
1476EXPORT_SYMBOL(schedule_timeout_interruptible);
1477
Matthew Wilcox294d5cc2007-12-06 11:59:46 -05001478signed long __sched schedule_timeout_killable(signed long timeout)
1479{
1480 __set_current_state(TASK_KILLABLE);
1481 return schedule_timeout(timeout);
1482}
1483EXPORT_SYMBOL(schedule_timeout_killable);
1484
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001485signed long __sched schedule_timeout_uninterruptible(signed long timeout)
1486{
Andrew Mortona5a0d522005-10-30 15:01:42 -08001487 __set_current_state(TASK_UNINTERRUPTIBLE);
1488 return schedule_timeout(timeout);
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001489}
1490EXPORT_SYMBOL(schedule_timeout_uninterruptible);
1491
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492/* Thread ID - the internal kernel "pid" */
Heiko Carstens58fd3aa2009-01-14 14:14:03 +01001493SYSCALL_DEFINE0(gettid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494{
Pavel Emelyanovb4888932007-10-18 23:40:14 -07001495 return task_pid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496}
1497
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -07001498/**
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001499 * do_sysinfo - fill in sysinfo struct
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -07001500 * @info: pointer to buffer to fill
Thomas Gleixner68194572007-07-19 01:49:16 -07001501 */
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001502int do_sysinfo(struct sysinfo *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 unsigned long mem_total, sav_total;
1505 unsigned int mem_unit, bitcount;
Thomas Gleixner2d024942009-05-02 20:08:52 +02001506 struct timespec tp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001507
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001508 memset(info, 0, sizeof(struct sysinfo));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509
Thomas Gleixner2d024942009-05-02 20:08:52 +02001510 ktime_get_ts(&tp);
1511 monotonic_to_bootbased(&tp);
1512 info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513
Thomas Gleixner2d024942009-05-02 20:08:52 +02001514 get_avenrun(info->loads, 0, SI_LOAD_SHIFT - FSHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515
Thomas Gleixner2d024942009-05-02 20:08:52 +02001516 info->procs = nr_threads;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001518 si_meminfo(info);
1519 si_swapinfo(info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520
1521 /*
1522 * If the sum of all the available memory (i.e. ram + swap)
1523 * is less than can be stored in a 32 bit unsigned long then
1524 * we can be binary compatible with 2.2.x kernels. If not,
1525 * well, in that case 2.2.x was broken anyways...
1526 *
1527 * -Erik Andersen <andersee@debian.org>
1528 */
1529
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001530 mem_total = info->totalram + info->totalswap;
1531 if (mem_total < info->totalram || mem_total < info->totalswap)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 goto out;
1533 bitcount = 0;
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001534 mem_unit = info->mem_unit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535 while (mem_unit > 1) {
1536 bitcount++;
1537 mem_unit >>= 1;
1538 sav_total = mem_total;
1539 mem_total <<= 1;
1540 if (mem_total < sav_total)
1541 goto out;
1542 }
1543
1544 /*
1545 * If mem_total did not overflow, multiply all memory values by
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001546 * info->mem_unit and set it to 1. This leaves things compatible
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547 * with 2.2.x, and also retains compatibility with earlier 2.4.x
1548 * kernels...
1549 */
1550
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001551 info->mem_unit = 1;
1552 info->totalram <<= bitcount;
1553 info->freeram <<= bitcount;
1554 info->sharedram <<= bitcount;
1555 info->bufferram <<= bitcount;
1556 info->totalswap <<= bitcount;
1557 info->freeswap <<= bitcount;
1558 info->totalhigh <<= bitcount;
1559 info->freehigh <<= bitcount;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001561out:
1562 return 0;
1563}
1564
Heiko Carstens1e7bfb22009-01-14 14:14:29 +01001565SYSCALL_DEFINE1(sysinfo, struct sysinfo __user *, info)
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001566{
1567 struct sysinfo val;
1568
1569 do_sysinfo(&val);
1570
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571 if (copy_to_user(info, &val, sizeof(struct sysinfo)))
1572 return -EFAULT;
1573
1574 return 0;
1575}
1576
Adrian Bunkb4be6252007-12-18 18:05:58 +01001577static int __cpuinit init_timers_cpu(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578{
1579 int j;
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001580 struct tvec_base *base;
Adrian Bunkb4be6252007-12-18 18:05:58 +01001581 static char __cpuinitdata tvec_base_done[NR_CPUS];
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001582
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001583 if (!tvec_base_done[cpu]) {
Jan Beulicha4a61982006-03-24 03:15:54 -08001584 static char boot_done;
1585
Jan Beulicha4a61982006-03-24 03:15:54 -08001586 if (boot_done) {
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001587 /*
1588 * The APs use this path later in boot
1589 */
Christoph Lameter94f60302007-07-17 04:03:29 -07001590 base = kmalloc_node(sizeof(*base),
1591 GFP_KERNEL | __GFP_ZERO,
Jan Beulicha4a61982006-03-24 03:15:54 -08001592 cpu_to_node(cpu));
1593 if (!base)
1594 return -ENOMEM;
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001595
1596 /* Make sure that tvec_base is 2 byte aligned */
1597 if (tbase_get_deferrable(base)) {
1598 WARN_ON(1);
1599 kfree(base);
1600 return -ENOMEM;
1601 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001602 per_cpu(tvec_bases, cpu) = base;
Jan Beulicha4a61982006-03-24 03:15:54 -08001603 } else {
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001604 /*
1605 * This is for the boot CPU - we use compile-time
1606 * static initialisation because per-cpu memory isn't
1607 * ready yet and because the memory allocators are not
1608 * initialised either.
1609 */
Jan Beulicha4a61982006-03-24 03:15:54 -08001610 boot_done = 1;
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001611 base = &boot_tvec_bases;
Jan Beulicha4a61982006-03-24 03:15:54 -08001612 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001613 tvec_base_done[cpu] = 1;
1614 } else {
1615 base = per_cpu(tvec_bases, cpu);
Jan Beulicha4a61982006-03-24 03:15:54 -08001616 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001617
Oleg Nesterov3691c512006-03-31 02:30:30 -08001618 spin_lock_init(&base->lock);
Ingo Molnard730e882006-07-03 00:25:10 -07001619
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620 for (j = 0; j < TVN_SIZE; j++) {
1621 INIT_LIST_HEAD(base->tv5.vec + j);
1622 INIT_LIST_HEAD(base->tv4.vec + j);
1623 INIT_LIST_HEAD(base->tv3.vec + j);
1624 INIT_LIST_HEAD(base->tv2.vec + j);
1625 }
1626 for (j = 0; j < TVR_SIZE; j++)
1627 INIT_LIST_HEAD(base->tv1.vec + j);
1628
1629 base->timer_jiffies = jiffies;
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +02001630 base->next_timer = base->timer_jiffies;
Jan Beulicha4a61982006-03-24 03:15:54 -08001631 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632}
1633
1634#ifdef CONFIG_HOTPLUG_CPU
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001635static void migrate_timer_list(struct tvec_base *new_base, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001636{
1637 struct timer_list *timer;
1638
1639 while (!list_empty(head)) {
Pavel Emelianovb5e61812007-05-08 00:30:19 -07001640 timer = list_first_entry(head, struct timer_list, entry);
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001641 detach_timer(timer, 0);
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001642 timer_set_base(timer, new_base);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +02001643 if (time_before(timer->expires, new_base->next_timer) &&
1644 !tbase_get_deferrable(timer->base))
1645 new_base->next_timer = timer->expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001646 internal_add_timer(new_base, timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648}
1649
Randy Dunlap48ccf3d2008-01-21 17:18:25 -08001650static void __cpuinit migrate_timers(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651{
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001652 struct tvec_base *old_base;
1653 struct tvec_base *new_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654 int i;
1655
1656 BUG_ON(cpu_online(cpu));
Jan Beulicha4a61982006-03-24 03:15:54 -08001657 old_base = per_cpu(tvec_bases, cpu);
1658 new_base = get_cpu_var(tvec_bases);
Oleg Nesterovd82f0b02008-08-20 16:46:04 -07001659 /*
1660 * The caller is globally serialized and nobody else
1661 * takes two locks at once, deadlock is not possible.
1662 */
1663 spin_lock_irq(&new_base->lock);
Oleg Nesterov0d180402008-04-04 20:54:10 +02001664 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665
Oleg Nesterov3691c512006-03-31 02:30:30 -08001666 BUG_ON(old_base->running_timer);
1667
Linus Torvalds1da177e2005-04-16 15:20:36 -07001668 for (i = 0; i < TVR_SIZE; i++)
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001669 migrate_timer_list(new_base, old_base->tv1.vec + i);
1670 for (i = 0; i < TVN_SIZE; i++) {
1671 migrate_timer_list(new_base, old_base->tv2.vec + i);
1672 migrate_timer_list(new_base, old_base->tv3.vec + i);
1673 migrate_timer_list(new_base, old_base->tv4.vec + i);
1674 migrate_timer_list(new_base, old_base->tv5.vec + i);
1675 }
1676
Oleg Nesterov0d180402008-04-04 20:54:10 +02001677 spin_unlock(&old_base->lock);
Oleg Nesterovd82f0b02008-08-20 16:46:04 -07001678 spin_unlock_irq(&new_base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001679 put_cpu_var(tvec_bases);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001680}
1681#endif /* CONFIG_HOTPLUG_CPU */
1682
Chandra Seetharaman8c78f302006-07-30 03:03:35 -07001683static int __cpuinit timer_cpu_notify(struct notifier_block *self,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684 unsigned long action, void *hcpu)
1685{
1686 long cpu = (long)hcpu;
Akinobu Mita80b51842010-05-26 14:43:32 -07001687 int err;
1688
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689 switch(action) {
1690 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001691 case CPU_UP_PREPARE_FROZEN:
Akinobu Mita80b51842010-05-26 14:43:32 -07001692 err = init_timers_cpu(cpu);
1693 if (err < 0)
1694 return notifier_from_errno(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695 break;
1696#ifdef CONFIG_HOTPLUG_CPU
1697 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001698 case CPU_DEAD_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001699 migrate_timers(cpu);
1700 break;
1701#endif
1702 default:
1703 break;
1704 }
1705 return NOTIFY_OK;
1706}
1707
Chandra Seetharaman8c78f302006-07-30 03:03:35 -07001708static struct notifier_block __cpuinitdata timers_nb = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001709 .notifier_call = timer_cpu_notify,
1710};
1711
1712
1713void __init init_timers(void)
1714{
Akinobu Mita07dccf32006-09-29 02:00:22 -07001715 int err = timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716 (void *)(long)smp_processor_id());
Akinobu Mita07dccf32006-09-29 02:00:22 -07001717
Ingo Molnar82f67cd2007-02-16 01:28:13 -08001718 init_timer_stats();
1719
Akinobu Mita9e506f72010-06-04 14:15:04 -07001720 BUG_ON(err != NOTIFY_OK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001721 register_cpu_notifier(&timers_nb);
Carlos R. Mafra962cf362008-05-15 11:15:37 -03001722 open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001723}
1724
Linus Torvalds1da177e2005-04-16 15:20:36 -07001725/**
1726 * msleep - sleep safely even with waitqueue interruptions
1727 * @msecs: Time in milliseconds to sleep for
1728 */
1729void msleep(unsigned int msecs)
1730{
1731 unsigned long timeout = msecs_to_jiffies(msecs) + 1;
1732
Nishanth Aravamudan75bcc8c2005-09-10 00:27:24 -07001733 while (timeout)
1734 timeout = schedule_timeout_uninterruptible(timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735}
1736
1737EXPORT_SYMBOL(msleep);
1738
1739/**
Domen Puncer96ec3ef2005-06-25 14:58:43 -07001740 * msleep_interruptible - sleep waiting for signals
Linus Torvalds1da177e2005-04-16 15:20:36 -07001741 * @msecs: Time in milliseconds to sleep for
1742 */
1743unsigned long msleep_interruptible(unsigned int msecs)
1744{
1745 unsigned long timeout = msecs_to_jiffies(msecs) + 1;
1746
Nishanth Aravamudan75bcc8c2005-09-10 00:27:24 -07001747 while (timeout && !signal_pending(current))
1748 timeout = schedule_timeout_interruptible(timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001749 return jiffies_to_msecs(timeout);
1750}
1751
1752EXPORT_SYMBOL(msleep_interruptible);