blob: 327d2deb44515b438c8e43cec56415d85b22342d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/workqueue.c
3 *
4 * Generic mechanism for defining kernel helper threads for running
5 * arbitrary tasks in process context.
6 *
7 * Started by Ingo Molnar, Copyright (C) 2002
8 *
9 * Derived from the taskqueue/keventd code by:
10 *
11 * David Woodhouse <dwmw2@infradead.org>
Francois Camie1f8e872008-10-15 22:01:59 -070012 * Andrew Morton
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
14 * Theodore Ts'o <tytso@mit.edu>
Christoph Lameter89ada672005-10-30 15:01:59 -080015 *
Christoph Lametercde53532008-07-04 09:59:22 -070016 * Made to use alloc_percpu by Christoph Lameter.
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
18
19#include <linux/module.h>
20#include <linux/kernel.h>
21#include <linux/sched.h>
22#include <linux/init.h>
23#include <linux/signal.h>
24#include <linux/completion.h>
25#include <linux/workqueue.h>
26#include <linux/slab.h>
27#include <linux/cpu.h>
28#include <linux/notifier.h>
29#include <linux/kthread.h>
James Bottomley1fa44ec2006-02-23 12:43:43 -060030#include <linux/hardirq.h>
Christoph Lameter469340232006-10-11 01:21:26 -070031#include <linux/mempolicy.h>
Rafael J. Wysocki341a5952006-12-06 20:34:49 -080032#include <linux/freezer.h>
Peter Zijlstrad5abe662006-12-06 20:37:26 -080033#include <linux/kallsyms.h>
34#include <linux/debug_locks.h>
Johannes Berg4e6045f2007-10-18 23:39:55 -070035#include <linux/lockdep.h>
Zhaoleifb39125f2009-04-17 15:15:51 +080036#define CREATE_TRACE_POINTS
37#include <trace/events/workqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038
39/*
Nathan Lynchf756d5e2006-01-08 01:05:12 -080040 * The per-CPU workqueue (if single thread, we always use the first
41 * possible cpu).
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 */
43struct cpu_workqueue_struct {
44
45 spinlock_t lock;
46
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 struct list_head worklist;
48 wait_queue_head_t more_work;
Oleg Nesterov3af244332007-05-09 02:34:09 -070049 struct work_struct *current_work;
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51 struct workqueue_struct *wq;
Ingo Molnar36c8b582006-07-03 00:25:41 -070052 struct task_struct *thread;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053} ____cacheline_aligned;
54
55/*
56 * The externally visible workqueue abstraction is an array of
57 * per-CPU workqueues:
58 */
59struct workqueue_struct {
Christoph Lameter89ada672005-10-30 15:01:59 -080060 struct cpu_workqueue_struct *cpu_wq;
Oleg Nesterovcce1a162007-05-09 02:34:13 -070061 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 const char *name;
Oleg Nesterovcce1a162007-05-09 02:34:13 -070063 int singlethread;
Oleg Nesterov319c2a92007-05-09 02:34:06 -070064 int freezeable; /* Freeze threads during suspend */
Heiko Carstens0d557dc2008-10-13 23:50:09 +020065 int rt;
Johannes Berg4e6045f2007-10-18 23:39:55 -070066#ifdef CONFIG_LOCKDEP
67 struct lockdep_map lockdep_map;
68#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070069};
70
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +090071#ifdef CONFIG_DEBUG_OBJECTS_WORK
72
73static struct debug_obj_descr work_debug_descr;
74
75/*
76 * fixup_init is called when:
77 * - an active object is initialized
78 */
79static int work_fixup_init(void *addr, enum debug_obj_state state)
80{
81 struct work_struct *work = addr;
82
83 switch (state) {
84 case ODEBUG_STATE_ACTIVE:
85 cancel_work_sync(work);
86 debug_object_init(work, &work_debug_descr);
87 return 1;
88 default:
89 return 0;
90 }
91}
92
93/*
94 * fixup_activate is called when:
95 * - an active object is activated
96 * - an unknown object is activated (might be a statically initialized object)
97 */
98static int work_fixup_activate(void *addr, enum debug_obj_state state)
99{
100 struct work_struct *work = addr;
101
102 switch (state) {
103
104 case ODEBUG_STATE_NOTAVAILABLE:
105 /*
106 * This is not really a fixup. The work struct was
107 * statically initialized. We just make sure that it
108 * is tracked in the object tracker.
109 */
110 if (test_bit(WORK_STRUCT_STATIC, work_data_bits(work))) {
111 debug_object_init(work, &work_debug_descr);
112 debug_object_activate(work, &work_debug_descr);
113 return 0;
114 }
115 WARN_ON_ONCE(1);
116 return 0;
117
118 case ODEBUG_STATE_ACTIVE:
119 WARN_ON(1);
120
121 default:
122 return 0;
123 }
124}
125
126/*
127 * fixup_free is called when:
128 * - an active object is freed
129 */
130static int work_fixup_free(void *addr, enum debug_obj_state state)
131{
132 struct work_struct *work = addr;
133
134 switch (state) {
135 case ODEBUG_STATE_ACTIVE:
136 cancel_work_sync(work);
137 debug_object_free(work, &work_debug_descr);
138 return 1;
139 default:
140 return 0;
141 }
142}
143
144static struct debug_obj_descr work_debug_descr = {
145 .name = "work_struct",
146 .fixup_init = work_fixup_init,
147 .fixup_activate = work_fixup_activate,
148 .fixup_free = work_fixup_free,
149};
150
151static inline void debug_work_activate(struct work_struct *work)
152{
153 debug_object_activate(work, &work_debug_descr);
154}
155
156static inline void debug_work_deactivate(struct work_struct *work)
157{
158 debug_object_deactivate(work, &work_debug_descr);
159}
160
161void __init_work(struct work_struct *work, int onstack)
162{
163 if (onstack)
164 debug_object_init_on_stack(work, &work_debug_descr);
165 else
166 debug_object_init(work, &work_debug_descr);
167}
168EXPORT_SYMBOL_GPL(__init_work);
169
170void destroy_work_on_stack(struct work_struct *work)
171{
172 debug_object_free(work, &work_debug_descr);
173}
174EXPORT_SYMBOL_GPL(destroy_work_on_stack);
175
176#else
177static inline void debug_work_activate(struct work_struct *work) { }
178static inline void debug_work_deactivate(struct work_struct *work) { }
179#endif
180
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100181/* Serializes the accesses to the list of workqueues. */
182static DEFINE_SPINLOCK(workqueue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183static LIST_HEAD(workqueues);
184
Oleg Nesterov3af244332007-05-09 02:34:09 -0700185static int singlethread_cpu __read_mostly;
Rusty Russelle7577c52009-01-01 10:12:25 +1030186static const struct cpumask *cpu_singlethread_map __read_mostly;
Oleg Nesterov14441962007-05-23 13:57:57 -0700187/*
188 * _cpu_down() first removes CPU from cpu_online_map, then CPU_DEAD
189 * flushes cwq->worklist. This means that flush_workqueue/wait_on_work
190 * which comes in between can't use for_each_online_cpu(). We could
191 * use cpu_possible_map, the cpumask below is more a documentation
192 * than optimization.
193 */
Rusty Russelle7577c52009-01-01 10:12:25 +1030194static cpumask_var_t cpu_populated_map __read_mostly;
Nathan Lynchf756d5e2006-01-08 01:05:12 -0800195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196/* If it's single threaded, it isn't in the list of workqueues. */
David Howells6cc88bc2008-11-14 10:39:21 +1100197static inline int is_wq_single_threaded(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198{
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700199 return wq->singlethread;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200}
201
Rusty Russelle7577c52009-01-01 10:12:25 +1030202static const struct cpumask *wq_cpu_map(struct workqueue_struct *wq)
Oleg Nesterovb1f4ec172007-05-09 02:34:12 -0700203{
David Howells6cc88bc2008-11-14 10:39:21 +1100204 return is_wq_single_threaded(wq)
Rusty Russelle7577c52009-01-01 10:12:25 +1030205 ? cpu_singlethread_map : cpu_populated_map;
Oleg Nesterovb1f4ec172007-05-09 02:34:12 -0700206}
207
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700208static
209struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu)
210{
David Howells6cc88bc2008-11-14 10:39:21 +1100211 if (unlikely(is_wq_single_threaded(wq)))
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700212 cpu = singlethread_cpu;
213 return per_cpu_ptr(wq->cpu_wq, cpu);
214}
215
David Howells4594bf12006-12-07 11:33:26 +0000216/*
217 * Set the workqueue on which a work item is to be run
218 * - Must *only* be called if the pending flag is set
219 */
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700220static inline void set_wq_data(struct work_struct *work,
221 struct cpu_workqueue_struct *cwq)
David Howells365970a2006-11-22 14:54:49 +0000222{
David Howells4594bf12006-12-07 11:33:26 +0000223 unsigned long new;
David Howells365970a2006-11-22 14:54:49 +0000224
David Howells4594bf12006-12-07 11:33:26 +0000225 BUG_ON(!work_pending(work));
226
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700227 new = (unsigned long) cwq | (1UL << WORK_STRUCT_PENDING);
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800228 new |= WORK_STRUCT_FLAG_MASK & *work_data_bits(work);
229 atomic_long_set(&work->data, new);
David Howells365970a2006-11-22 14:54:49 +0000230}
231
Oleg Nesterov4d707b92010-04-23 17:40:40 +0200232/*
233 * Clear WORK_STRUCT_PENDING and the workqueue on which it was queued.
234 */
235static inline void clear_wq_data(struct work_struct *work)
236{
237 unsigned long flags = *work_data_bits(work) &
238 (1UL << WORK_STRUCT_STATIC);
239 atomic_long_set(&work->data, flags);
240}
241
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700242static inline
243struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
David Howells365970a2006-11-22 14:54:49 +0000244{
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800245 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
David Howells365970a2006-11-22 14:54:49 +0000246}
247
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700248static void insert_work(struct cpu_workqueue_struct *cwq,
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700249 struct work_struct *work, struct list_head *head)
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700250{
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +0100251 trace_workqueue_insertion(cwq->thread, work);
252
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700253 set_wq_data(work, cwq);
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700254 /*
255 * Ensure that we get the right work->data if we see the
256 * result of list_add() below, see try_to_grab_pending().
257 */
258 smp_wmb();
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700259 list_add_tail(&work->entry, head);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700260 wake_up(&cwq->more_work);
261}
262
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263static void __queue_work(struct cpu_workqueue_struct *cwq,
264 struct work_struct *work)
265{
266 unsigned long flags;
267
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900268 debug_work_activate(work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 spin_lock_irqsave(&cwq->lock, flags);
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700270 insert_work(cwq, work, &cwq->worklist);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 spin_unlock_irqrestore(&cwq->lock, flags);
272}
273
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700274/**
275 * queue_work - queue work on a workqueue
276 * @wq: workqueue to use
277 * @work: work to queue
278 *
Alan Stern057647f2006-10-28 10:38:58 -0700279 * Returns 0 if @work was already on a queue, non-zero otherwise.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 *
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -0700281 * We queue the work to the CPU on which it was submitted, but if the CPU dies
282 * it can be processed by another CPU.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800284int queue_work(struct workqueue_struct *wq, struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285{
Oleg Nesterovef1ca232008-07-25 01:47:53 -0700286 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287
Oleg Nesterovef1ca232008-07-25 01:47:53 -0700288 ret = queue_work_on(get_cpu(), wq, work);
289 put_cpu();
290
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 return ret;
292}
Dave Jonesae90dd52006-06-30 01:40:45 -0400293EXPORT_SYMBOL_GPL(queue_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294
Zhang Ruic1a220e2008-07-23 21:28:39 -0700295/**
296 * queue_work_on - queue work on specific cpu
297 * @cpu: CPU number to execute work on
298 * @wq: workqueue to use
299 * @work: work to queue
300 *
301 * Returns 0 if @work was already on a queue, non-zero otherwise.
302 *
303 * We queue the work to a specific CPU, the caller must ensure it
304 * can't go away.
305 */
306int
307queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work)
308{
309 int ret = 0;
310
311 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
312 BUG_ON(!list_empty(&work->entry));
313 __queue_work(wq_per_cpu(wq, cpu), work);
314 ret = 1;
315 }
316 return ret;
317}
318EXPORT_SYMBOL_GPL(queue_work_on);
319
Li Zefan6d141c32008-02-08 04:21:09 -0800320static void delayed_work_timer_fn(unsigned long __data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321{
David Howells52bad642006-11-22 14:54:01 +0000322 struct delayed_work *dwork = (struct delayed_work *)__data;
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700323 struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work);
324 struct workqueue_struct *wq = cwq->wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700326 __queue_work(wq_per_cpu(wq, smp_processor_id()), &dwork->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327}
328
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700329/**
330 * queue_delayed_work - queue work on a workqueue after delay
331 * @wq: workqueue to use
Randy Dunlapaf9997e2006-12-22 01:06:52 -0800332 * @dwork: delayable work to queue
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700333 * @delay: number of jiffies to wait before queueing
334 *
Alan Stern057647f2006-10-28 10:38:58 -0700335 * Returns 0 if @work was already on a queue, non-zero otherwise.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700336 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800337int queue_delayed_work(struct workqueue_struct *wq,
David Howells52bad642006-11-22 14:54:01 +0000338 struct delayed_work *dwork, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339{
David Howells52bad642006-11-22 14:54:01 +0000340 if (delay == 0)
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700341 return queue_work(wq, &dwork->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700343 return queue_delayed_work_on(-1, wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344}
Dave Jonesae90dd52006-06-30 01:40:45 -0400345EXPORT_SYMBOL_GPL(queue_delayed_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700347/**
348 * queue_delayed_work_on - queue work on specific CPU after delay
349 * @cpu: CPU number to execute work on
350 * @wq: workqueue to use
Randy Dunlapaf9997e2006-12-22 01:06:52 -0800351 * @dwork: work to queue
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700352 * @delay: number of jiffies to wait before queueing
353 *
Alan Stern057647f2006-10-28 10:38:58 -0700354 * Returns 0 if @work was already on a queue, non-zero otherwise.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700355 */
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700356int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
David Howells52bad642006-11-22 14:54:01 +0000357 struct delayed_work *dwork, unsigned long delay)
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700358{
359 int ret = 0;
David Howells52bad642006-11-22 14:54:01 +0000360 struct timer_list *timer = &dwork->timer;
361 struct work_struct *work = &dwork->work;
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700362
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800363 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700364 BUG_ON(timer_pending(timer));
365 BUG_ON(!list_empty(&work->entry));
366
Andrew Liu8a3e77c2008-05-01 04:35:14 -0700367 timer_stats_timer_set_start_info(&dwork->timer);
368
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700369 /* This stores cwq for the moment, for the timer_fn */
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700370 set_wq_data(work, wq_per_cpu(wq, raw_smp_processor_id()));
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700371 timer->expires = jiffies + delay;
David Howells52bad642006-11-22 14:54:01 +0000372 timer->data = (unsigned long)dwork;
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700373 timer->function = delayed_work_timer_fn;
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700374
375 if (unlikely(cpu >= 0))
376 add_timer_on(timer, cpu);
377 else
378 add_timer(timer);
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700379 ret = 1;
380 }
381 return ret;
382}
Dave Jonesae90dd52006-06-30 01:40:45 -0400383EXPORT_SYMBOL_GPL(queue_delayed_work_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384
Arjan van de Ven858119e2006-01-14 13:20:43 -0800385static void run_workqueue(struct cpu_workqueue_struct *cwq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386{
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700387 spin_lock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 while (!list_empty(&cwq->worklist)) {
389 struct work_struct *work = list_entry(cwq->worklist.next,
390 struct work_struct, entry);
David Howells6bb49e52006-11-22 14:54:45 +0000391 work_func_t f = work->func;
Johannes Berg4e6045f2007-10-18 23:39:55 -0700392#ifdef CONFIG_LOCKDEP
393 /*
394 * It is permissible to free the struct work_struct
395 * from inside the function that is called from it,
396 * this we need to take into account for lockdep too.
397 * To avoid bogus "held lock freed" warnings as well
398 * as problems when looking into work->lockdep_map,
399 * make a copy and use that here.
400 */
401 struct lockdep_map lockdep_map = work->lockdep_map;
402#endif
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +0100403 trace_workqueue_execution(cwq->thread, work);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900404 debug_work_deactivate(work);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700405 cwq->current_work = work;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 list_del_init(cwq->worklist.next);
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700407 spin_unlock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408
David Howells365970a2006-11-22 14:54:49 +0000409 BUG_ON(get_wq_data(work) != cwq);
Oleg Nesterov23b2e592007-05-09 02:34:19 -0700410 work_clear_pending(work);
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200411 lock_map_acquire(&cwq->wq->lockdep_map);
412 lock_map_acquire(&lockdep_map);
David Howells65f27f32006-11-22 14:55:48 +0000413 f(work);
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200414 lock_map_release(&lockdep_map);
415 lock_map_release(&cwq->wq->lockdep_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416
Peter Zijlstrad5abe662006-12-06 20:37:26 -0800417 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
418 printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
419 "%s/0x%08x/%d\n",
420 current->comm, preempt_count(),
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700421 task_pid_nr(current));
Peter Zijlstrad5abe662006-12-06 20:37:26 -0800422 printk(KERN_ERR " last function: ");
423 print_symbol("%s\n", (unsigned long)f);
424 debug_show_held_locks(current);
425 dump_stack();
426 }
427
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700428 spin_lock_irq(&cwq->lock);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700429 cwq->current_work = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 }
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700431 spin_unlock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432}
433
434static int worker_thread(void *__cwq)
435{
436 struct cpu_workqueue_struct *cwq = __cwq;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700437 DEFINE_WAIT(wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438
Rafael J. Wysocki83144182007-07-17 04:03:35 -0700439 if (cwq->wq->freezeable)
440 set_freezable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441
Oleg Nesterov3af244332007-05-09 02:34:09 -0700442 for (;;) {
Oleg Nesterov3af244332007-05-09 02:34:09 -0700443 prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
Oleg Nesterov14441962007-05-23 13:57:57 -0700444 if (!freezing(current) &&
445 !kthread_should_stop() &&
446 list_empty(&cwq->worklist))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 schedule();
Oleg Nesterov3af244332007-05-09 02:34:09 -0700448 finish_wait(&cwq->more_work, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449
Oleg Nesterov85f41862007-05-09 02:34:20 -0700450 try_to_freeze();
451
Oleg Nesterov14441962007-05-23 13:57:57 -0700452 if (kthread_should_stop())
Oleg Nesterov3af244332007-05-09 02:34:09 -0700453 break;
454
455 run_workqueue(cwq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 }
Oleg Nesterov3af244332007-05-09 02:34:09 -0700457
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 return 0;
459}
460
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700461struct wq_barrier {
462 struct work_struct work;
463 struct completion done;
464};
465
466static void wq_barrier_func(struct work_struct *work)
467{
468 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
469 complete(&barr->done);
470}
471
Oleg Nesterov83c22522007-05-09 02:33:54 -0700472static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700473 struct wq_barrier *barr, struct list_head *head)
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700474{
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900475 /*
476 * debugobject calls are safe here even with cwq->lock locked
477 * as we know for sure that this will not trigger any of the
478 * checks and call back into the fixup functions where we
479 * might deadlock.
480 */
481 INIT_WORK_ON_STACK(&barr->work, wq_barrier_func);
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700482 __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work));
483
484 init_completion(&barr->done);
Oleg Nesterov83c22522007-05-09 02:33:54 -0700485
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900486 debug_work_activate(&barr->work);
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700487 insert_work(cwq, &barr->work, head);
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700488}
489
Oleg Nesterov14441962007-05-23 13:57:57 -0700490static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491{
Lai Jiangshan2355b702009-04-02 16:58:24 -0700492 int active = 0;
493 struct wq_barrier barr;
Oleg Nesterov14441962007-05-23 13:57:57 -0700494
Lai Jiangshan2355b702009-04-02 16:58:24 -0700495 WARN_ON(cwq->thread == current);
496
497 spin_lock_irq(&cwq->lock);
498 if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) {
499 insert_wq_barrier(cwq, &barr, &cwq->worklist);
Oleg Nesterov14441962007-05-23 13:57:57 -0700500 active = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 }
Lai Jiangshan2355b702009-04-02 16:58:24 -0700502 spin_unlock_irq(&cwq->lock);
503
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900504 if (active) {
Lai Jiangshan2355b702009-04-02 16:58:24 -0700505 wait_for_completion(&barr.done);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900506 destroy_work_on_stack(&barr.work);
507 }
Oleg Nesterov14441962007-05-23 13:57:57 -0700508
509 return active;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510}
511
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700512/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 * flush_workqueue - ensure that any scheduled work has run to completion.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700514 * @wq: workqueue to flush
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 *
516 * Forces execution of the workqueue and blocks until its completion.
517 * This is typically used in driver shutdown handlers.
518 *
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700519 * We sleep until all works which were queued on entry have been handled,
520 * but we are not livelocked by new incoming ones.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 *
522 * This function used to run the workqueues itself. Now we just wait for the
523 * helper threads to do it.
524 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800525void flush_workqueue(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526{
Rusty Russelle7577c52009-01-01 10:12:25 +1030527 const struct cpumask *cpu_map = wq_cpu_map(wq);
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700528 int cpu;
Oleg Nesterovb1f4ec172007-05-09 02:34:12 -0700529
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700530 might_sleep();
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200531 lock_map_acquire(&wq->lockdep_map);
532 lock_map_release(&wq->lockdep_map);
Rusty Russellaa85ea52009-03-30 22:05:15 -0600533 for_each_cpu(cpu, cpu_map)
Oleg Nesterovb1f4ec172007-05-09 02:34:12 -0700534 flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535}
Dave Jonesae90dd52006-06-30 01:40:45 -0400536EXPORT_SYMBOL_GPL(flush_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537
Oleg Nesterovdb700892008-07-25 01:47:49 -0700538/**
539 * flush_work - block until a work_struct's callback has terminated
540 * @work: the work which is to be flushed
541 *
Oleg Nesterova67da702008-07-25 01:47:52 -0700542 * Returns false if @work has already terminated.
543 *
Oleg Nesterovdb700892008-07-25 01:47:49 -0700544 * It is expected that, prior to calling flush_work(), the caller has
545 * arranged for the work to not be requeued, otherwise it doesn't make
546 * sense to use this function.
547 */
548int flush_work(struct work_struct *work)
549{
550 struct cpu_workqueue_struct *cwq;
551 struct list_head *prev;
552 struct wq_barrier barr;
553
554 might_sleep();
555 cwq = get_wq_data(work);
556 if (!cwq)
557 return 0;
558
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200559 lock_map_acquire(&cwq->wq->lockdep_map);
560 lock_map_release(&cwq->wq->lockdep_map);
Oleg Nesterova67da702008-07-25 01:47:52 -0700561
Oleg Nesterovdb700892008-07-25 01:47:49 -0700562 prev = NULL;
563 spin_lock_irq(&cwq->lock);
564 if (!list_empty(&work->entry)) {
565 /*
566 * See the comment near try_to_grab_pending()->smp_rmb().
567 * If it was re-queued under us we are not going to wait.
568 */
569 smp_rmb();
570 if (unlikely(cwq != get_wq_data(work)))
571 goto out;
572 prev = &work->entry;
573 } else {
574 if (cwq->current_work != work)
575 goto out;
576 prev = &cwq->worklist;
577 }
578 insert_wq_barrier(cwq, &barr, prev->next);
579out:
580 spin_unlock_irq(&cwq->lock);
581 if (!prev)
582 return 0;
583
584 wait_for_completion(&barr.done);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900585 destroy_work_on_stack(&barr.work);
Oleg Nesterovdb700892008-07-25 01:47:49 -0700586 return 1;
587}
588EXPORT_SYMBOL_GPL(flush_work);
589
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700590/*
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700591 * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700592 * so this work can't be re-armed in any way.
593 */
594static int try_to_grab_pending(struct work_struct *work)
595{
596 struct cpu_workqueue_struct *cwq;
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700597 int ret = -1;
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700598
599 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work)))
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700600 return 0;
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700601
602 /*
603 * The queueing is in progress, or it is already queued. Try to
604 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
605 */
606
607 cwq = get_wq_data(work);
608 if (!cwq)
609 return ret;
610
611 spin_lock_irq(&cwq->lock);
612 if (!list_empty(&work->entry)) {
613 /*
614 * This work is queued, but perhaps we locked the wrong cwq.
615 * In that case we must see the new value after rmb(), see
616 * insert_work()->wmb().
617 */
618 smp_rmb();
619 if (cwq == get_wq_data(work)) {
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900620 debug_work_deactivate(work);
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700621 list_del_init(&work->entry);
622 ret = 1;
623 }
624 }
625 spin_unlock_irq(&cwq->lock);
626
627 return ret;
628}
629
630static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq,
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700631 struct work_struct *work)
632{
633 struct wq_barrier barr;
634 int running = 0;
635
636 spin_lock_irq(&cwq->lock);
637 if (unlikely(cwq->current_work == work)) {
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700638 insert_wq_barrier(cwq, &barr, cwq->worklist.next);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700639 running = 1;
640 }
641 spin_unlock_irq(&cwq->lock);
642
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900643 if (unlikely(running)) {
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700644 wait_for_completion(&barr.done);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900645 destroy_work_on_stack(&barr.work);
646 }
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700647}
648
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700649static void wait_on_work(struct work_struct *work)
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700650{
651 struct cpu_workqueue_struct *cwq;
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700652 struct workqueue_struct *wq;
Rusty Russelle7577c52009-01-01 10:12:25 +1030653 const struct cpumask *cpu_map;
Oleg Nesterovb1f4ec172007-05-09 02:34:12 -0700654 int cpu;
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700655
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700656 might_sleep();
657
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200658 lock_map_acquire(&work->lockdep_map);
659 lock_map_release(&work->lockdep_map);
Johannes Berg4e6045f2007-10-18 23:39:55 -0700660
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700661 cwq = get_wq_data(work);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700662 if (!cwq)
Oleg Nesterov3af244332007-05-09 02:34:09 -0700663 return;
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700664
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700665 wq = cwq->wq;
666 cpu_map = wq_cpu_map(wq);
667
Rusty Russellaa85ea52009-03-30 22:05:15 -0600668 for_each_cpu(cpu, cpu_map)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700669 wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work);
670}
671
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700672static int __cancel_work_timer(struct work_struct *work,
673 struct timer_list* timer)
674{
675 int ret;
676
677 do {
678 ret = (timer && likely(del_timer(timer)));
679 if (!ret)
680 ret = try_to_grab_pending(work);
681 wait_on_work(work);
682 } while (unlikely(ret < 0));
683
Oleg Nesterov4d707b92010-04-23 17:40:40 +0200684 clear_wq_data(work);
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700685 return ret;
686}
687
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700688/**
689 * cancel_work_sync - block until a work_struct's callback has terminated
690 * @work: the work which is to be flushed
691 *
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700692 * Returns true if @work was pending.
693 *
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700694 * cancel_work_sync() will cancel the work if it is queued. If the work's
695 * callback appears to be running, cancel_work_sync() will block until it
696 * has completed.
697 *
698 * It is possible to use this function if the work re-queues itself. It can
699 * cancel the work even if it migrates to another workqueue, however in that
700 * case it only guarantees that work->func() has completed on the last queued
701 * workqueue.
702 *
703 * cancel_work_sync(&delayed_work->work) should be used only if ->timer is not
704 * pending, otherwise it goes into a busy-wait loop until the timer expires.
705 *
706 * The caller must ensure that workqueue_struct on which this work was last
707 * queued can't be destroyed before this function returns.
708 */
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700709int cancel_work_sync(struct work_struct *work)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700710{
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700711 return __cancel_work_timer(work, NULL);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700712}
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700713EXPORT_SYMBOL_GPL(cancel_work_sync);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700714
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700715/**
Oleg Nesterovf5a421a2007-07-15 23:41:44 -0700716 * cancel_delayed_work_sync - reliably kill off a delayed work.
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700717 * @dwork: the delayed work struct
718 *
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700719 * Returns true if @dwork was pending.
720 *
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700721 * It is possible to use this function if @dwork rearms itself via queue_work()
722 * or queue_delayed_work(). See also the comment for cancel_work_sync().
723 */
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700724int cancel_delayed_work_sync(struct delayed_work *dwork)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700725{
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700726 return __cancel_work_timer(&dwork->work, &dwork->timer);
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700727}
Oleg Nesterovf5a421a2007-07-15 23:41:44 -0700728EXPORT_SYMBOL(cancel_delayed_work_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700730static struct workqueue_struct *keventd_wq __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700732/**
733 * schedule_work - put work task in global workqueue
734 * @work: job to be done
735 *
Bart Van Assche5b0f437d2009-07-30 19:00:53 +0200736 * Returns zero if @work was already on the kernel-global workqueue and
737 * non-zero otherwise.
738 *
739 * This puts a job in the kernel-global workqueue if it was not already
740 * queued and leaves it in the same position on the kernel-global
741 * workqueue otherwise.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700742 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800743int schedule_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744{
745 return queue_work(keventd_wq, work);
746}
Dave Jonesae90dd52006-06-30 01:40:45 -0400747EXPORT_SYMBOL(schedule_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748
Zhang Ruic1a220e2008-07-23 21:28:39 -0700749/*
750 * schedule_work_on - put work task on a specific cpu
751 * @cpu: cpu to put the work task on
752 * @work: job to be done
753 *
754 * This puts a job on a specific cpu
755 */
756int schedule_work_on(int cpu, struct work_struct *work)
757{
758 return queue_work_on(cpu, keventd_wq, work);
759}
760EXPORT_SYMBOL(schedule_work_on);
761
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700762/**
763 * schedule_delayed_work - put work task in global workqueue after delay
David Howells52bad642006-11-22 14:54:01 +0000764 * @dwork: job to be done
765 * @delay: number of jiffies to wait or 0 for immediate execution
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700766 *
767 * After waiting for a given time this puts a job in the kernel-global
768 * workqueue.
769 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800770int schedule_delayed_work(struct delayed_work *dwork,
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800771 unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772{
David Howells52bad642006-11-22 14:54:01 +0000773 return queue_delayed_work(keventd_wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774}
Dave Jonesae90dd52006-06-30 01:40:45 -0400775EXPORT_SYMBOL(schedule_delayed_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700777/**
Linus Torvalds8c53e462009-10-14 09:16:42 -0700778 * flush_delayed_work - block until a dwork_struct's callback has terminated
779 * @dwork: the delayed work which is to be flushed
780 *
781 * Any timeout is cancelled, and any pending work is run immediately.
782 */
783void flush_delayed_work(struct delayed_work *dwork)
784{
785 if (del_timer_sync(&dwork->timer)) {
786 struct cpu_workqueue_struct *cwq;
Oleg Nesterov47dd5be2010-04-30 07:23:51 +0200787 cwq = wq_per_cpu(get_wq_data(&dwork->work)->wq, get_cpu());
Linus Torvalds8c53e462009-10-14 09:16:42 -0700788 __queue_work(cwq, &dwork->work);
789 put_cpu();
790 }
791 flush_work(&dwork->work);
792}
793EXPORT_SYMBOL(flush_delayed_work);
794
795/**
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700796 * schedule_delayed_work_on - queue work in global workqueue on CPU after delay
797 * @cpu: cpu to use
David Howells52bad642006-11-22 14:54:01 +0000798 * @dwork: job to be done
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700799 * @delay: number of jiffies to wait
800 *
801 * After waiting for a given time this puts a job in the kernel-global
802 * workqueue on the specified CPU.
803 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804int schedule_delayed_work_on(int cpu,
David Howells52bad642006-11-22 14:54:01 +0000805 struct delayed_work *dwork, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806{
David Howells52bad642006-11-22 14:54:01 +0000807 return queue_delayed_work_on(cpu, keventd_wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808}
Dave Jonesae90dd52006-06-30 01:40:45 -0400809EXPORT_SYMBOL(schedule_delayed_work_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810
Andrew Mortonb6136772006-06-25 05:47:49 -0700811/**
812 * schedule_on_each_cpu - call a function on each online CPU from keventd
813 * @func: the function to call
Andrew Mortonb6136772006-06-25 05:47:49 -0700814 *
815 * Returns zero on success.
816 * Returns -ve errno on failure.
817 *
Andrew Mortonb6136772006-06-25 05:47:49 -0700818 * schedule_on_each_cpu() is very slow.
819 */
David Howells65f27f32006-11-22 14:55:48 +0000820int schedule_on_each_cpu(work_func_t func)
Christoph Lameter15316ba2006-01-08 01:00:43 -0800821{
822 int cpu;
Andi Kleen65a64462009-10-14 06:22:47 +0200823 int orig = -1;
Andrew Mortonb6136772006-06-25 05:47:49 -0700824 struct work_struct *works;
Christoph Lameter15316ba2006-01-08 01:00:43 -0800825
Andrew Mortonb6136772006-06-25 05:47:49 -0700826 works = alloc_percpu(struct work_struct);
827 if (!works)
Christoph Lameter15316ba2006-01-08 01:00:43 -0800828 return -ENOMEM;
Andrew Mortonb6136772006-06-25 05:47:49 -0700829
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100830 get_online_cpus();
Tejun Heo93981802009-11-17 14:06:20 -0800831
832 /*
833 * When running in keventd don't schedule a work item on
834 * itself. Can just call directly because the work queue is
835 * already bound. This also is faster.
836 */
837 if (current_is_keventd())
838 orig = raw_smp_processor_id();
839
Christoph Lameter15316ba2006-01-08 01:00:43 -0800840 for_each_online_cpu(cpu) {
Ingo Molnar9bfb1832006-12-18 20:05:09 +0100841 struct work_struct *work = per_cpu_ptr(works, cpu);
842
843 INIT_WORK(work, func);
Andi Kleen65a64462009-10-14 06:22:47 +0200844 if (cpu != orig)
Tejun Heo93981802009-11-17 14:06:20 -0800845 schedule_work_on(cpu, work);
Andi Kleen65a64462009-10-14 06:22:47 +0200846 }
Tejun Heo93981802009-11-17 14:06:20 -0800847 if (orig >= 0)
848 func(per_cpu_ptr(works, orig));
849
850 for_each_online_cpu(cpu)
851 flush_work(per_cpu_ptr(works, cpu));
852
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100853 put_online_cpus();
Andrew Mortonb6136772006-06-25 05:47:49 -0700854 free_percpu(works);
Christoph Lameter15316ba2006-01-08 01:00:43 -0800855 return 0;
856}
857
Alan Sterneef6a7d2010-02-12 17:39:21 +0900858/**
859 * flush_scheduled_work - ensure that any scheduled work has run to completion.
860 *
861 * Forces execution of the kernel-global workqueue and blocks until its
862 * completion.
863 *
864 * Think twice before calling this function! It's very easy to get into
865 * trouble if you don't take great care. Either of the following situations
866 * will lead to deadlock:
867 *
868 * One of the work items currently on the workqueue needs to acquire
869 * a lock held by your code or its caller.
870 *
871 * Your code is running in the context of a work routine.
872 *
873 * They will be detected by lockdep when they occur, but the first might not
874 * occur very often. It depends on what work items are on the workqueue and
875 * what locks they need, which you have no control over.
876 *
877 * In most situations flushing the entire workqueue is overkill; you merely
878 * need to know that a particular work item isn't queued and isn't running.
879 * In such cases you should use cancel_delayed_work_sync() or
880 * cancel_work_sync() instead.
881 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882void flush_scheduled_work(void)
883{
884 flush_workqueue(keventd_wq);
885}
Dave Jonesae90dd52006-06-30 01:40:45 -0400886EXPORT_SYMBOL(flush_scheduled_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887
888/**
James Bottomley1fa44ec2006-02-23 12:43:43 -0600889 * execute_in_process_context - reliably execute the routine with user context
890 * @fn: the function to execute
James Bottomley1fa44ec2006-02-23 12:43:43 -0600891 * @ew: guaranteed storage for the execute work structure (must
892 * be available when the work executes)
893 *
894 * Executes the function immediately if process context is available,
895 * otherwise schedules the function for delayed execution.
896 *
897 * Returns: 0 - function was executed
898 * 1 - function was scheduled for execution
899 */
David Howells65f27f32006-11-22 14:55:48 +0000900int execute_in_process_context(work_func_t fn, struct execute_work *ew)
James Bottomley1fa44ec2006-02-23 12:43:43 -0600901{
902 if (!in_interrupt()) {
David Howells65f27f32006-11-22 14:55:48 +0000903 fn(&ew->work);
James Bottomley1fa44ec2006-02-23 12:43:43 -0600904 return 0;
905 }
906
David Howells65f27f32006-11-22 14:55:48 +0000907 INIT_WORK(&ew->work, fn);
James Bottomley1fa44ec2006-02-23 12:43:43 -0600908 schedule_work(&ew->work);
909
910 return 1;
911}
912EXPORT_SYMBOL_GPL(execute_in_process_context);
913
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914int keventd_up(void)
915{
916 return keventd_wq != NULL;
917}
918
919int current_is_keventd(void)
920{
921 struct cpu_workqueue_struct *cwq;
Hugh Dickinsd2437692007-08-27 16:06:19 +0100922 int cpu = raw_smp_processor_id(); /* preempt-safe: keventd is per-cpu */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923 int ret = 0;
924
925 BUG_ON(!keventd_wq);
926
Christoph Lameter89ada672005-10-30 15:01:59 -0800927 cwq = per_cpu_ptr(keventd_wq->cpu_wq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 if (current == cwq->thread)
929 ret = 1;
930
931 return ret;
932
933}
934
Oleg Nesterov3af244332007-05-09 02:34:09 -0700935static struct cpu_workqueue_struct *
936init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937{
Christoph Lameter89ada672005-10-30 15:01:59 -0800938 struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939
Oleg Nesterov3af244332007-05-09 02:34:09 -0700940 cwq->wq = wq;
941 spin_lock_init(&cwq->lock);
942 INIT_LIST_HEAD(&cwq->worklist);
943 init_waitqueue_head(&cwq->more_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944
Oleg Nesterov3af244332007-05-09 02:34:09 -0700945 return cwq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946}
947
Oleg Nesterov3af244332007-05-09 02:34:09 -0700948static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949{
Heiko Carstens0d557dc2008-10-13 23:50:09 +0200950 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
Oleg Nesterov3af244332007-05-09 02:34:09 -0700951 struct workqueue_struct *wq = cwq->wq;
David Howells6cc88bc2008-11-14 10:39:21 +1100952 const char *fmt = is_wq_single_threaded(wq) ? "%s" : "%s/%d";
Oleg Nesterov3af244332007-05-09 02:34:09 -0700953 struct task_struct *p;
954
955 p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);
956 /*
957 * Nobody can add the work_struct to this cwq,
958 * if (caller is __create_workqueue)
959 * nobody should see this wq
960 * else // caller is CPU_UP_PREPARE
961 * cpu is not on cpu_online_map
962 * so we can abort safely.
963 */
964 if (IS_ERR(p))
965 return PTR_ERR(p);
Heiko Carstens0d557dc2008-10-13 23:50:09 +0200966 if (cwq->wq->rt)
967 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700968 cwq->thread = p;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700969
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +0100970 trace_workqueue_creation(cwq->thread, cpu);
971
Oleg Nesterov3af244332007-05-09 02:34:09 -0700972 return 0;
973}
974
Oleg Nesterov06ba38a2007-05-09 02:34:15 -0700975static void start_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
976{
977 struct task_struct *p = cwq->thread;
978
979 if (p != NULL) {
980 if (cpu >= 0)
981 kthread_bind(p, cpu);
982 wake_up_process(p);
983 }
984}
985
Johannes Berg4e6045f2007-10-18 23:39:55 -0700986struct workqueue_struct *__create_workqueue_key(const char *name,
987 int singlethread,
988 int freezeable,
Heiko Carstens0d557dc2008-10-13 23:50:09 +0200989 int rt,
Johannes Bergeb13ba82008-01-16 09:51:58 +0100990 struct lock_class_key *key,
991 const char *lock_name)
Oleg Nesterov3af244332007-05-09 02:34:09 -0700992{
993 struct workqueue_struct *wq;
994 struct cpu_workqueue_struct *cwq;
995 int err = 0, cpu;
996
997 wq = kzalloc(sizeof(*wq), GFP_KERNEL);
998 if (!wq)
999 return NULL;
1000
1001 wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct);
1002 if (!wq->cpu_wq) {
1003 kfree(wq);
1004 return NULL;
1005 }
1006
1007 wq->name = name;
Johannes Bergeb13ba82008-01-16 09:51:58 +01001008 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
Oleg Nesterovcce1a162007-05-09 02:34:13 -07001009 wq->singlethread = singlethread;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001010 wq->freezeable = freezeable;
Heiko Carstens0d557dc2008-10-13 23:50:09 +02001011 wq->rt = rt;
Oleg Nesterovcce1a162007-05-09 02:34:13 -07001012 INIT_LIST_HEAD(&wq->list);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001013
1014 if (singlethread) {
Oleg Nesterov3af244332007-05-09 02:34:09 -07001015 cwq = init_cpu_workqueue(wq, singlethread_cpu);
1016 err = create_workqueue_thread(cwq, singlethread_cpu);
Oleg Nesterov06ba38a2007-05-09 02:34:15 -07001017 start_workqueue_thread(cwq, -1);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001018 } else {
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001019 cpu_maps_update_begin();
Oleg Nesterov6af8bf32008-07-29 22:33:49 -07001020 /*
1021 * We must place this wq on list even if the code below fails.
1022 * cpu_down(cpu) can remove cpu from cpu_populated_map before
1023 * destroy_workqueue() takes the lock, in that case we leak
1024 * cwq[cpu]->thread.
1025 */
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001026 spin_lock(&workqueue_lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001027 list_add(&wq->list, &workqueues);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001028 spin_unlock(&workqueue_lock);
Oleg Nesterov6af8bf32008-07-29 22:33:49 -07001029 /*
1030 * We must initialize cwqs for each possible cpu even if we
1031 * are going to call destroy_workqueue() finally. Otherwise
1032 * cpu_up() can hit the uninitialized cwq once we drop the
1033 * lock.
1034 */
Oleg Nesterov3af244332007-05-09 02:34:09 -07001035 for_each_possible_cpu(cpu) {
1036 cwq = init_cpu_workqueue(wq, cpu);
1037 if (err || !cpu_online(cpu))
1038 continue;
1039 err = create_workqueue_thread(cwq, cpu);
Oleg Nesterov06ba38a2007-05-09 02:34:15 -07001040 start_workqueue_thread(cwq, cpu);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001041 }
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001042 cpu_maps_update_done();
Oleg Nesterov3af244332007-05-09 02:34:09 -07001043 }
1044
1045 if (err) {
1046 destroy_workqueue(wq);
1047 wq = NULL;
1048 }
1049 return wq;
1050}
Johannes Berg4e6045f2007-10-18 23:39:55 -07001051EXPORT_SYMBOL_GPL(__create_workqueue_key);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001052
Oleg Nesterov1e35eaa22008-04-29 01:00:28 -07001053static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
Oleg Nesterov3af244332007-05-09 02:34:09 -07001054{
Oleg Nesterov14441962007-05-23 13:57:57 -07001055 /*
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001056 * Our caller is either destroy_workqueue() or CPU_POST_DEAD,
1057 * cpu_add_remove_lock protects cwq->thread.
Oleg Nesterov14441962007-05-23 13:57:57 -07001058 */
1059 if (cwq->thread == NULL)
1060 return;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001061
Ingo Molnar3295f0e2008-08-11 10:30:30 +02001062 lock_map_acquire(&cwq->wq->lockdep_map);
1063 lock_map_release(&cwq->wq->lockdep_map);
Johannes Berg4e6045f2007-10-18 23:39:55 -07001064
Oleg Nesterov13c22162007-07-17 04:03:55 -07001065 flush_cpu_workqueue(cwq);
Oleg Nesterov14441962007-05-23 13:57:57 -07001066 /*
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001067 * If the caller is CPU_POST_DEAD and cwq->worklist was not empty,
Oleg Nesterov13c22162007-07-17 04:03:55 -07001068 * a concurrent flush_workqueue() can insert a barrier after us.
1069 * However, in that case run_workqueue() won't return and check
1070 * kthread_should_stop() until it flushes all work_struct's.
Oleg Nesterov14441962007-05-23 13:57:57 -07001071 * When ->worklist becomes empty it is safe to exit because no
1072 * more work_structs can be queued on this cwq: flush_workqueue
1073 * checks list_empty(), and a "normal" queue_work() can't use
1074 * a dead CPU.
1075 */
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +01001076 trace_workqueue_destruction(cwq->thread);
Oleg Nesterov14441962007-05-23 13:57:57 -07001077 kthread_stop(cwq->thread);
1078 cwq->thread = NULL;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001079}
1080
1081/**
1082 * destroy_workqueue - safely terminate a workqueue
1083 * @wq: target workqueue
1084 *
1085 * Safely destroy a workqueue. All work currently pending will be done first.
1086 */
1087void destroy_workqueue(struct workqueue_struct *wq)
1088{
Rusty Russelle7577c52009-01-01 10:12:25 +10301089 const struct cpumask *cpu_map = wq_cpu_map(wq);
Oleg Nesterovb1f4ec172007-05-09 02:34:12 -07001090 int cpu;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001091
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001092 cpu_maps_update_begin();
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001093 spin_lock(&workqueue_lock);
Oleg Nesterovb1f4ec172007-05-09 02:34:12 -07001094 list_del(&wq->list);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001095 spin_unlock(&workqueue_lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001096
Rusty Russellaa85ea52009-03-30 22:05:15 -06001097 for_each_cpu(cpu, cpu_map)
Oleg Nesterov1e35eaa22008-04-29 01:00:28 -07001098 cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu));
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001099 cpu_maps_update_done();
Oleg Nesterov3af244332007-05-09 02:34:09 -07001100
1101 free_percpu(wq->cpu_wq);
1102 kfree(wq);
1103}
1104EXPORT_SYMBOL_GPL(destroy_workqueue);
1105
1106static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
1107 unsigned long action,
1108 void *hcpu)
1109{
1110 unsigned int cpu = (unsigned long)hcpu;
1111 struct cpu_workqueue_struct *cwq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112 struct workqueue_struct *wq;
Akinobu Mita80b51842010-05-26 14:43:32 -07001113 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001115 action &= ~CPU_TASKS_FROZEN;
1116
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117 switch (action) {
1118 case CPU_UP_PREPARE:
Rusty Russelle7577c52009-01-01 10:12:25 +10301119 cpumask_set_cpu(cpu, cpu_populated_map);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001120 }
Oleg Nesterov84485022008-07-25 01:47:54 -07001121undo:
Oleg Nesterov3af244332007-05-09 02:34:09 -07001122 list_for_each_entry(wq, &workqueues, list) {
1123 cwq = per_cpu_ptr(wq->cpu_wq, cpu);
Christoph Lameter89ada672005-10-30 15:01:59 -08001124
Oleg Nesterov3af244332007-05-09 02:34:09 -07001125 switch (action) {
1126 case CPU_UP_PREPARE:
Akinobu Mita80b51842010-05-26 14:43:32 -07001127 err = create_workqueue_thread(cwq, cpu);
1128 if (!err)
Oleg Nesterov3af244332007-05-09 02:34:09 -07001129 break;
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001130 printk(KERN_ERR "workqueue [%s] for %i failed\n",
1131 wq->name, cpu);
Oleg Nesterov84485022008-07-25 01:47:54 -07001132 action = CPU_UP_CANCELED;
Akinobu Mita80b51842010-05-26 14:43:32 -07001133 err = -ENOMEM;
Oleg Nesterov84485022008-07-25 01:47:54 -07001134 goto undo;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001135
1136 case CPU_ONLINE:
Oleg Nesterov06ba38a2007-05-09 02:34:15 -07001137 start_workqueue_thread(cwq, cpu);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001138 break;
1139
1140 case CPU_UP_CANCELED:
Oleg Nesterov06ba38a2007-05-09 02:34:15 -07001141 start_workqueue_thread(cwq, -1);
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001142 case CPU_POST_DEAD:
Oleg Nesterov1e35eaa22008-04-29 01:00:28 -07001143 cleanup_workqueue_thread(cwq);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001144 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146 }
1147
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -07001148 switch (action) {
1149 case CPU_UP_CANCELED:
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001150 case CPU_POST_DEAD:
Rusty Russelle7577c52009-01-01 10:12:25 +10301151 cpumask_clear_cpu(cpu, cpu_populated_map);
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -07001152 }
1153
Akinobu Mita80b51842010-05-26 14:43:32 -07001154 return notifier_from_errno(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156
Rusty Russell2d3854a2008-11-05 13:39:10 +11001157#ifdef CONFIG_SMP
Rusty Russell8ccad402009-01-16 15:31:15 -08001158
Rusty Russell2d3854a2008-11-05 13:39:10 +11001159struct work_for_cpu {
Andrew Morton6b44003e2009-04-09 09:50:37 -06001160 struct completion completion;
Rusty Russell2d3854a2008-11-05 13:39:10 +11001161 long (*fn)(void *);
1162 void *arg;
1163 long ret;
1164};
1165
Andrew Morton6b44003e2009-04-09 09:50:37 -06001166static int do_work_for_cpu(void *_wfc)
Rusty Russell2d3854a2008-11-05 13:39:10 +11001167{
Andrew Morton6b44003e2009-04-09 09:50:37 -06001168 struct work_for_cpu *wfc = _wfc;
Rusty Russell2d3854a2008-11-05 13:39:10 +11001169 wfc->ret = wfc->fn(wfc->arg);
Andrew Morton6b44003e2009-04-09 09:50:37 -06001170 complete(&wfc->completion);
1171 return 0;
Rusty Russell2d3854a2008-11-05 13:39:10 +11001172}
1173
1174/**
1175 * work_on_cpu - run a function in user context on a particular cpu
1176 * @cpu: the cpu to run on
1177 * @fn: the function to run
1178 * @arg: the function arg
1179 *
Rusty Russell31ad9082009-01-16 15:31:15 -08001180 * This will return the value @fn returns.
1181 * It is up to the caller to ensure that the cpu doesn't go offline.
Andrew Morton6b44003e2009-04-09 09:50:37 -06001182 * The caller must not hold any locks which would prevent @fn from completing.
Rusty Russell2d3854a2008-11-05 13:39:10 +11001183 */
1184long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg)
1185{
Andrew Morton6b44003e2009-04-09 09:50:37 -06001186 struct task_struct *sub_thread;
1187 struct work_for_cpu wfc = {
1188 .completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion),
1189 .fn = fn,
1190 .arg = arg,
1191 };
Rusty Russell2d3854a2008-11-05 13:39:10 +11001192
Andrew Morton6b44003e2009-04-09 09:50:37 -06001193 sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu");
1194 if (IS_ERR(sub_thread))
1195 return PTR_ERR(sub_thread);
1196 kthread_bind(sub_thread, cpu);
1197 wake_up_process(sub_thread);
1198 wait_for_completion(&wfc.completion);
Rusty Russell2d3854a2008-11-05 13:39:10 +11001199 return wfc.ret;
1200}
1201EXPORT_SYMBOL_GPL(work_on_cpu);
1202#endif /* CONFIG_SMP */
1203
Oleg Nesterovc12920d2007-05-09 02:34:14 -07001204void __init init_workqueues(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205{
Rusty Russelle7577c52009-01-01 10:12:25 +10301206 alloc_cpumask_var(&cpu_populated_map, GFP_KERNEL);
1207
1208 cpumask_copy(cpu_populated_map, cpu_online_mask);
1209 singlethread_cpu = cpumask_first(cpu_possible_mask);
1210 cpu_singlethread_map = cpumask_of(singlethread_cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 hotcpu_notifier(workqueue_cpu_callback, 0);
1212 keventd_wq = create_workqueue("events");
1213 BUG_ON(!keventd_wq);
1214}