blob: 6e32635e5e570c7632aaebeb724c1d34e10172a1 [file] [log] [blame]
Steven Rostedt352ad252008-05-12 21:20:42 +02001/*
2 * trace task wakeup timings
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Based on code from the latency_tracer, that is:
8 *
9 * Copyright (C) 2004-2006 Ingo Molnar
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +010010 * Copyright (C) 2004 Nadia Yvette Chambers
Steven Rostedt352ad252008-05-12 21:20:42 +020011 */
12#include <linux/module.h>
13#include <linux/fs.h>
14#include <linux/debugfs.h>
15#include <linux/kallsyms.h>
16#include <linux/uaccess.h>
17#include <linux/ftrace.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060018#include <linux/sched/rt.h>
Dario Faggioli2d3d8912013-11-07 14:43:44 +010019#include <linux/sched/deadline.h>
Steven Rostedtad8d75f2009-04-14 19:39:12 -040020#include <trace/events/sched.h>
Steven Rostedt352ad252008-05-12 21:20:42 +020021#include "trace.h"
22
23static struct trace_array *wakeup_trace;
24static int __read_mostly tracer_enabled;
25
26static struct task_struct *wakeup_task;
27static int wakeup_cpu;
Steven Rostedt478142c32009-09-09 10:36:01 -040028static int wakeup_current_cpu;
Steven Rostedt352ad252008-05-12 21:20:42 +020029static unsigned wakeup_prio = -1;
Steven Rostedt32443512009-01-21 16:24:46 -050030static int wakeup_rt;
Dario Faggioliaf6ace72013-11-07 14:43:42 +010031static int wakeup_dl;
32static int tracing_dl = 0;
Steven Rostedt352ad252008-05-12 21:20:42 +020033
Thomas Gleixner445c8952009-12-02 19:49:50 +010034static arch_spinlock_t wakeup_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +010035 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt352ad252008-05-12 21:20:42 +020036
Jiri Olsa7495a5be2010-09-23 14:00:53 +020037static void wakeup_reset(struct trace_array *tr);
Ingo Molnare309b412008-05-12 21:20:51 +020038static void __wakeup_reset(struct trace_array *tr);
Jiri Olsa7495a5be2010-09-23 14:00:53 +020039static int wakeup_graph_entry(struct ftrace_graph_ent *trace);
40static void wakeup_graph_return(struct ftrace_graph_ret *trace);
Steven Rostedt352ad252008-05-12 21:20:42 +020041
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -040042static int save_flags;
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -040043static bool function_enabled;
Steven Rostedte9d25fe2009-03-04 22:15:30 -050044
Jiri Olsa7495a5be2010-09-23 14:00:53 +020045#define TRACE_DISPLAY_GRAPH 1
46
47static struct tracer_opt trace_opts[] = {
48#ifdef CONFIG_FUNCTION_GRAPH_TRACER
49 /* display latency trace as call graph */
50 { TRACER_OPT(display-graph, TRACE_DISPLAY_GRAPH) },
51#endif
52 { } /* Empty entry */
53};
54
55static struct tracer_flags tracer_flags = {
56 .val = 0,
57 .opts = trace_opts,
58};
59
60#define is_graph() (tracer_flags.val & TRACE_DISPLAY_GRAPH)
61
Steven Rostedt606576c2008-10-06 19:06:12 -040062#ifdef CONFIG_FUNCTION_TRACER
Steven Rostedt542181d2010-10-05 16:38:49 -040063
64/*
65 * Prologue for the wakeup function tracers.
66 *
67 * Returns 1 if it is OK to continue, and preemption
68 * is disabled and data->disabled is incremented.
69 * 0 if the trace is to be ignored, and preemption
70 * is not disabled and data->disabled is
71 * kept the same.
72 *
73 * Note, this function is also used outside this ifdef but
74 * inside the #ifdef of the function graph tracer below.
75 * This is OK, since the function graph tracer is
76 * dependent on the function tracer.
77 */
78static int
79func_prolog_preempt_disable(struct trace_array *tr,
80 struct trace_array_cpu **data,
81 int *pc)
82{
83 long disabled;
84 int cpu;
85
86 if (likely(!wakeup_task))
87 return 0;
88
89 *pc = preempt_count();
90 preempt_disable_notrace();
91
92 cpu = raw_smp_processor_id();
93 if (cpu != wakeup_current_cpu)
94 goto out_enable;
95
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -050096 *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
Steven Rostedt542181d2010-10-05 16:38:49 -040097 disabled = atomic_inc_return(&(*data)->disabled);
98 if (unlikely(disabled != 1))
99 goto out;
100
101 return 1;
102
103out:
104 atomic_dec(&(*data)->disabled);
105
106out_enable:
107 preempt_enable_notrace();
108 return 0;
109}
110
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400111/*
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200112 * wakeup uses its own tracer function to keep the overhead down:
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400113 */
114static void
Steven Rostedta1e2e312011-08-09 12:50:46 -0400115wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
116 struct ftrace_ops *op, struct pt_regs *pt_regs)
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400117{
118 struct trace_array *tr = wakeup_trace;
119 struct trace_array_cpu *data;
120 unsigned long flags;
Steven Rostedt38697052008-10-01 13:14:09 -0400121 int pc;
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400122
Steven Rostedt542181d2010-10-05 16:38:49 -0400123 if (!func_prolog_preempt_disable(tr, &data, &pc))
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400124 return;
125
Steven Rostedte59494f2008-07-16 00:13:45 -0400126 local_irq_save(flags);
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -0500127 trace_function(tr, ip, parent_ip, flags, pc);
Steven Rostedte59494f2008-07-16 00:13:45 -0400128 local_irq_restore(flags);
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400129
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400130 atomic_dec(&data->disabled);
Steven Rostedt5168ae52010-06-03 09:36:50 -0400131 preempt_enable_notrace();
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400132}
133
134static struct ftrace_ops trace_ops __read_mostly =
135{
136 .func = wakeup_tracer_call,
Steven Rostedt47409742012-07-20 11:04:44 -0400137 .flags = FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_RECURSION_SAFE,
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400138};
Steven Rostedt7e407982010-10-19 10:56:19 -0400139#endif /* CONFIG_FUNCTION_TRACER */
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200140
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400141static int register_wakeup_function(int graph, int set)
142{
143 int ret;
144
145 /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
146 if (function_enabled || (!set && !(trace_flags & TRACE_ITER_FUNCTION)))
147 return 0;
148
149 if (graph)
150 ret = register_ftrace_graph(&wakeup_graph_return,
151 &wakeup_graph_entry);
152 else
153 ret = register_ftrace_function(&trace_ops);
154
155 if (!ret)
156 function_enabled = true;
157
158 return ret;
159}
160
161static void unregister_wakeup_function(int graph)
162{
163 if (!function_enabled)
164 return;
165
166 if (graph)
167 unregister_ftrace_graph();
168 else
169 unregister_ftrace_function(&trace_ops);
170
171 function_enabled = false;
172}
173
174static void wakeup_function_set(int set)
175{
176 if (set)
177 register_wakeup_function(is_graph(), 1);
178 else
179 unregister_wakeup_function(is_graph());
180}
181
182static int wakeup_flag_changed(struct tracer *tracer, u32 mask, int set)
183{
184 if (mask & TRACE_ITER_FUNCTION)
185 wakeup_function_set(set);
186
187 return trace_keep_overwrite(tracer, mask, set);
188}
189
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200190static int start_func_tracer(int graph)
191{
192 int ret;
193
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400194 ret = register_wakeup_function(graph, 0);
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200195
196 if (!ret && tracing_is_enabled())
197 tracer_enabled = 1;
198 else
199 tracer_enabled = 0;
200
201 return ret;
202}
203
204static void stop_func_tracer(int graph)
205{
206 tracer_enabled = 0;
207
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400208 unregister_wakeup_function(graph);
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200209}
210
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200211#ifdef CONFIG_FUNCTION_GRAPH_TRACER
212static int wakeup_set_flag(u32 old_flags, u32 bit, int set)
213{
214
215 if (!(bit & TRACE_DISPLAY_GRAPH))
216 return -EINVAL;
217
218 if (!(is_graph() ^ set))
219 return 0;
220
221 stop_func_tracer(!set);
222
223 wakeup_reset(wakeup_trace);
224 tracing_max_latency = 0;
225
226 return start_func_tracer(set);
227}
228
229static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
230{
231 struct trace_array *tr = wakeup_trace;
232 struct trace_array_cpu *data;
233 unsigned long flags;
Steven Rostedt542181d2010-10-05 16:38:49 -0400234 int pc, ret = 0;
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200235
Steven Rostedt542181d2010-10-05 16:38:49 -0400236 if (!func_prolog_preempt_disable(tr, &data, &pc))
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200237 return 0;
238
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200239 local_save_flags(flags);
240 ret = __trace_graph_entry(tr, trace, flags, pc);
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200241 atomic_dec(&data->disabled);
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200242 preempt_enable_notrace();
Steven Rostedt542181d2010-10-05 16:38:49 -0400243
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200244 return ret;
245}
246
247static void wakeup_graph_return(struct ftrace_graph_ret *trace)
248{
249 struct trace_array *tr = wakeup_trace;
250 struct trace_array_cpu *data;
251 unsigned long flags;
Steven Rostedt542181d2010-10-05 16:38:49 -0400252 int pc;
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200253
Steven Rostedt542181d2010-10-05 16:38:49 -0400254 if (!func_prolog_preempt_disable(tr, &data, &pc))
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200255 return;
256
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200257 local_save_flags(flags);
258 __trace_graph_return(tr, trace, flags, pc);
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200259 atomic_dec(&data->disabled);
260
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200261 preempt_enable_notrace();
262 return;
263}
264
265static void wakeup_trace_open(struct trace_iterator *iter)
266{
267 if (is_graph())
268 graph_trace_open(iter);
269}
270
271static void wakeup_trace_close(struct trace_iterator *iter)
272{
273 if (iter->private)
274 graph_trace_close(iter);
275}
276
Jiri Olsa321e68b02011-06-03 16:58:47 +0200277#define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \
278 TRACE_GRAPH_PRINT_ABS_TIME | \
279 TRACE_GRAPH_PRINT_DURATION)
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200280
281static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
282{
283 /*
284 * In graph mode call the graph tracer output function,
285 * otherwise go with the TRACE_FN event handler
286 */
287 if (is_graph())
288 return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
289
290 return TRACE_TYPE_UNHANDLED;
291}
292
293static void wakeup_print_header(struct seq_file *s)
294{
295 if (is_graph())
296 print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
297 else
298 trace_default_header(s);
299}
300
301static void
302__trace_function(struct trace_array *tr,
303 unsigned long ip, unsigned long parent_ip,
304 unsigned long flags, int pc)
305{
306 if (is_graph())
307 trace_graph_function(tr, ip, parent_ip, flags, pc);
308 else
309 trace_function(tr, ip, parent_ip, flags, pc);
310}
311#else
312#define __trace_function trace_function
313
314static int wakeup_set_flag(u32 old_flags, u32 bit, int set)
315{
316 return -EINVAL;
317}
318
319static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
320{
321 return -1;
322}
323
324static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
325{
326 return TRACE_TYPE_UNHANDLED;
327}
328
329static void wakeup_graph_return(struct ftrace_graph_ret *trace) { }
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200330static void wakeup_trace_open(struct trace_iterator *iter) { }
331static void wakeup_trace_close(struct trace_iterator *iter) { }
Jiri Olsa7e9a49e2011-11-07 16:08:49 +0100332
333#ifdef CONFIG_FUNCTION_TRACER
334static void wakeup_print_header(struct seq_file *s)
335{
336 trace_default_header(s);
337}
338#else
339static void wakeup_print_header(struct seq_file *s)
340{
341 trace_latency_header(s);
342}
343#endif /* CONFIG_FUNCTION_TRACER */
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200344#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
345
Steven Rostedt352ad252008-05-12 21:20:42 +0200346/*
347 * Should this new latency be reported/recorded?
348 */
Ingo Molnare309b412008-05-12 21:20:51 +0200349static int report_latency(cycle_t delta)
Steven Rostedt352ad252008-05-12 21:20:42 +0200350{
351 if (tracing_thresh) {
352 if (delta < tracing_thresh)
353 return 0;
354 } else {
355 if (delta <= tracing_max_latency)
356 return 0;
357 }
358 return 1;
359}
360
Steven Rostedt38516ab2010-04-20 17:04:50 -0400361static void
362probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu)
Steven Rostedt478142c32009-09-09 10:36:01 -0400363{
364 if (task != wakeup_task)
365 return;
366
367 wakeup_current_cpu = cpu;
368}
369
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200370static void notrace
Steven Rostedt38516ab2010-04-20 17:04:50 -0400371probe_wakeup_sched_switch(void *ignore,
372 struct task_struct *prev, struct task_struct *next)
Steven Rostedt352ad252008-05-12 21:20:42 +0200373{
Steven Rostedt352ad252008-05-12 21:20:42 +0200374 struct trace_array_cpu *data;
375 cycle_t T0, T1, delta;
376 unsigned long flags;
377 long disabled;
378 int cpu;
Steven Rostedt38697052008-10-01 13:14:09 -0400379 int pc;
Steven Rostedt352ad252008-05-12 21:20:42 +0200380
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400381 tracing_record_cmdline(prev);
382
Steven Rostedt352ad252008-05-12 21:20:42 +0200383 if (unlikely(!tracer_enabled))
384 return;
385
386 /*
387 * When we start a new trace, we set wakeup_task to NULL
388 * and then set tracer_enabled = 1. We want to make sure
389 * that another CPU does not see the tracer_enabled = 1
390 * and the wakeup_task with an older task, that might
391 * actually be the same as next.
392 */
393 smp_rmb();
394
395 if (next != wakeup_task)
396 return;
397
Steven Rostedt38697052008-10-01 13:14:09 -0400398 pc = preempt_count();
399
Steven Rostedt352ad252008-05-12 21:20:42 +0200400 /* disable local data, not wakeup_cpu data */
401 cpu = raw_smp_processor_id();
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500402 disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
Steven Rostedt352ad252008-05-12 21:20:42 +0200403 if (likely(disabled != 1))
404 goto out;
405
Steven Rostedte59494f2008-07-16 00:13:45 -0400406 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100407 arch_spin_lock(&wakeup_lock);
Steven Rostedt352ad252008-05-12 21:20:42 +0200408
409 /* We could race with grabbing wakeup_lock */
410 if (unlikely(!tracer_enabled || next != wakeup_task))
411 goto out_unlock;
412
Steven Rostedt9be24412009-03-26 10:25:24 -0400413 /* The task we are waiting for is waking up */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500414 data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu);
Steven Rostedt9be24412009-03-26 10:25:24 -0400415
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200416 __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc);
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -0500417 tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
Steven Rostedt352ad252008-05-12 21:20:42 +0200418
Steven Rostedt352ad252008-05-12 21:20:42 +0200419 T0 = data->preempt_timestamp;
Ingo Molnar750ed1a2008-05-12 21:20:46 +0200420 T1 = ftrace_now(cpu);
Steven Rostedt352ad252008-05-12 21:20:42 +0200421 delta = T1-T0;
422
423 if (!report_latency(delta))
424 goto out_unlock;
425
Carsten Emdeb5130b12009-09-13 01:43:07 +0200426 if (likely(!is_tracing_stopped())) {
427 tracing_max_latency = delta;
428 update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu);
429 }
Steven Rostedt352ad252008-05-12 21:20:42 +0200430
Steven Rostedt352ad252008-05-12 21:20:42 +0200431out_unlock:
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400432 __wakeup_reset(wakeup_trace);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100433 arch_spin_unlock(&wakeup_lock);
Steven Rostedte59494f2008-07-16 00:13:45 -0400434 local_irq_restore(flags);
Steven Rostedt352ad252008-05-12 21:20:42 +0200435out:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500436 atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200437}
438
Ingo Molnare309b412008-05-12 21:20:51 +0200439static void __wakeup_reset(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200440{
Steven Rostedt352ad252008-05-12 21:20:42 +0200441 wakeup_cpu = -1;
442 wakeup_prio = -1;
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100443 tracing_dl = 0;
Steven Rostedt352ad252008-05-12 21:20:42 +0200444
445 if (wakeup_task)
446 put_task_struct(wakeup_task);
447
448 wakeup_task = NULL;
449}
450
Ingo Molnare309b412008-05-12 21:20:51 +0200451static void wakeup_reset(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200452{
453 unsigned long flags;
454
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500455 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -0400456
Steven Rostedte59494f2008-07-16 00:13:45 -0400457 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100458 arch_spin_lock(&wakeup_lock);
Steven Rostedt352ad252008-05-12 21:20:42 +0200459 __wakeup_reset(tr);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100460 arch_spin_unlock(&wakeup_lock);
Steven Rostedte59494f2008-07-16 00:13:45 -0400461 local_irq_restore(flags);
Steven Rostedt352ad252008-05-12 21:20:42 +0200462}
463
Ingo Molnare309b412008-05-12 21:20:51 +0200464static void
Steven Rostedt38516ab2010-04-20 17:04:50 -0400465probe_wakeup(void *ignore, struct task_struct *p, int success)
Steven Rostedt352ad252008-05-12 21:20:42 +0200466{
Steven Rostedtf8ec1062009-01-21 17:17:04 -0500467 struct trace_array_cpu *data;
Steven Rostedt352ad252008-05-12 21:20:42 +0200468 int cpu = smp_processor_id();
469 unsigned long flags;
470 long disabled;
Steven Rostedt38697052008-10-01 13:14:09 -0400471 int pc;
Steven Rostedt352ad252008-05-12 21:20:42 +0200472
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400473 if (likely(!tracer_enabled))
Steven Rostedt352ad252008-05-12 21:20:42 +0200474 return;
475
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400476 tracing_record_cmdline(p);
477 tracing_record_cmdline(current);
478
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100479 /*
480 * Semantic is like this:
481 * - wakeup tracer handles all tasks in the system, independently
482 * from their scheduling class;
483 * - wakeup_rt tracer handles tasks belonging to sched_dl and
484 * sched_rt class;
485 * - wakeup_dl handles tasks belonging to sched_dl class only.
486 */
487 if (tracing_dl || (wakeup_dl && !dl_task(p)) ||
488 (wakeup_rt && !dl_task(p) && !rt_task(p)) ||
489 (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio)))
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400490 return;
491
Steven Rostedt38697052008-10-01 13:14:09 -0400492 pc = preempt_count();
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500493 disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
Steven Rostedt352ad252008-05-12 21:20:42 +0200494 if (unlikely(disabled != 1))
495 goto out;
496
497 /* interrupts should be off from try_to_wake_up */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100498 arch_spin_lock(&wakeup_lock);
Steven Rostedt352ad252008-05-12 21:20:42 +0200499
500 /* check for races. */
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100501 if (!tracer_enabled || tracing_dl ||
502 (!dl_task(p) && p->prio >= wakeup_prio))
Steven Rostedt352ad252008-05-12 21:20:42 +0200503 goto out_locked;
504
505 /* reset the trace */
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400506 __wakeup_reset(wakeup_trace);
Steven Rostedt352ad252008-05-12 21:20:42 +0200507
508 wakeup_cpu = task_cpu(p);
Steven Rostedt478142c32009-09-09 10:36:01 -0400509 wakeup_current_cpu = wakeup_cpu;
Steven Rostedt352ad252008-05-12 21:20:42 +0200510 wakeup_prio = p->prio;
511
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100512 /*
513 * Once you start tracing a -deadline task, don't bother tracing
514 * another task until the first one wakes up.
515 */
516 if (dl_task(p))
517 tracing_dl = 1;
518 else
519 tracing_dl = 0;
520
Steven Rostedt352ad252008-05-12 21:20:42 +0200521 wakeup_task = p;
522 get_task_struct(wakeup_task);
523
524 local_save_flags(flags);
525
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500526 data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu);
Steven Rostedtf8ec1062009-01-21 17:17:04 -0500527 data->preempt_timestamp = ftrace_now(cpu);
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -0500528 tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc);
Steven Rostedt301fd742009-04-03 11:12:23 -0400529
530 /*
531 * We must be careful in using CALLER_ADDR2. But since wake_up
532 * is not called by an assembly function (where as schedule is)
533 * it should be safe to use it here.
534 */
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200535 __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
Steven Rostedt352ad252008-05-12 21:20:42 +0200536
537out_locked:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100538 arch_spin_unlock(&wakeup_lock);
Steven Rostedt352ad252008-05-12 21:20:42 +0200539out:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500540 atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
Steven Rostedt352ad252008-05-12 21:20:42 +0200541}
542
Ingo Molnare309b412008-05-12 21:20:51 +0200543static void start_wakeup_tracer(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200544{
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200545 int ret;
546
Steven Rostedt38516ab2010-04-20 17:04:50 -0400547 ret = register_trace_sched_wakeup(probe_wakeup, NULL);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200548 if (ret) {
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400549 pr_info("wakeup trace: Couldn't activate tracepoint"
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200550 " probe to kernel_sched_wakeup\n");
551 return;
552 }
553
Steven Rostedt38516ab2010-04-20 17:04:50 -0400554 ret = register_trace_sched_wakeup_new(probe_wakeup, NULL);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200555 if (ret) {
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400556 pr_info("wakeup trace: Couldn't activate tracepoint"
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200557 " probe to kernel_sched_wakeup_new\n");
558 goto fail_deprobe;
559 }
560
Steven Rostedt38516ab2010-04-20 17:04:50 -0400561 ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200562 if (ret) {
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400563 pr_info("sched trace: Couldn't activate tracepoint"
Wenji Huang73d8b8b2009-02-17 01:10:02 -0500564 " probe to kernel_sched_switch\n");
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200565 goto fail_deprobe_wake_new;
566 }
567
Steven Rostedt38516ab2010-04-20 17:04:50 -0400568 ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
Steven Rostedt478142c32009-09-09 10:36:01 -0400569 if (ret) {
570 pr_info("wakeup trace: Couldn't activate tracepoint"
571 " probe to kernel_sched_migrate_task\n");
572 return;
573 }
574
Steven Rostedt352ad252008-05-12 21:20:42 +0200575 wakeup_reset(tr);
576
577 /*
578 * Don't let the tracer_enabled = 1 show up before
579 * the wakeup_task is reset. This may be overkill since
580 * wakeup_reset does a spin_unlock after setting the
581 * wakeup_task to NULL, but I want to be safe.
582 * This is a slow path anyway.
583 */
584 smp_wmb();
585
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200586 if (start_func_tracer(is_graph()))
587 printk(KERN_ERR "failed to start wakeup tracer\n");
Steven Rostedtad591242008-07-10 20:58:13 -0400588
Steven Rostedt352ad252008-05-12 21:20:42 +0200589 return;
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200590fail_deprobe_wake_new:
Steven Rostedt38516ab2010-04-20 17:04:50 -0400591 unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200592fail_deprobe:
Steven Rostedt38516ab2010-04-20 17:04:50 -0400593 unregister_trace_sched_wakeup(probe_wakeup, NULL);
Steven Rostedt352ad252008-05-12 21:20:42 +0200594}
595
Ingo Molnare309b412008-05-12 21:20:51 +0200596static void stop_wakeup_tracer(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200597{
598 tracer_enabled = 0;
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200599 stop_func_tracer(is_graph());
Steven Rostedt38516ab2010-04-20 17:04:50 -0400600 unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
601 unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
602 unregister_trace_sched_wakeup(probe_wakeup, NULL);
603 unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
Steven Rostedt352ad252008-05-12 21:20:42 +0200604}
605
Steven Rostedt32443512009-01-21 16:24:46 -0500606static int __wakeup_tracer_init(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200607{
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -0400608 save_flags = trace_flags;
609
610 /* non overwrite screws up the latency tracers */
Steven Rostedt2b6080f2012-05-11 13:29:49 -0400611 set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
612 set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
Steven Rostedte9d25fe2009-03-04 22:15:30 -0500613
Steven Rostedt745b1622009-01-15 23:40:11 -0500614 tracing_max_latency = 0;
Steven Rostedt352ad252008-05-12 21:20:42 +0200615 wakeup_trace = tr;
Steven Rostedtc76f0692008-11-07 22:36:02 -0500616 start_wakeup_tracer(tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +0100617 return 0;
Steven Rostedt352ad252008-05-12 21:20:42 +0200618}
619
Steven Rostedt32443512009-01-21 16:24:46 -0500620static int wakeup_tracer_init(struct trace_array *tr)
621{
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100622 wakeup_dl = 0;
Steven Rostedt32443512009-01-21 16:24:46 -0500623 wakeup_rt = 0;
624 return __wakeup_tracer_init(tr);
625}
626
627static int wakeup_rt_tracer_init(struct trace_array *tr)
628{
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100629 wakeup_dl = 0;
Steven Rostedt32443512009-01-21 16:24:46 -0500630 wakeup_rt = 1;
631 return __wakeup_tracer_init(tr);
632}
633
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100634static int wakeup_dl_tracer_init(struct trace_array *tr)
635{
636 wakeup_dl = 1;
637 wakeup_rt = 0;
638 return __wakeup_tracer_init(tr);
639}
640
Ingo Molnare309b412008-05-12 21:20:51 +0200641static void wakeup_tracer_reset(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200642{
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -0400643 int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
644 int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
645
Steven Rostedtc76f0692008-11-07 22:36:02 -0500646 stop_wakeup_tracer(tr);
647 /* make sure we put back any tasks we are tracing */
648 wakeup_reset(tr);
Steven Rostedte9d25fe2009-03-04 22:15:30 -0500649
Steven Rostedt2b6080f2012-05-11 13:29:49 -0400650 set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
651 set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
Steven Rostedt352ad252008-05-12 21:20:42 +0200652}
653
Steven Rostedt90369902008-11-05 16:05:44 -0500654static void wakeup_tracer_start(struct trace_array *tr)
655{
656 wakeup_reset(tr);
657 tracer_enabled = 1;
Steven Rostedt90369902008-11-05 16:05:44 -0500658}
659
660static void wakeup_tracer_stop(struct trace_array *tr)
661{
662 tracer_enabled = 0;
Steven Rostedt352ad252008-05-12 21:20:42 +0200663}
664
665static struct tracer wakeup_tracer __read_mostly =
666{
667 .name = "wakeup",
668 .init = wakeup_tracer_init,
669 .reset = wakeup_tracer_reset,
Steven Rostedt90369902008-11-05 16:05:44 -0500670 .start = wakeup_tracer_start,
671 .stop = wakeup_tracer_stop,
Hiraku Toyookaf43c7382012-10-02 17:27:10 +0900672 .print_max = true,
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200673 .print_header = wakeup_print_header,
674 .print_line = wakeup_print_line,
675 .flags = &tracer_flags,
676 .set_flag = wakeup_set_flag,
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400677 .flag_changed = wakeup_flag_changed,
Steven Rostedt60a11772008-05-12 21:20:44 +0200678#ifdef CONFIG_FTRACE_SELFTEST
679 .selftest = trace_selftest_startup_wakeup,
680#endif
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200681 .open = wakeup_trace_open,
682 .close = wakeup_trace_close,
Hiraku Toyookaf43c7382012-10-02 17:27:10 +0900683 .use_max_tr = true,
Steven Rostedt352ad252008-05-12 21:20:42 +0200684};
685
Steven Rostedt32443512009-01-21 16:24:46 -0500686static struct tracer wakeup_rt_tracer __read_mostly =
687{
688 .name = "wakeup_rt",
689 .init = wakeup_rt_tracer_init,
690 .reset = wakeup_tracer_reset,
691 .start = wakeup_tracer_start,
692 .stop = wakeup_tracer_stop,
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +0100693 .wait_pipe = poll_wait_pipe,
Hiraku Toyookaf43c7382012-10-02 17:27:10 +0900694 .print_max = true,
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200695 .print_header = wakeup_print_header,
696 .print_line = wakeup_print_line,
697 .flags = &tracer_flags,
698 .set_flag = wakeup_set_flag,
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400699 .flag_changed = wakeup_flag_changed,
Steven Rostedt32443512009-01-21 16:24:46 -0500700#ifdef CONFIG_FTRACE_SELFTEST
701 .selftest = trace_selftest_startup_wakeup,
702#endif
Jiri Olsa7495a5be2010-09-23 14:00:53 +0200703 .open = wakeup_trace_open,
704 .close = wakeup_trace_close,
Hiraku Toyookaf43c7382012-10-02 17:27:10 +0900705 .use_max_tr = true,
Steven Rostedt32443512009-01-21 16:24:46 -0500706};
707
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100708static struct tracer wakeup_dl_tracer __read_mostly =
709{
710 .name = "wakeup_dl",
711 .init = wakeup_dl_tracer_init,
712 .reset = wakeup_tracer_reset,
713 .start = wakeup_tracer_start,
714 .stop = wakeup_tracer_stop,
715 .wait_pipe = poll_wait_pipe,
716 .print_max = true,
717 .print_header = wakeup_print_header,
718 .print_line = wakeup_print_line,
719 .flags = &tracer_flags,
720 .set_flag = wakeup_set_flag,
721 .flag_changed = wakeup_flag_changed,
722#ifdef CONFIG_FTRACE_SELFTEST
723 .selftest = trace_selftest_startup_wakeup,
724#endif
725 .open = wakeup_trace_open,
726 .close = wakeup_trace_close,
727 .use_max_tr = true,
728};
729
Steven Rostedt352ad252008-05-12 21:20:42 +0200730__init static int init_wakeup_tracer(void)
731{
732 int ret;
733
734 ret = register_tracer(&wakeup_tracer);
735 if (ret)
736 return ret;
737
Steven Rostedt32443512009-01-21 16:24:46 -0500738 ret = register_tracer(&wakeup_rt_tracer);
739 if (ret)
740 return ret;
741
Dario Faggioliaf6ace72013-11-07 14:43:42 +0100742 ret = register_tracer(&wakeup_dl_tracer);
743 if (ret)
744 return ret;
745
Steven Rostedt352ad252008-05-12 21:20:42 +0200746 return 0;
747}
Steven Rostedt6f415672012-10-05 12:13:07 -0400748core_initcall(init_wakeup_tracer);