blob: a5779bd975db0a386cd67d67629a2bf53e72764e [file] [log] [blame]
Ingo Molnarf06c3812008-05-12 21:20:47 +02001/*
2 * trace stack traces
3 *
Ingo Molnar8a9e94c2008-05-12 21:20:54 +02004 * Copyright (C) 2004-2008, Soeren Sandmann
Ingo Molnarf06c3812008-05-12 21:20:47 +02005 * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
Ingo Molnarf06c3812008-05-12 21:20:47 +02007 */
Ingo Molnar0075fa82008-05-12 21:20:47 +02008#include <linux/kallsyms.h>
9#include <linux/debugfs.h>
10#include <linux/hrtimer.h>
11#include <linux/uaccess.h>
12#include <linux/ftrace.h>
Ingo Molnarf06c3812008-05-12 21:20:47 +020013#include <linux/module.h>
Ingo Molnar56a08bd2008-05-12 21:20:47 +020014#include <linux/irq.h>
Ingo Molnarf06c3812008-05-12 21:20:47 +020015#include <linux/fs.h>
Ingo Molnarf06c3812008-05-12 21:20:47 +020016
Soeren Sandmann Pedersencd2134b2008-05-12 21:20:54 +020017#include <asm/stacktrace.h>
18
Ingo Molnarf06c3812008-05-12 21:20:47 +020019#include "trace.h"
20
Ingo Molnar56a08bd2008-05-12 21:20:47 +020021static struct trace_array *sysprof_trace;
Ingo Molnarf06c3812008-05-12 21:20:47 +020022static int __read_mostly tracer_enabled;
23
Ingo Molnar56a08bd2008-05-12 21:20:47 +020024/*
Ingo Molnard618b3e2008-05-12 21:20:49 +020025 * 1 msec sample interval by default:
Ingo Molnar56a08bd2008-05-12 21:20:47 +020026 */
Ingo Molnard618b3e2008-05-12 21:20:49 +020027static unsigned long sample_period = 1000000;
Ingo Molnar842af312008-05-12 21:20:47 +020028static const unsigned int sample_max_depth = 512;
Ingo Molnar0075fa82008-05-12 21:20:47 +020029
Ingo Molnard618b3e2008-05-12 21:20:49 +020030static DEFINE_MUTEX(sample_timer_lock);
Ingo Molnar0075fa82008-05-12 21:20:47 +020031/*
32 * Per CPU hrtimers that do the profiling:
33 */
34static DEFINE_PER_CPU(struct hrtimer, stack_trace_hrtimer);
35
Ingo Molnar56a08bd2008-05-12 21:20:47 +020036struct stack_frame {
37 const void __user *next_fp;
38 unsigned long return_address;
39};
40
41static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
42{
Ingo Molnar9f6b4e32008-05-12 21:20:48 +020043 int ret;
44
Ingo Molnar56a08bd2008-05-12 21:20:47 +020045 if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
46 return 0;
47
Ingo Molnar9f6b4e32008-05-12 21:20:48 +020048 ret = 1;
49 pagefault_disable();
50 if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
51 ret = 0;
52 pagefault_enable();
Ingo Molnar56a08bd2008-05-12 21:20:47 +020053
Ingo Molnar9f6b4e32008-05-12 21:20:48 +020054 return ret;
Ingo Molnar56a08bd2008-05-12 21:20:47 +020055}
56
Soeren Sandmann Pedersencd2134b2008-05-12 21:20:54 +020057struct backtrace_info {
58 struct trace_array_cpu *data;
59 struct trace_array *tr;
60 int pos;
61};
62
63static void
64backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
65{
66 /* Ignore warnings */
67}
68
69static void backtrace_warning(void *data, char *msg)
70{
71 /* Ignore warnings */
72}
73
74static int backtrace_stack(void *data, char *name)
75{
76 /* Don't bother with IRQ stacks for now */
77 return -1;
78}
79
80static void backtrace_address(void *data, unsigned long addr, int reliable)
81{
82 struct backtrace_info *info = data;
83
84 if (info->pos < sample_max_depth && reliable) {
85 __trace_special(info->tr, info->data, 1, addr, 0);
86
87 info->pos++;
88 }
89}
90
91const static struct stacktrace_ops backtrace_ops = {
92 .warning = backtrace_warning,
93 .warning_symbol = backtrace_warning_symbol,
94 .stack = backtrace_stack,
95 .address = backtrace_address,
96};
97
Soeren Sandmanncf3271a2008-05-12 05:28:50 +020098static int
Soeren Sandmann Pedersencd2134b2008-05-12 21:20:54 +020099trace_kernel(struct pt_regs *regs, struct trace_array *tr,
100 struct trace_array_cpu *data)
101{
102 struct backtrace_info info;
103 unsigned long bp;
Soeren Sandmann Pedersencd2134b2008-05-12 21:20:54 +0200104 char *stack;
105
106 info.tr = tr;
107 info.data = data;
108 info.pos = 1;
109
110 __trace_special(info.tr, info.data, 1, regs->ip, 0);
111
112 stack = ((char *)regs + sizeof(struct pt_regs));
113#ifdef CONFIG_FRAME_POINTER
114 bp = regs->bp;
115#else
116 bp = 0;
117#endif
118
119 dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, &info);
120
Soeren Sandmanncf3271a2008-05-12 05:28:50 +0200121 return info.pos;
Soeren Sandmann Pedersencd2134b2008-05-12 21:20:54 +0200122}
123
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200124static void timer_notify(struct pt_regs *regs, int cpu)
125{
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200126 struct trace_array_cpu *data;
127 struct stack_frame frame;
128 struct trace_array *tr;
Ingo Molnar9f6b4e32008-05-12 21:20:48 +0200129 const void __user *fp;
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200130 int is_user;
131 int i;
132
133 if (!regs)
134 return;
135
136 tr = sysprof_trace;
137 data = tr->data[cpu];
138 is_user = user_mode(regs);
139
140 if (!current || current->pid == 0)
141 return;
142
143 if (is_user && current->state != TASK_RUNNING)
144 return;
145
Soeren Sandmann Pedersencd2134b2008-05-12 21:20:54 +0200146 __trace_special(tr, data, 0, 0, current->pid);
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200147
Soeren Sandmann Pedersencd2134b2008-05-12 21:20:54 +0200148 if (!is_user)
Soeren Sandmanncf3271a2008-05-12 05:28:50 +0200149 i = trace_kernel(regs, tr, data);
150 else
151 i = 0;
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200152
Soeren Sandmanncf3271a2008-05-12 05:28:50 +0200153 /*
154 * Trace user stack if we are not a kernel thread
155 */
156 if (current->mm && i < sample_max_depth) {
157 regs = (struct pt_regs *)current->thread.sp0 - 1;
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200158
Soeren Sandmanncf3271a2008-05-12 05:28:50 +0200159 fp = (void __user *)regs->bp;
Soeren Sandmann Pedersencd2134b2008-05-12 21:20:54 +0200160
Soeren Sandmanncf3271a2008-05-12 05:28:50 +0200161 __trace_special(tr, data, 2, regs->ip, 0);
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200162
Soeren Sandmanncf3271a2008-05-12 05:28:50 +0200163 while (i < sample_max_depth) {
Harvey Harrisona89cc192008-07-25 01:48:39 -0700164 frame.next_fp = NULL;
Soeren Sandmanncf3271a2008-05-12 05:28:50 +0200165 frame.return_address = 0;
166 if (!copy_stack_frame(fp, &frame))
167 break;
168 if ((unsigned long)fp < regs->sp)
169 break;
170
171 __trace_special(tr, data, 2, frame.return_address,
172 (unsigned long)fp);
173 fp = frame.next_fp;
174
175 i++;
176 }
177
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200178 }
179
Ingo Molnar9f6b4e32008-05-12 21:20:48 +0200180 /*
181 * Special trace entry if we overflow the max depth:
182 */
Ingo Molnar842af312008-05-12 21:20:47 +0200183 if (i == sample_max_depth)
Thomas Gleixner9caee612008-05-23 23:55:54 +0200184 __trace_special(tr, data, -1, -1, -1);
Soeren Sandmanncf3271a2008-05-12 05:28:50 +0200185
186 __trace_special(tr, data, 3, current->pid, i);
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200187}
188
Ingo Molnar0075fa82008-05-12 21:20:47 +0200189static enum hrtimer_restart stack_trace_timer_fn(struct hrtimer *hrtimer)
190{
191 /* trace here */
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200192 timer_notify(get_irq_regs(), smp_processor_id());
Ingo Molnar0075fa82008-05-12 21:20:47 +0200193
194 hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
195
196 return HRTIMER_RESTART;
197}
198
199static void start_stack_timer(int cpu)
200{
201 struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu);
202
203 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
204 hrtimer->function = stack_trace_timer_fn;
Ingo Molnar0075fa82008-05-12 21:20:47 +0200205
206 hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL);
207}
208
209static void start_stack_timers(void)
210{
211 cpumask_t saved_mask = current->cpus_allowed;
212 int cpu;
213
214 for_each_online_cpu(cpu) {
Mike Travis0bc3cc032008-07-24 18:21:31 -0700215 set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
Ingo Molnar0075fa82008-05-12 21:20:47 +0200216 start_stack_timer(cpu);
Ingo Molnar0075fa82008-05-12 21:20:47 +0200217 }
218 set_cpus_allowed_ptr(current, &saved_mask);
219}
220
221static void stop_stack_timer(int cpu)
222{
223 struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu);
224
225 hrtimer_cancel(hrtimer);
Ingo Molnar0075fa82008-05-12 21:20:47 +0200226}
227
228static void stop_stack_timers(void)
229{
230 int cpu;
231
232 for_each_online_cpu(cpu)
233 stop_stack_timer(cpu);
234}
235
Thomas Gleixnerada6b832008-05-23 23:50:41 +0200236static void start_stack_trace(struct trace_array *tr)
Ingo Molnarf06c3812008-05-12 21:20:47 +0200237{
Ingo Molnard618b3e2008-05-12 21:20:49 +0200238 mutex_lock(&sample_timer_lock);
Pekka J Enberg213cc062008-12-19 12:08:39 +0200239 tracing_reset_online_cpus(tr);
Ingo Molnar0075fa82008-05-12 21:20:47 +0200240 start_stack_timers();
Ingo Molnarf06c3812008-05-12 21:20:47 +0200241 tracer_enabled = 1;
Ingo Molnard618b3e2008-05-12 21:20:49 +0200242 mutex_unlock(&sample_timer_lock);
Ingo Molnarf06c3812008-05-12 21:20:47 +0200243}
244
Thomas Gleixnerada6b832008-05-23 23:50:41 +0200245static void stop_stack_trace(struct trace_array *tr)
Ingo Molnarf06c3812008-05-12 21:20:47 +0200246{
Ingo Molnard618b3e2008-05-12 21:20:49 +0200247 mutex_lock(&sample_timer_lock);
Ingo Molnar0075fa82008-05-12 21:20:47 +0200248 stop_stack_timers();
Ingo Molnarf06c3812008-05-12 21:20:47 +0200249 tracer_enabled = 0;
Ingo Molnard618b3e2008-05-12 21:20:49 +0200250 mutex_unlock(&sample_timer_lock);
Ingo Molnarf06c3812008-05-12 21:20:47 +0200251}
252
Frederic Weisbecker1c800252008-11-16 05:57:26 +0100253static int stack_trace_init(struct trace_array *tr)
Ingo Molnarf06c3812008-05-12 21:20:47 +0200254{
Ingo Molnar56a08bd2008-05-12 21:20:47 +0200255 sysprof_trace = tr;
Ingo Molnarf06c3812008-05-12 21:20:47 +0200256
Steven Rostedtc76f0692008-11-07 22:36:02 -0500257 start_stack_trace(tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +0100258 return 0;
Ingo Molnarf06c3812008-05-12 21:20:47 +0200259}
260
Thomas Gleixnerada6b832008-05-23 23:50:41 +0200261static void stack_trace_reset(struct trace_array *tr)
Ingo Molnarf06c3812008-05-12 21:20:47 +0200262{
Steven Rostedtc76f0692008-11-07 22:36:02 -0500263 stop_stack_trace(tr);
Ingo Molnarf06c3812008-05-12 21:20:47 +0200264}
265
Ingo Molnarf06c3812008-05-12 21:20:47 +0200266static struct tracer stack_trace __read_mostly =
267{
268 .name = "sysprof",
269 .init = stack_trace_init,
270 .reset = stack_trace_reset,
Ingo Molnarf06c3812008-05-12 21:20:47 +0200271#ifdef CONFIG_FTRACE_SELFTEST
Ingo Molnara6dd24f2008-05-12 21:20:47 +0200272 .selftest = trace_selftest_startup_sysprof,
Ingo Molnarf06c3812008-05-12 21:20:47 +0200273#endif
274};
275
276__init static int init_stack_trace(void)
277{
278 return register_tracer(&stack_trace);
279}
280device_initcall(init_stack_trace);
Ingo Molnard618b3e2008-05-12 21:20:49 +0200281
282#define MAX_LONG_DIGITS 22
283
284static ssize_t
285sysprof_sample_read(struct file *filp, char __user *ubuf,
286 size_t cnt, loff_t *ppos)
287{
288 char buf[MAX_LONG_DIGITS];
289 int r;
290
291 r = sprintf(buf, "%ld\n", nsecs_to_usecs(sample_period));
292
293 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
294}
295
296static ssize_t
297sysprof_sample_write(struct file *filp, const char __user *ubuf,
298 size_t cnt, loff_t *ppos)
299{
300 char buf[MAX_LONG_DIGITS];
301 unsigned long val;
302
303 if (cnt > MAX_LONG_DIGITS-1)
304 cnt = MAX_LONG_DIGITS-1;
305
306 if (copy_from_user(&buf, ubuf, cnt))
307 return -EFAULT;
308
309 buf[cnt] = 0;
310
311 val = simple_strtoul(buf, NULL, 10);
312 /*
313 * Enforce a minimum sample period of 100 usecs:
314 */
315 if (val < 100)
316 val = 100;
317
318 mutex_lock(&sample_timer_lock);
319 stop_stack_timers();
320 sample_period = val * 1000;
321 start_stack_timers();
322 mutex_unlock(&sample_timer_lock);
323
324 return cnt;
325}
326
327static struct file_operations sysprof_sample_fops = {
328 .read = sysprof_sample_read,
329 .write = sysprof_sample_write,
330};
331
332void init_tracer_sysprof_debugfs(struct dentry *d_tracer)
333{
334 struct dentry *entry;
335
336 entry = debugfs_create_file("sysprof_sample_period", 0644,
337 d_tracer, NULL, &sysprof_sample_fops);
338 if (entry)
339 return;
340 pr_warning("Could not create debugfs 'dyn_ftrace_total_info' entry\n");
341}