Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 1 | /* |
| 2 | * trace stack traces |
| 3 | * |
Ingo Molnar | 8a9e94c | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 4 | * Copyright (C) 2004-2008, Soeren Sandmann |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 5 | * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com> |
| 6 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 7 | */ |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 8 | #include <linux/kallsyms.h> |
| 9 | #include <linux/debugfs.h> |
| 10 | #include <linux/hrtimer.h> |
| 11 | #include <linux/uaccess.h> |
| 12 | #include <linux/ftrace.h> |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 13 | #include <linux/module.h> |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 14 | #include <linux/irq.h> |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 15 | #include <linux/fs.h> |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 16 | |
Soeren Sandmann Pedersen | cd2134b | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 17 | #include <asm/stacktrace.h> |
| 18 | |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 19 | #include "trace.h" |
| 20 | |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 21 | static struct trace_array *sysprof_trace; |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 22 | static int __read_mostly tracer_enabled; |
| 23 | |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 24 | /* |
Ingo Molnar | d618b3e | 2008-05-12 21:20:49 +0200 | [diff] [blame] | 25 | * 1 msec sample interval by default: |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 26 | */ |
Ingo Molnar | d618b3e | 2008-05-12 21:20:49 +0200 | [diff] [blame] | 27 | static unsigned long sample_period = 1000000; |
Ingo Molnar | 842af31 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 28 | static const unsigned int sample_max_depth = 512; |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 29 | |
Ingo Molnar | d618b3e | 2008-05-12 21:20:49 +0200 | [diff] [blame] | 30 | static DEFINE_MUTEX(sample_timer_lock); |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 31 | /* |
| 32 | * Per CPU hrtimers that do the profiling: |
| 33 | */ |
| 34 | static DEFINE_PER_CPU(struct hrtimer, stack_trace_hrtimer); |
| 35 | |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 36 | struct stack_frame { |
| 37 | const void __user *next_fp; |
| 38 | unsigned long return_address; |
| 39 | }; |
| 40 | |
| 41 | static int copy_stack_frame(const void __user *fp, struct stack_frame *frame) |
| 42 | { |
Ingo Molnar | 9f6b4e3 | 2008-05-12 21:20:48 +0200 | [diff] [blame] | 43 | int ret; |
| 44 | |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 45 | if (!access_ok(VERIFY_READ, fp, sizeof(*frame))) |
| 46 | return 0; |
| 47 | |
Ingo Molnar | 9f6b4e3 | 2008-05-12 21:20:48 +0200 | [diff] [blame] | 48 | ret = 1; |
| 49 | pagefault_disable(); |
| 50 | if (__copy_from_user_inatomic(frame, fp, sizeof(*frame))) |
| 51 | ret = 0; |
| 52 | pagefault_enable(); |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 53 | |
Ingo Molnar | 9f6b4e3 | 2008-05-12 21:20:48 +0200 | [diff] [blame] | 54 | return ret; |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 55 | } |
| 56 | |
Soeren Sandmann Pedersen | cd2134b | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 57 | struct backtrace_info { |
| 58 | struct trace_array_cpu *data; |
| 59 | struct trace_array *tr; |
| 60 | int pos; |
| 61 | }; |
| 62 | |
| 63 | static void |
| 64 | backtrace_warning_symbol(void *data, char *msg, unsigned long symbol) |
| 65 | { |
| 66 | /* Ignore warnings */ |
| 67 | } |
| 68 | |
| 69 | static void backtrace_warning(void *data, char *msg) |
| 70 | { |
| 71 | /* Ignore warnings */ |
| 72 | } |
| 73 | |
| 74 | static int backtrace_stack(void *data, char *name) |
| 75 | { |
| 76 | /* Don't bother with IRQ stacks for now */ |
| 77 | return -1; |
| 78 | } |
| 79 | |
| 80 | static void backtrace_address(void *data, unsigned long addr, int reliable) |
| 81 | { |
| 82 | struct backtrace_info *info = data; |
| 83 | |
| 84 | if (info->pos < sample_max_depth && reliable) { |
| 85 | __trace_special(info->tr, info->data, 1, addr, 0); |
| 86 | |
| 87 | info->pos++; |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | const static struct stacktrace_ops backtrace_ops = { |
| 92 | .warning = backtrace_warning, |
| 93 | .warning_symbol = backtrace_warning_symbol, |
| 94 | .stack = backtrace_stack, |
| 95 | .address = backtrace_address, |
| 96 | }; |
| 97 | |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 98 | static int |
Soeren Sandmann Pedersen | cd2134b | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 99 | trace_kernel(struct pt_regs *regs, struct trace_array *tr, |
| 100 | struct trace_array_cpu *data) |
| 101 | { |
| 102 | struct backtrace_info info; |
| 103 | unsigned long bp; |
Soeren Sandmann Pedersen | cd2134b | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 104 | char *stack; |
| 105 | |
| 106 | info.tr = tr; |
| 107 | info.data = data; |
| 108 | info.pos = 1; |
| 109 | |
| 110 | __trace_special(info.tr, info.data, 1, regs->ip, 0); |
| 111 | |
| 112 | stack = ((char *)regs + sizeof(struct pt_regs)); |
| 113 | #ifdef CONFIG_FRAME_POINTER |
| 114 | bp = regs->bp; |
| 115 | #else |
| 116 | bp = 0; |
| 117 | #endif |
| 118 | |
| 119 | dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, &info); |
| 120 | |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 121 | return info.pos; |
Soeren Sandmann Pedersen | cd2134b | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 122 | } |
| 123 | |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 124 | static void timer_notify(struct pt_regs *regs, int cpu) |
| 125 | { |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 126 | struct trace_array_cpu *data; |
| 127 | struct stack_frame frame; |
| 128 | struct trace_array *tr; |
Ingo Molnar | 9f6b4e3 | 2008-05-12 21:20:48 +0200 | [diff] [blame] | 129 | const void __user *fp; |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 130 | int is_user; |
| 131 | int i; |
| 132 | |
| 133 | if (!regs) |
| 134 | return; |
| 135 | |
| 136 | tr = sysprof_trace; |
| 137 | data = tr->data[cpu]; |
| 138 | is_user = user_mode(regs); |
| 139 | |
| 140 | if (!current || current->pid == 0) |
| 141 | return; |
| 142 | |
| 143 | if (is_user && current->state != TASK_RUNNING) |
| 144 | return; |
| 145 | |
Soeren Sandmann Pedersen | cd2134b | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 146 | __trace_special(tr, data, 0, 0, current->pid); |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 147 | |
Soeren Sandmann Pedersen | cd2134b | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 148 | if (!is_user) |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 149 | i = trace_kernel(regs, tr, data); |
| 150 | else |
| 151 | i = 0; |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 152 | |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 153 | /* |
| 154 | * Trace user stack if we are not a kernel thread |
| 155 | */ |
| 156 | if (current->mm && i < sample_max_depth) { |
| 157 | regs = (struct pt_regs *)current->thread.sp0 - 1; |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 158 | |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 159 | fp = (void __user *)regs->bp; |
Soeren Sandmann Pedersen | cd2134b | 2008-05-12 21:20:54 +0200 | [diff] [blame] | 160 | |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 161 | __trace_special(tr, data, 2, regs->ip, 0); |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 162 | |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 163 | while (i < sample_max_depth) { |
Harvey Harrison | a89cc19 | 2008-07-25 01:48:39 -0700 | [diff] [blame] | 164 | frame.next_fp = NULL; |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 165 | frame.return_address = 0; |
| 166 | if (!copy_stack_frame(fp, &frame)) |
| 167 | break; |
| 168 | if ((unsigned long)fp < regs->sp) |
| 169 | break; |
| 170 | |
| 171 | __trace_special(tr, data, 2, frame.return_address, |
| 172 | (unsigned long)fp); |
| 173 | fp = frame.next_fp; |
| 174 | |
| 175 | i++; |
| 176 | } |
| 177 | |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 178 | } |
| 179 | |
Ingo Molnar | 9f6b4e3 | 2008-05-12 21:20:48 +0200 | [diff] [blame] | 180 | /* |
| 181 | * Special trace entry if we overflow the max depth: |
| 182 | */ |
Ingo Molnar | 842af31 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 183 | if (i == sample_max_depth) |
Thomas Gleixner | 9caee61 | 2008-05-23 23:55:54 +0200 | [diff] [blame] | 184 | __trace_special(tr, data, -1, -1, -1); |
Soeren Sandmann | cf3271a | 2008-05-12 05:28:50 +0200 | [diff] [blame] | 185 | |
| 186 | __trace_special(tr, data, 3, current->pid, i); |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 187 | } |
| 188 | |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 189 | static enum hrtimer_restart stack_trace_timer_fn(struct hrtimer *hrtimer) |
| 190 | { |
| 191 | /* trace here */ |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 192 | timer_notify(get_irq_regs(), smp_processor_id()); |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 193 | |
| 194 | hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); |
| 195 | |
| 196 | return HRTIMER_RESTART; |
| 197 | } |
| 198 | |
| 199 | static void start_stack_timer(int cpu) |
| 200 | { |
| 201 | struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu); |
| 202 | |
| 203 | hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 204 | hrtimer->function = stack_trace_timer_fn; |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 205 | |
| 206 | hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL); |
| 207 | } |
| 208 | |
| 209 | static void start_stack_timers(void) |
| 210 | { |
| 211 | cpumask_t saved_mask = current->cpus_allowed; |
| 212 | int cpu; |
| 213 | |
| 214 | for_each_online_cpu(cpu) { |
Mike Travis | 0bc3cc03 | 2008-07-24 18:21:31 -0700 | [diff] [blame] | 215 | set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 216 | start_stack_timer(cpu); |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 217 | } |
| 218 | set_cpus_allowed_ptr(current, &saved_mask); |
| 219 | } |
| 220 | |
| 221 | static void stop_stack_timer(int cpu) |
| 222 | { |
| 223 | struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu); |
| 224 | |
| 225 | hrtimer_cancel(hrtimer); |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 226 | } |
| 227 | |
| 228 | static void stop_stack_timers(void) |
| 229 | { |
| 230 | int cpu; |
| 231 | |
| 232 | for_each_online_cpu(cpu) |
| 233 | stop_stack_timer(cpu); |
| 234 | } |
| 235 | |
Thomas Gleixner | ada6b83 | 2008-05-23 23:50:41 +0200 | [diff] [blame] | 236 | static void start_stack_trace(struct trace_array *tr) |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 237 | { |
Ingo Molnar | d618b3e | 2008-05-12 21:20:49 +0200 | [diff] [blame] | 238 | mutex_lock(&sample_timer_lock); |
Pekka J Enberg | 213cc06 | 2008-12-19 12:08:39 +0200 | [diff] [blame] | 239 | tracing_reset_online_cpus(tr); |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 240 | start_stack_timers(); |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 241 | tracer_enabled = 1; |
Ingo Molnar | d618b3e | 2008-05-12 21:20:49 +0200 | [diff] [blame] | 242 | mutex_unlock(&sample_timer_lock); |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 243 | } |
| 244 | |
Thomas Gleixner | ada6b83 | 2008-05-23 23:50:41 +0200 | [diff] [blame] | 245 | static void stop_stack_trace(struct trace_array *tr) |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 246 | { |
Ingo Molnar | d618b3e | 2008-05-12 21:20:49 +0200 | [diff] [blame] | 247 | mutex_lock(&sample_timer_lock); |
Ingo Molnar | 0075fa8 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 248 | stop_stack_timers(); |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 249 | tracer_enabled = 0; |
Ingo Molnar | d618b3e | 2008-05-12 21:20:49 +0200 | [diff] [blame] | 250 | mutex_unlock(&sample_timer_lock); |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 251 | } |
| 252 | |
Frederic Weisbecker | 1c80025 | 2008-11-16 05:57:26 +0100 | [diff] [blame] | 253 | static int stack_trace_init(struct trace_array *tr) |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 254 | { |
Ingo Molnar | 56a08bd | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 255 | sysprof_trace = tr; |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 256 | |
Steven Rostedt | c76f069 | 2008-11-07 22:36:02 -0500 | [diff] [blame] | 257 | start_stack_trace(tr); |
Frederic Weisbecker | 1c80025 | 2008-11-16 05:57:26 +0100 | [diff] [blame] | 258 | return 0; |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 259 | } |
| 260 | |
Thomas Gleixner | ada6b83 | 2008-05-23 23:50:41 +0200 | [diff] [blame] | 261 | static void stack_trace_reset(struct trace_array *tr) |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 262 | { |
Steven Rostedt | c76f069 | 2008-11-07 22:36:02 -0500 | [diff] [blame] | 263 | stop_stack_trace(tr); |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 264 | } |
| 265 | |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 266 | static struct tracer stack_trace __read_mostly = |
| 267 | { |
| 268 | .name = "sysprof", |
| 269 | .init = stack_trace_init, |
| 270 | .reset = stack_trace_reset, |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 271 | #ifdef CONFIG_FTRACE_SELFTEST |
Ingo Molnar | a6dd24f | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 272 | .selftest = trace_selftest_startup_sysprof, |
Ingo Molnar | f06c381 | 2008-05-12 21:20:47 +0200 | [diff] [blame] | 273 | #endif |
| 274 | }; |
| 275 | |
| 276 | __init static int init_stack_trace(void) |
| 277 | { |
| 278 | return register_tracer(&stack_trace); |
| 279 | } |
| 280 | device_initcall(init_stack_trace); |
Ingo Molnar | d618b3e | 2008-05-12 21:20:49 +0200 | [diff] [blame] | 281 | |
| 282 | #define MAX_LONG_DIGITS 22 |
| 283 | |
| 284 | static ssize_t |
| 285 | sysprof_sample_read(struct file *filp, char __user *ubuf, |
| 286 | size_t cnt, loff_t *ppos) |
| 287 | { |
| 288 | char buf[MAX_LONG_DIGITS]; |
| 289 | int r; |
| 290 | |
| 291 | r = sprintf(buf, "%ld\n", nsecs_to_usecs(sample_period)); |
| 292 | |
| 293 | return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); |
| 294 | } |
| 295 | |
| 296 | static ssize_t |
| 297 | sysprof_sample_write(struct file *filp, const char __user *ubuf, |
| 298 | size_t cnt, loff_t *ppos) |
| 299 | { |
| 300 | char buf[MAX_LONG_DIGITS]; |
| 301 | unsigned long val; |
| 302 | |
| 303 | if (cnt > MAX_LONG_DIGITS-1) |
| 304 | cnt = MAX_LONG_DIGITS-1; |
| 305 | |
| 306 | if (copy_from_user(&buf, ubuf, cnt)) |
| 307 | return -EFAULT; |
| 308 | |
| 309 | buf[cnt] = 0; |
| 310 | |
| 311 | val = simple_strtoul(buf, NULL, 10); |
| 312 | /* |
| 313 | * Enforce a minimum sample period of 100 usecs: |
| 314 | */ |
| 315 | if (val < 100) |
| 316 | val = 100; |
| 317 | |
| 318 | mutex_lock(&sample_timer_lock); |
| 319 | stop_stack_timers(); |
| 320 | sample_period = val * 1000; |
| 321 | start_stack_timers(); |
| 322 | mutex_unlock(&sample_timer_lock); |
| 323 | |
| 324 | return cnt; |
| 325 | } |
| 326 | |
| 327 | static struct file_operations sysprof_sample_fops = { |
| 328 | .read = sysprof_sample_read, |
| 329 | .write = sysprof_sample_write, |
| 330 | }; |
| 331 | |
| 332 | void init_tracer_sysprof_debugfs(struct dentry *d_tracer) |
| 333 | { |
| 334 | struct dentry *entry; |
| 335 | |
| 336 | entry = debugfs_create_file("sysprof_sample_period", 0644, |
| 337 | d_tracer, NULL, &sysprof_sample_fops); |
| 338 | if (entry) |
| 339 | return; |
| 340 | pr_warning("Could not create debugfs 'dyn_ftrace_total_info' entry\n"); |
| 341 | } |