Cyrill Gorcunov | 47a486c | 2008-06-24 22:52:03 +0200 | [diff] [blame] | 1 | /* |
| 2 | * local apic based NMI watchdog for various CPUs. |
| 3 | * |
| 4 | * This file also handles reservation of performance counters for coordination |
| 5 | * with other users (like oprofile). |
| 6 | * |
| 7 | * Note that these events normally don't tick when the CPU idles. This means |
| 8 | * the frequency varies with CPU load. |
| 9 | * |
| 10 | * Original code for K7/P6 written by Keith Owens |
| 11 | * |
| 12 | */ |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 13 | |
| 14 | #include <linux/percpu.h> |
| 15 | #include <linux/module.h> |
| 16 | #include <linux/kernel.h> |
| 17 | #include <linux/bitops.h> |
| 18 | #include <linux/smp.h> |
Don Zickus | 4a7863c | 2010-12-22 14:00:03 -0500 | [diff] [blame] | 19 | #include <asm/nmi.h> |
Ingo Molnar | 8b1fa1d | 2008-07-29 12:36:02 +0200 | [diff] [blame] | 20 | #include <linux/kprobes.h> |
| 21 | |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 22 | #include <asm/apic.h> |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 23 | #include <asm/perf_event.h> |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 24 | |
Cyrill Gorcunov | 47a486c | 2008-06-24 22:52:03 +0200 | [diff] [blame] | 25 | /* |
| 26 | * this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's |
| 27 | * offset from MSR_P4_BSU_ESCR0. |
| 28 | * |
| 29 | * It will be the max for all platforms (for now) |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 30 | */ |
| 31 | #define NMI_MAX_COUNTER_BITS 66 |
| 32 | |
Cyrill Gorcunov | 47a486c | 2008-06-24 22:52:03 +0200 | [diff] [blame] | 33 | /* |
| 34 | * perfctr_nmi_owner tracks the ownership of the perfctr registers: |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 35 | * evtsel_nmi_owner tracks the ownership of the event selection |
| 36 | * - different performance counters/ event selection may be reserved for |
| 37 | * different subsystems this reservation system just tries to coordinate |
| 38 | * things a little |
| 39 | */ |
| 40 | static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS); |
| 41 | static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS); |
| 42 | |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 43 | /* converts an msr to an appropriate reservation bit */ |
| 44 | static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr) |
| 45 | { |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 46 | /* returns the bit offset of the performance counter register */ |
| 47 | switch (boot_cpu_data.x86_vendor) { |
| 48 | case X86_VENDOR_AMD: |
Robert Richter | 69d8e1e | 2011-02-02 17:40:58 +0100 | [diff] [blame] | 49 | if (msr >= MSR_F15H_PERF_CTR) |
| 50 | return (msr - MSR_F15H_PERF_CTR) >> 1; |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 51 | return msr - MSR_K7_PERFCTR0; |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 52 | case X86_VENDOR_INTEL: |
| 53 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 54 | return msr - MSR_ARCH_PERFMON_PERFCTR0; |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 55 | |
| 56 | switch (boot_cpu_data.x86) { |
| 57 | case 6: |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 58 | return msr - MSR_P6_PERFCTR0; |
Vince Weaver | e717bf4 | 2012-09-26 14:12:52 -0400 | [diff] [blame] | 59 | case 11: |
| 60 | return msr - MSR_KNC_PERFCTR0; |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 61 | case 15: |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 62 | return msr - MSR_P4_BPU_PERFCTR0; |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 63 | } |
| 64 | } |
| 65 | return 0; |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 66 | } |
| 67 | |
Cyrill Gorcunov | 47a486c | 2008-06-24 22:52:03 +0200 | [diff] [blame] | 68 | /* |
| 69 | * converts an msr to an appropriate reservation bit |
| 70 | * returns the bit offset of the event selection register |
| 71 | */ |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 72 | static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr) |
| 73 | { |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 74 | /* returns the bit offset of the event selection register */ |
| 75 | switch (boot_cpu_data.x86_vendor) { |
| 76 | case X86_VENDOR_AMD: |
Robert Richter | 69d8e1e | 2011-02-02 17:40:58 +0100 | [diff] [blame] | 77 | if (msr >= MSR_F15H_PERF_CTL) |
| 78 | return (msr - MSR_F15H_PERF_CTL) >> 1; |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 79 | return msr - MSR_K7_EVNTSEL0; |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 80 | case X86_VENDOR_INTEL: |
| 81 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 82 | return msr - MSR_ARCH_PERFMON_EVENTSEL0; |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 83 | |
| 84 | switch (boot_cpu_data.x86) { |
| 85 | case 6: |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 86 | return msr - MSR_P6_EVNTSEL0; |
Vince Weaver | e717bf4 | 2012-09-26 14:12:52 -0400 | [diff] [blame] | 87 | case 11: |
| 88 | return msr - MSR_KNC_EVNTSEL0; |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 89 | case 15: |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 90 | return msr - MSR_P4_BSU_ESCR0; |
Andi Kleen | 5dcccd8 | 2007-07-04 01:38:13 +0200 | [diff] [blame] | 91 | } |
| 92 | } |
| 93 | return 0; |
| 94 | |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 95 | } |
| 96 | |
| 97 | /* checks for a bit availability (hack for oprofile) */ |
| 98 | int avail_to_resrv_perfctr_nmi_bit(unsigned int counter) |
| 99 | { |
| 100 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); |
| 101 | |
Alan Cox | 8bdbd96 | 2009-07-04 00:35:45 +0100 | [diff] [blame] | 102 | return !test_bit(counter, perfctr_nmi_owner); |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 103 | } |
Cyrill Gorcunov | 47a486c | 2008-06-24 22:52:03 +0200 | [diff] [blame] | 104 | EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit); |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 105 | |
| 106 | int reserve_perfctr_nmi(unsigned int msr) |
| 107 | { |
| 108 | unsigned int counter; |
| 109 | |
| 110 | counter = nmi_perfctr_msr_to_bit(msr); |
Stephane Eranian | 124d395 | 2007-10-19 20:35:04 +0200 | [diff] [blame] | 111 | /* register not managed by the allocator? */ |
| 112 | if (counter > NMI_MAX_COUNTER_BITS) |
| 113 | return 1; |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 114 | |
| 115 | if (!test_and_set_bit(counter, perfctr_nmi_owner)) |
| 116 | return 1; |
| 117 | return 0; |
| 118 | } |
Cyrill Gorcunov | 47a486c | 2008-06-24 22:52:03 +0200 | [diff] [blame] | 119 | EXPORT_SYMBOL(reserve_perfctr_nmi); |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 120 | |
| 121 | void release_perfctr_nmi(unsigned int msr) |
| 122 | { |
| 123 | unsigned int counter; |
| 124 | |
| 125 | counter = nmi_perfctr_msr_to_bit(msr); |
Stephane Eranian | 124d395 | 2007-10-19 20:35:04 +0200 | [diff] [blame] | 126 | /* register not managed by the allocator? */ |
| 127 | if (counter > NMI_MAX_COUNTER_BITS) |
| 128 | return; |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 129 | |
| 130 | clear_bit(counter, perfctr_nmi_owner); |
| 131 | } |
Cyrill Gorcunov | 47a486c | 2008-06-24 22:52:03 +0200 | [diff] [blame] | 132 | EXPORT_SYMBOL(release_perfctr_nmi); |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 133 | |
| 134 | int reserve_evntsel_nmi(unsigned int msr) |
| 135 | { |
| 136 | unsigned int counter; |
| 137 | |
| 138 | counter = nmi_evntsel_msr_to_bit(msr); |
Stephane Eranian | 124d395 | 2007-10-19 20:35:04 +0200 | [diff] [blame] | 139 | /* register not managed by the allocator? */ |
| 140 | if (counter > NMI_MAX_COUNTER_BITS) |
| 141 | return 1; |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 142 | |
| 143 | if (!test_and_set_bit(counter, evntsel_nmi_owner)) |
| 144 | return 1; |
| 145 | return 0; |
| 146 | } |
Cyrill Gorcunov | 47a486c | 2008-06-24 22:52:03 +0200 | [diff] [blame] | 147 | EXPORT_SYMBOL(reserve_evntsel_nmi); |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 148 | |
| 149 | void release_evntsel_nmi(unsigned int msr) |
| 150 | { |
| 151 | unsigned int counter; |
| 152 | |
| 153 | counter = nmi_evntsel_msr_to_bit(msr); |
Stephane Eranian | 124d395 | 2007-10-19 20:35:04 +0200 | [diff] [blame] | 154 | /* register not managed by the allocator? */ |
| 155 | if (counter > NMI_MAX_COUNTER_BITS) |
| 156 | return; |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 157 | |
| 158 | clear_bit(counter, evntsel_nmi_owner); |
| 159 | } |
Andi Kleen | 09198e6 | 2007-05-02 19:27:20 +0200 | [diff] [blame] | 160 | EXPORT_SYMBOL(release_evntsel_nmi); |