Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Performance event support - powerpc architecture code |
| 3 | * |
| 4 | * Copyright 2008-2009 Paul Mackerras, IBM Corporation. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | */ |
| 11 | #include <linux/kernel.h> |
| 12 | #include <linux/sched.h> |
| 13 | #include <linux/perf_event.h> |
| 14 | #include <linux/percpu.h> |
| 15 | #include <linux/hardirq.h> |
Michael Neuling | 6912318 | 2013-05-13 18:44:58 +0000 | [diff] [blame] | 16 | #include <linux/uaccess.h> |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 17 | #include <asm/reg.h> |
| 18 | #include <asm/pmc.h> |
| 19 | #include <asm/machdep.h> |
| 20 | #include <asm/firmware.h> |
| 21 | #include <asm/ptrace.h> |
Michael Neuling | 6912318 | 2013-05-13 18:44:58 +0000 | [diff] [blame] | 22 | #include <asm/code-patching.h> |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 23 | |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 24 | #define BHRB_MAX_ENTRIES 32 |
| 25 | #define BHRB_TARGET 0x0000000000000002 |
| 26 | #define BHRB_PREDICTION 0x0000000000000001 |
Anton Blanchard | b0d436c | 2013-08-07 02:01:24 +1000 | [diff] [blame] | 27 | #define BHRB_EA 0xFFFFFFFFFFFFFFFCUL |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 28 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 29 | struct cpu_hw_events { |
| 30 | int n_events; |
| 31 | int n_percpu; |
| 32 | int disabled; |
| 33 | int n_added; |
| 34 | int n_limited; |
| 35 | u8 pmcs_enabled; |
| 36 | struct perf_event *event[MAX_HWEVENTS]; |
| 37 | u64 events[MAX_HWEVENTS]; |
| 38 | unsigned int flags[MAX_HWEVENTS]; |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 39 | /* |
| 40 | * The order of the MMCR array is: |
| 41 | * - 64-bit, MMCR0, MMCR1, MMCRA, MMCR2 |
| 42 | * - 32-bit, MMCR0, MMCR1, MMCR2 |
| 43 | */ |
| 44 | unsigned long mmcr[4]; |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 45 | struct perf_event *limited_counter[MAX_LIMITED_HWCOUNTERS]; |
| 46 | u8 limited_hwidx[MAX_LIMITED_HWCOUNTERS]; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 47 | u64 alternatives[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES]; |
| 48 | unsigned long amasks[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES]; |
| 49 | unsigned long avalues[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES]; |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 50 | |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 51 | unsigned int txn_flags; |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 52 | int n_txn_start; |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 53 | |
| 54 | /* BHRB bits */ |
| 55 | u64 bhrb_filter; /* BHRB HW branch filter */ |
Anshuman Khandual | f0322f7 | 2015-06-30 13:50:28 +0530 | [diff] [blame] | 56 | unsigned int bhrb_users; |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 57 | void *bhrb_context; |
| 58 | struct perf_branch_stack bhrb_stack; |
| 59 | struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES]; |
Madhavan Srinivasan | 356d8ce | 2017-02-12 22:33:14 +0530 | [diff] [blame] | 60 | u64 ic_init; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 61 | }; |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 62 | |
Anton Blanchard | e51df2c | 2014-08-20 08:55:18 +1000 | [diff] [blame] | 63 | static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 64 | |
Anton Blanchard | e51df2c | 2014-08-20 08:55:18 +1000 | [diff] [blame] | 65 | static struct power_pmu *ppmu; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 66 | |
| 67 | /* |
Ingo Molnar | 57c0c15 | 2009-09-21 12:20:38 +0200 | [diff] [blame] | 68 | * Normally, to ignore kernel events we set the FCS (freeze counters |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 69 | * in supervisor mode) bit in MMCR0, but if the kernel runs with the |
| 70 | * hypervisor bit set in the MSR, or if we are running on a processor |
| 71 | * where the hypervisor bit is forced to 1 (as on Apple G5 processors), |
| 72 | * then we need to use the FCHV bit to ignore kernel events. |
| 73 | */ |
| 74 | static unsigned int freeze_events_kernel = MMCR0_FCS; |
| 75 | |
| 76 | /* |
| 77 | * 32-bit doesn't have MMCRA but does have an MMCR2, |
| 78 | * and a few other names are different. |
| 79 | */ |
| 80 | #ifdef CONFIG_PPC32 |
| 81 | |
| 82 | #define MMCR0_FCHV 0 |
| 83 | #define MMCR0_PMCjCE MMCR0_PMCnCE |
Michael Ellerman | 7a7a41f | 2013-06-28 18:15:12 +1000 | [diff] [blame] | 84 | #define MMCR0_FC56 0 |
Michael Ellerman | 378a6ee | 2013-06-28 18:15:11 +1000 | [diff] [blame] | 85 | #define MMCR0_PMAO 0 |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 86 | #define MMCR0_EBE 0 |
Michael Ellerman | 76cb8a7 | 2014-03-14 16:00:34 +1100 | [diff] [blame] | 87 | #define MMCR0_BHRBA 0 |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 88 | #define MMCR0_PMCC 0 |
| 89 | #define MMCR0_PMCC_U6 0 |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 90 | |
| 91 | #define SPRN_MMCRA SPRN_MMCR2 |
| 92 | #define MMCRA_SAMPLE_ENABLE 0 |
| 93 | |
| 94 | static inline unsigned long perf_ip_adjust(struct pt_regs *regs) |
| 95 | { |
| 96 | return 0; |
| 97 | } |
| 98 | static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp) { } |
| 99 | static inline u32 perf_get_misc_flags(struct pt_regs *regs) |
| 100 | { |
| 101 | return 0; |
| 102 | } |
Anton Blanchard | 75382aa | 2012-06-26 01:01:36 +0000 | [diff] [blame] | 103 | static inline void perf_read_regs(struct pt_regs *regs) |
| 104 | { |
| 105 | regs->result = 0; |
| 106 | } |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 107 | static inline int perf_intr_is_nmi(struct pt_regs *regs) |
| 108 | { |
| 109 | return 0; |
| 110 | } |
| 111 | |
sukadev@linux.vnet.ibm.com | e687883 | 2012-09-18 20:56:11 +0000 | [diff] [blame] | 112 | static inline int siar_valid(struct pt_regs *regs) |
| 113 | { |
| 114 | return 1; |
| 115 | } |
| 116 | |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 117 | static bool is_ebb_event(struct perf_event *event) { return false; } |
| 118 | static int ebb_event_check(struct perf_event *event) { return 0; } |
| 119 | static void ebb_event_add(struct perf_event *event) { } |
| 120 | static void ebb_switch_out(unsigned long mmcr0) { } |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 121 | static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw) |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 122 | { |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 123 | return cpuhw->mmcr[0]; |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 124 | } |
| 125 | |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 126 | static inline void power_pmu_bhrb_enable(struct perf_event *event) {} |
| 127 | static inline void power_pmu_bhrb_disable(struct perf_event *event) {} |
Peter Zijlstra | acba3c7 | 2015-01-14 14:15:39 +0100 | [diff] [blame] | 128 | static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) {} |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 129 | static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {} |
Michael Ellerman | c2e37a2 | 2014-03-14 16:00:29 +1100 | [diff] [blame] | 130 | static void pmao_restore_workaround(bool ebb) { } |
Madhavan Srinivasan | 356d8ce | 2017-02-12 22:33:14 +0530 | [diff] [blame] | 131 | static bool use_ic(u64 event) |
| 132 | { |
| 133 | return false; |
| 134 | } |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 135 | #endif /* CONFIG_PPC32 */ |
| 136 | |
Michael Ellerman | 3390405 | 2013-04-25 19:28:25 +0000 | [diff] [blame] | 137 | static bool regs_use_siar(struct pt_regs *regs) |
| 138 | { |
Anton Blanchard | 72e349f | 2015-05-26 15:10:24 +1000 | [diff] [blame] | 139 | /* |
| 140 | * When we take a performance monitor exception the regs are setup |
| 141 | * using perf_read_regs() which overloads some fields, in particular |
| 142 | * regs->result to tell us whether to use SIAR. |
| 143 | * |
| 144 | * However if the regs are from another exception, eg. a syscall, then |
| 145 | * they have not been setup using perf_read_regs() and so regs->result |
| 146 | * is something random. |
| 147 | */ |
| 148 | return ((TRAP(regs) == 0xf00) && regs->result); |
Michael Ellerman | 3390405 | 2013-04-25 19:28:25 +0000 | [diff] [blame] | 149 | } |
| 150 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 151 | /* |
| 152 | * Things that are specific to 64-bit implementations. |
| 153 | */ |
| 154 | #ifdef CONFIG_PPC64 |
| 155 | |
| 156 | static inline unsigned long perf_ip_adjust(struct pt_regs *regs) |
| 157 | { |
| 158 | unsigned long mmcra = regs->dsisr; |
| 159 | |
Michael Ellerman | 7a78683 | 2013-04-25 19:28:23 +0000 | [diff] [blame] | 160 | if ((ppmu->flags & PPMU_HAS_SSLOT) && (mmcra & MMCRA_SAMPLE_ENABLE)) { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 161 | unsigned long slot = (mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT; |
| 162 | if (slot > 1) |
| 163 | return 4 * (slot - 1); |
| 164 | } |
Michael Ellerman | 7a78683 | 2013-04-25 19:28:23 +0000 | [diff] [blame] | 165 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 166 | return 0; |
| 167 | } |
| 168 | |
| 169 | /* |
| 170 | * The user wants a data address recorded. |
| 171 | * If we're not doing instruction sampling, give them the SDAR |
| 172 | * (sampled data address). If we are doing instruction sampling, then |
| 173 | * only give them the SDAR if it corresponds to the instruction |
Michael Ellerman | 58a032c | 2013-05-15 20:19:31 +0000 | [diff] [blame] | 174 | * pointed to by SIAR; this is indicated by the [POWER6_]MMCRA_SDSYNC, the |
| 175 | * [POWER7P_]MMCRA_SDAR_VALID bit in MMCRA, or the SDAR_VALID bit in SIER. |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 176 | */ |
| 177 | static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp) |
| 178 | { |
| 179 | unsigned long mmcra = regs->dsisr; |
Michael Ellerman | 58a032c | 2013-05-15 20:19:31 +0000 | [diff] [blame] | 180 | bool sdar_valid; |
sukadev@linux.vnet.ibm.com | e687883 | 2012-09-18 20:56:11 +0000 | [diff] [blame] | 181 | |
Michael Ellerman | 58a032c | 2013-05-15 20:19:31 +0000 | [diff] [blame] | 182 | if (ppmu->flags & PPMU_HAS_SIER) |
| 183 | sdar_valid = regs->dar & SIER_SDAR_VALID; |
| 184 | else { |
| 185 | unsigned long sdsync; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 186 | |
Michael Ellerman | 58a032c | 2013-05-15 20:19:31 +0000 | [diff] [blame] | 187 | if (ppmu->flags & PPMU_SIAR_VALID) |
| 188 | sdsync = POWER7P_MMCRA_SDAR_VALID; |
| 189 | else if (ppmu->flags & PPMU_ALT_SIPR) |
| 190 | sdsync = POWER6_MMCRA_SDSYNC; |
Madhavan Srinivasan | f04d108 | 2017-02-20 19:26:30 +0530 | [diff] [blame] | 191 | else if (ppmu->flags & PPMU_NO_SIAR) |
| 192 | sdsync = MMCRA_SAMPLE_ENABLE; |
Michael Ellerman | 58a032c | 2013-05-15 20:19:31 +0000 | [diff] [blame] | 193 | else |
| 194 | sdsync = MMCRA_SDSYNC; |
| 195 | |
| 196 | sdar_valid = mmcra & sdsync; |
| 197 | } |
| 198 | |
| 199 | if (!(mmcra & MMCRA_SAMPLE_ENABLE) || sdar_valid) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 200 | *addrp = mfspr(SPRN_SDAR); |
| 201 | } |
| 202 | |
Michael Ellerman | 5682c46 | 2013-04-25 19:28:24 +0000 | [diff] [blame] | 203 | static bool regs_sihv(struct pt_regs *regs) |
Anton Blanchard | 68b30bb | 2012-06-26 01:00:13 +0000 | [diff] [blame] | 204 | { |
| 205 | unsigned long sihv = MMCRA_SIHV; |
| 206 | |
Michael Ellerman | 8f61aa3 | 2013-04-25 19:28:27 +0000 | [diff] [blame] | 207 | if (ppmu->flags & PPMU_HAS_SIER) |
| 208 | return !!(regs->dar & SIER_SIHV); |
| 209 | |
Anton Blanchard | 68b30bb | 2012-06-26 01:00:13 +0000 | [diff] [blame] | 210 | if (ppmu->flags & PPMU_ALT_SIPR) |
| 211 | sihv = POWER6_MMCRA_SIHV; |
| 212 | |
Michael Ellerman | 5682c46 | 2013-04-25 19:28:24 +0000 | [diff] [blame] | 213 | return !!(regs->dsisr & sihv); |
Anton Blanchard | 68b30bb | 2012-06-26 01:00:13 +0000 | [diff] [blame] | 214 | } |
| 215 | |
Michael Ellerman | 5682c46 | 2013-04-25 19:28:24 +0000 | [diff] [blame] | 216 | static bool regs_sipr(struct pt_regs *regs) |
Anton Blanchard | 68b30bb | 2012-06-26 01:00:13 +0000 | [diff] [blame] | 217 | { |
| 218 | unsigned long sipr = MMCRA_SIPR; |
| 219 | |
Michael Ellerman | 8f61aa3 | 2013-04-25 19:28:27 +0000 | [diff] [blame] | 220 | if (ppmu->flags & PPMU_HAS_SIER) |
| 221 | return !!(regs->dar & SIER_SIPR); |
| 222 | |
Anton Blanchard | 68b30bb | 2012-06-26 01:00:13 +0000 | [diff] [blame] | 223 | if (ppmu->flags & PPMU_ALT_SIPR) |
| 224 | sipr = POWER6_MMCRA_SIPR; |
| 225 | |
Michael Ellerman | 5682c46 | 2013-04-25 19:28:24 +0000 | [diff] [blame] | 226 | return !!(regs->dsisr & sipr); |
Anton Blanchard | 68b30bb | 2012-06-26 01:00:13 +0000 | [diff] [blame] | 227 | } |
| 228 | |
Benjamin Herrenschmidt | 1ce447b | 2012-03-26 20:47:34 +0000 | [diff] [blame] | 229 | static inline u32 perf_flags_from_msr(struct pt_regs *regs) |
| 230 | { |
| 231 | if (regs->msr & MSR_PR) |
| 232 | return PERF_RECORD_MISC_USER; |
| 233 | if ((regs->msr & MSR_HV) && freeze_events_kernel != MMCR0_FCHV) |
| 234 | return PERF_RECORD_MISC_HYPERVISOR; |
| 235 | return PERF_RECORD_MISC_KERNEL; |
| 236 | } |
| 237 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 238 | static inline u32 perf_get_misc_flags(struct pt_regs *regs) |
| 239 | { |
Michael Ellerman | 3390405 | 2013-04-25 19:28:25 +0000 | [diff] [blame] | 240 | bool use_siar = regs_use_siar(regs); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 241 | |
Anton Blanchard | 75382aa | 2012-06-26 01:01:36 +0000 | [diff] [blame] | 242 | if (!use_siar) |
Benjamin Herrenschmidt | 1ce447b | 2012-03-26 20:47:34 +0000 | [diff] [blame] | 243 | return perf_flags_from_msr(regs); |
| 244 | |
| 245 | /* |
| 246 | * If we don't have flags in MMCRA, rather than using |
| 247 | * the MSR, we intuit the flags from the address in |
| 248 | * SIAR which should give slightly more reliable |
| 249 | * results |
| 250 | */ |
Michael Ellerman | cbda6aa | 2013-05-15 20:19:30 +0000 | [diff] [blame] | 251 | if (ppmu->flags & PPMU_NO_SIPR) { |
Benjamin Herrenschmidt | 1ce447b | 2012-03-26 20:47:34 +0000 | [diff] [blame] | 252 | unsigned long siar = mfspr(SPRN_SIAR); |
Madhavan Srinivasan | a2391b3 | 2016-12-24 11:35:49 +0530 | [diff] [blame] | 253 | if (is_kernel_addr(siar)) |
Benjamin Herrenschmidt | 1ce447b | 2012-03-26 20:47:34 +0000 | [diff] [blame] | 254 | return PERF_RECORD_MISC_KERNEL; |
| 255 | return PERF_RECORD_MISC_USER; |
| 256 | } |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 257 | |
Michael Neuling | 7abb840 | 2009-10-14 19:32:15 +0000 | [diff] [blame] | 258 | /* PR has priority over HV, so order below is important */ |
Michael Ellerman | 5682c46 | 2013-04-25 19:28:24 +0000 | [diff] [blame] | 259 | if (regs_sipr(regs)) |
Michael Neuling | 7abb840 | 2009-10-14 19:32:15 +0000 | [diff] [blame] | 260 | return PERF_RECORD_MISC_USER; |
Michael Ellerman | 5682c46 | 2013-04-25 19:28:24 +0000 | [diff] [blame] | 261 | |
| 262 | if (regs_sihv(regs) && (freeze_events_kernel != MMCR0_FCHV)) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 263 | return PERF_RECORD_MISC_HYPERVISOR; |
Michael Ellerman | 5682c46 | 2013-04-25 19:28:24 +0000 | [diff] [blame] | 264 | |
Michael Neuling | 7abb840 | 2009-10-14 19:32:15 +0000 | [diff] [blame] | 265 | return PERF_RECORD_MISC_KERNEL; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 266 | } |
| 267 | |
| 268 | /* |
| 269 | * Overload regs->dsisr to store MMCRA so we only need to read it once |
| 270 | * on each interrupt. |
Michael Ellerman | 8f61aa3 | 2013-04-25 19:28:27 +0000 | [diff] [blame] | 271 | * Overload regs->dar to store SIER if we have it. |
Anton Blanchard | 75382aa | 2012-06-26 01:01:36 +0000 | [diff] [blame] | 272 | * Overload regs->result to specify whether we should use the MSR (result |
| 273 | * is zero) or the SIAR (result is non zero). |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 274 | */ |
| 275 | static inline void perf_read_regs(struct pt_regs *regs) |
| 276 | { |
Anton Blanchard | 75382aa | 2012-06-26 01:01:36 +0000 | [diff] [blame] | 277 | unsigned long mmcra = mfspr(SPRN_MMCRA); |
| 278 | int marked = mmcra & MMCRA_SAMPLE_ENABLE; |
| 279 | int use_siar; |
| 280 | |
Michael Ellerman | 5682c46 | 2013-04-25 19:28:24 +0000 | [diff] [blame] | 281 | regs->dsisr = mmcra; |
Michael Ellerman | 860aad7 | 2013-04-25 19:28:26 +0000 | [diff] [blame] | 282 | |
Michael Ellerman | cbda6aa | 2013-05-15 20:19:30 +0000 | [diff] [blame] | 283 | if (ppmu->flags & PPMU_HAS_SIER) |
| 284 | regs->dar = mfspr(SPRN_SIER); |
Michael Ellerman | 8f61aa3 | 2013-04-25 19:28:27 +0000 | [diff] [blame] | 285 | |
| 286 | /* |
Anton Blanchard | 5c093ef | 2012-06-26 01:02:15 +0000 | [diff] [blame] | 287 | * If this isn't a PMU exception (eg a software event) the SIAR is |
| 288 | * not valid. Use pt_regs. |
| 289 | * |
| 290 | * If it is a marked event use the SIAR. |
| 291 | * |
| 292 | * If the PMU doesn't update the SIAR for non marked events use |
| 293 | * pt_regs. |
| 294 | * |
| 295 | * If the PMU has HV/PR flags then check to see if they |
| 296 | * place the exception in userspace. If so, use pt_regs. In |
| 297 | * continuous sampling mode the SIAR and the PMU exception are |
| 298 | * not synchronised, so they may be many instructions apart. |
| 299 | * This can result in confusing backtraces. We still want |
| 300 | * hypervisor samples as well as samples in the kernel with |
| 301 | * interrupts off hence the userspace check. |
| 302 | */ |
Anton Blanchard | 75382aa | 2012-06-26 01:01:36 +0000 | [diff] [blame] | 303 | if (TRAP(regs) != 0xf00) |
| 304 | use_siar = 0; |
Madhavan Srinivasan | 27593d7 | 2017-01-18 09:12:56 +0530 | [diff] [blame] | 305 | else if ((ppmu->flags & PPMU_NO_SIAR)) |
| 306 | use_siar = 0; |
Anton Blanchard | 5c093ef | 2012-06-26 01:02:15 +0000 | [diff] [blame] | 307 | else if (marked) |
| 308 | use_siar = 1; |
| 309 | else if ((ppmu->flags & PPMU_NO_CONT_SAMPLING)) |
| 310 | use_siar = 0; |
Michael Ellerman | cbda6aa | 2013-05-15 20:19:30 +0000 | [diff] [blame] | 311 | else if (!(ppmu->flags & PPMU_NO_SIPR) && regs_sipr(regs)) |
Anton Blanchard | 75382aa | 2012-06-26 01:01:36 +0000 | [diff] [blame] | 312 | use_siar = 0; |
| 313 | else |
| 314 | use_siar = 1; |
| 315 | |
Michael Ellerman | cbda6aa | 2013-05-15 20:19:30 +0000 | [diff] [blame] | 316 | regs->result = use_siar; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 317 | } |
| 318 | |
| 319 | /* |
| 320 | * If interrupts were soft-disabled when a PMU interrupt occurs, treat |
| 321 | * it as an NMI. |
| 322 | */ |
| 323 | static inline int perf_intr_is_nmi(struct pt_regs *regs) |
| 324 | { |
Madhavan Srinivasan | 01417c6 | 2017-12-20 09:25:49 +0530 | [diff] [blame] | 325 | return (regs->softe & IRQS_DISABLED); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 326 | } |
| 327 | |
sukadev@linux.vnet.ibm.com | e687883 | 2012-09-18 20:56:11 +0000 | [diff] [blame] | 328 | /* |
| 329 | * On processors like P7+ that have the SIAR-Valid bit, marked instructions |
| 330 | * must be sampled only if the SIAR-valid bit is set. |
| 331 | * |
| 332 | * For unmarked instructions and for processors that don't have the SIAR-Valid |
| 333 | * bit, assume that SIAR is valid. |
| 334 | */ |
| 335 | static inline int siar_valid(struct pt_regs *regs) |
| 336 | { |
| 337 | unsigned long mmcra = regs->dsisr; |
| 338 | int marked = mmcra & MMCRA_SAMPLE_ENABLE; |
| 339 | |
Michael Ellerman | 58a032c | 2013-05-15 20:19:31 +0000 | [diff] [blame] | 340 | if (marked) { |
| 341 | if (ppmu->flags & PPMU_HAS_SIER) |
| 342 | return regs->dar & SIER_SIAR_VALID; |
| 343 | |
| 344 | if (ppmu->flags & PPMU_SIAR_VALID) |
| 345 | return mmcra & POWER7P_MMCRA_SIAR_VALID; |
| 346 | } |
sukadev@linux.vnet.ibm.com | e687883 | 2012-09-18 20:56:11 +0000 | [diff] [blame] | 347 | |
| 348 | return 1; |
| 349 | } |
| 350 | |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 351 | |
| 352 | /* Reset all possible BHRB entries */ |
| 353 | static void power_pmu_bhrb_reset(void) |
| 354 | { |
| 355 | asm volatile(PPC_CLRBHRB); |
| 356 | } |
| 357 | |
| 358 | static void power_pmu_bhrb_enable(struct perf_event *event) |
| 359 | { |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 360 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 361 | |
| 362 | if (!ppmu->bhrb_nr) |
| 363 | return; |
| 364 | |
| 365 | /* Clear BHRB if we changed task context to avoid data leaks */ |
| 366 | if (event->ctx->task && cpuhw->bhrb_context != event->ctx) { |
| 367 | power_pmu_bhrb_reset(); |
| 368 | cpuhw->bhrb_context = event->ctx; |
| 369 | } |
| 370 | cpuhw->bhrb_users++; |
Peter Zijlstra | acba3c7 | 2015-01-14 14:15:39 +0100 | [diff] [blame] | 371 | perf_sched_cb_inc(event->ctx->pmu); |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 372 | } |
| 373 | |
| 374 | static void power_pmu_bhrb_disable(struct perf_event *event) |
| 375 | { |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 376 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 377 | |
| 378 | if (!ppmu->bhrb_nr) |
| 379 | return; |
| 380 | |
Anshuman Khandual | f0322f7 | 2015-06-30 13:50:28 +0530 | [diff] [blame] | 381 | WARN_ON_ONCE(!cpuhw->bhrb_users); |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 382 | cpuhw->bhrb_users--; |
Peter Zijlstra | acba3c7 | 2015-01-14 14:15:39 +0100 | [diff] [blame] | 383 | perf_sched_cb_dec(event->ctx->pmu); |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 384 | |
| 385 | if (!cpuhw->disabled && !cpuhw->bhrb_users) { |
| 386 | /* BHRB cannot be turned off when other |
| 387 | * events are active on the PMU. |
| 388 | */ |
| 389 | |
| 390 | /* avoid stale pointer */ |
| 391 | cpuhw->bhrb_context = NULL; |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | /* Called from ctxsw to prevent one process's branch entries to |
| 396 | * mingle with the other process's entries during context switch. |
| 397 | */ |
Peter Zijlstra | acba3c7 | 2015-01-14 14:15:39 +0100 | [diff] [blame] | 398 | static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 399 | { |
Peter Zijlstra | acba3c7 | 2015-01-14 14:15:39 +0100 | [diff] [blame] | 400 | if (!ppmu->bhrb_nr) |
| 401 | return; |
| 402 | |
| 403 | if (sched_in) |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 404 | power_pmu_bhrb_reset(); |
| 405 | } |
Michael Neuling | 6912318 | 2013-05-13 18:44:58 +0000 | [diff] [blame] | 406 | /* Calculate the to address for a branch */ |
| 407 | static __u64 power_pmu_bhrb_to(u64 addr) |
| 408 | { |
| 409 | unsigned int instr; |
| 410 | int ret; |
| 411 | __u64 target; |
| 412 | |
Ravi Bangoria | f41d84dd | 2017-12-12 17:59:15 +0530 | [diff] [blame] | 413 | if (is_kernel_addr(addr)) { |
| 414 | if (probe_kernel_read(&instr, (void *)addr, sizeof(instr))) |
| 415 | return 0; |
| 416 | |
| 417 | return branch_target(&instr); |
| 418 | } |
Michael Neuling | 6912318 | 2013-05-13 18:44:58 +0000 | [diff] [blame] | 419 | |
| 420 | /* Userspace: need copy instruction here then translate it */ |
| 421 | pagefault_disable(); |
| 422 | ret = __get_user_inatomic(instr, (unsigned int __user *)addr); |
| 423 | if (ret) { |
| 424 | pagefault_enable(); |
| 425 | return 0; |
| 426 | } |
| 427 | pagefault_enable(); |
| 428 | |
| 429 | target = branch_target(&instr); |
| 430 | if ((!target) || (instr & BRANCH_ABSOLUTE)) |
| 431 | return target; |
| 432 | |
| 433 | /* Translate relative branch target from kernel to user address */ |
| 434 | return target - (unsigned long)&instr + addr; |
| 435 | } |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 436 | |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 437 | /* Processing BHRB entries */ |
Anton Blanchard | e51df2c | 2014-08-20 08:55:18 +1000 | [diff] [blame] | 438 | static void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 439 | { |
| 440 | u64 val; |
| 441 | u64 addr; |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 442 | int r_index, u_index, pred; |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 443 | |
| 444 | r_index = 0; |
| 445 | u_index = 0; |
| 446 | while (r_index < ppmu->bhrb_nr) { |
| 447 | /* Assembly read function */ |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 448 | val = read_bhrb(r_index++); |
| 449 | if (!val) |
| 450 | /* Terminal marker: End of valid BHRB entries */ |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 451 | break; |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 452 | else { |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 453 | addr = val & BHRB_EA; |
| 454 | pred = val & BHRB_PREDICTION; |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 455 | |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 456 | if (!addr) |
| 457 | /* invalid entry */ |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 458 | continue; |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 459 | |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 460 | /* Branches are read most recent first (ie. mfbhrb 0 is |
| 461 | * the most recent branch). |
| 462 | * There are two types of valid entries: |
| 463 | * 1) a target entry which is the to address of a |
| 464 | * computed goto like a blr,bctr,btar. The next |
| 465 | * entry read from the bhrb will be branch |
| 466 | * corresponding to this target (ie. the actual |
| 467 | * blr/bctr/btar instruction). |
| 468 | * 2) a from address which is an actual branch. If a |
| 469 | * target entry proceeds this, then this is the |
| 470 | * matching branch for that target. If this is not |
| 471 | * following a target entry, then this is a branch |
| 472 | * where the target is given as an immediate field |
| 473 | * in the instruction (ie. an i or b form branch). |
| 474 | * In this case we need to read the instruction from |
| 475 | * memory to determine the target/to address. |
| 476 | */ |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 477 | |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 478 | if (val & BHRB_TARGET) { |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 479 | /* Target branches use two entries |
| 480 | * (ie. computed gotos/XL form) |
| 481 | */ |
| 482 | cpuhw->bhrb_entries[u_index].to = addr; |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 483 | cpuhw->bhrb_entries[u_index].mispred = pred; |
| 484 | cpuhw->bhrb_entries[u_index].predicted = ~pred; |
| 485 | |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 486 | /* Get from address in next entry */ |
| 487 | val = read_bhrb(r_index++); |
| 488 | addr = val & BHRB_EA; |
| 489 | if (val & BHRB_TARGET) { |
| 490 | /* Shouldn't have two targets in a |
| 491 | row.. Reset index and try again */ |
| 492 | r_index--; |
| 493 | addr = 0; |
| 494 | } |
| 495 | cpuhw->bhrb_entries[u_index].from = addr; |
| 496 | } else { |
| 497 | /* Branches to immediate field |
| 498 | (ie I or B form) */ |
| 499 | cpuhw->bhrb_entries[u_index].from = addr; |
Michael Neuling | 6912318 | 2013-05-13 18:44:58 +0000 | [diff] [blame] | 500 | cpuhw->bhrb_entries[u_index].to = |
| 501 | power_pmu_bhrb_to(addr); |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 502 | cpuhw->bhrb_entries[u_index].mispred = pred; |
| 503 | cpuhw->bhrb_entries[u_index].predicted = ~pred; |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 504 | } |
Michael Neuling | 506e70d | 2013-05-13 18:44:57 +0000 | [diff] [blame] | 505 | u_index++; |
| 506 | |
Michael Neuling | d52f2dc | 2013-05-13 18:44:56 +0000 | [diff] [blame] | 507 | } |
| 508 | } |
| 509 | cpuhw->bhrb_stack.nr = u_index; |
| 510 | return; |
| 511 | } |
| 512 | |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 513 | static bool is_ebb_event(struct perf_event *event) |
| 514 | { |
| 515 | /* |
| 516 | * This could be a per-PMU callback, but we'd rather avoid the cost. We |
| 517 | * check that the PMU supports EBB, meaning those that don't can still |
| 518 | * use bit 63 of the event code for something else if they wish. |
| 519 | */ |
Joel Stanley | 4d9690d | 2014-07-08 16:08:21 +0930 | [diff] [blame] | 520 | return (ppmu->flags & PPMU_ARCH_207S) && |
Michael Ellerman | 8d7c55d | 2013-07-23 18:07:45 +1000 | [diff] [blame] | 521 | ((event->attr.config >> PERF_EVENT_CONFIG_EBB_SHIFT) & 1); |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 522 | } |
| 523 | |
| 524 | static int ebb_event_check(struct perf_event *event) |
| 525 | { |
| 526 | struct perf_event *leader = event->group_leader; |
| 527 | |
| 528 | /* Event and group leader must agree on EBB */ |
| 529 | if (is_ebb_event(leader) != is_ebb_event(event)) |
| 530 | return -EINVAL; |
| 531 | |
| 532 | if (is_ebb_event(event)) { |
| 533 | if (!(event->attach_state & PERF_ATTACH_TASK)) |
| 534 | return -EINVAL; |
| 535 | |
| 536 | if (!leader->attr.pinned || !leader->attr.exclusive) |
| 537 | return -EINVAL; |
| 538 | |
Michael Ellerman | 58b5fb0 | 2014-03-14 16:00:30 +1100 | [diff] [blame] | 539 | if (event->attr.freq || |
| 540 | event->attr.inherit || |
| 541 | event->attr.sample_type || |
| 542 | event->attr.sample_period || |
| 543 | event->attr.enable_on_exec) |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 544 | return -EINVAL; |
| 545 | } |
| 546 | |
| 547 | return 0; |
| 548 | } |
| 549 | |
| 550 | static void ebb_event_add(struct perf_event *event) |
| 551 | { |
| 552 | if (!is_ebb_event(event) || current->thread.used_ebb) |
| 553 | return; |
| 554 | |
| 555 | /* |
| 556 | * IFF this is the first time we've added an EBB event, set |
| 557 | * PMXE in the user MMCR0 so we can detect when it's cleared by |
| 558 | * userspace. We need this so that we can context switch while |
| 559 | * userspace is in the EBB handler (where PMXE is 0). |
| 560 | */ |
| 561 | current->thread.used_ebb = 1; |
| 562 | current->thread.mmcr0 |= MMCR0_PMXE; |
| 563 | } |
| 564 | |
| 565 | static void ebb_switch_out(unsigned long mmcr0) |
| 566 | { |
| 567 | if (!(mmcr0 & MMCR0_EBE)) |
| 568 | return; |
| 569 | |
| 570 | current->thread.siar = mfspr(SPRN_SIAR); |
| 571 | current->thread.sier = mfspr(SPRN_SIER); |
| 572 | current->thread.sdar = mfspr(SPRN_SDAR); |
| 573 | current->thread.mmcr0 = mmcr0 & MMCR0_USER_MASK; |
| 574 | current->thread.mmcr2 = mfspr(SPRN_MMCR2) & MMCR2_USER_MASK; |
| 575 | } |
| 576 | |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 577 | static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw) |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 578 | { |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 579 | unsigned long mmcr0 = cpuhw->mmcr[0]; |
| 580 | |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 581 | if (!ebb) |
| 582 | goto out; |
| 583 | |
Michael Ellerman | 76cb8a7 | 2014-03-14 16:00:34 +1100 | [diff] [blame] | 584 | /* Enable EBB and read/write to all 6 PMCs and BHRB for userspace */ |
| 585 | mmcr0 |= MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC_U6; |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 586 | |
Michael Ellerman | c2e37a2 | 2014-03-14 16:00:29 +1100 | [diff] [blame] | 587 | /* |
| 588 | * Add any bits from the user MMCR0, FC or PMAO. This is compatible |
| 589 | * with pmao_restore_workaround() because we may add PMAO but we never |
| 590 | * clear it here. |
| 591 | */ |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 592 | mmcr0 |= current->thread.mmcr0; |
| 593 | |
Michael Ellerman | c2e37a2 | 2014-03-14 16:00:29 +1100 | [diff] [blame] | 594 | /* |
| 595 | * Be careful not to set PMXE if userspace had it cleared. This is also |
| 596 | * compatible with pmao_restore_workaround() because it has already |
| 597 | * cleared PMXE and we leave PMAO alone. |
| 598 | */ |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 599 | if (!(current->thread.mmcr0 & MMCR0_PMXE)) |
| 600 | mmcr0 &= ~MMCR0_PMXE; |
| 601 | |
| 602 | mtspr(SPRN_SIAR, current->thread.siar); |
| 603 | mtspr(SPRN_SIER, current->thread.sier); |
| 604 | mtspr(SPRN_SDAR, current->thread.sdar); |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 605 | |
| 606 | /* |
| 607 | * Merge the kernel & user values of MMCR2. The semantics we implement |
| 608 | * are that the user MMCR2 can set bits, ie. cause counters to freeze, |
| 609 | * but not clear bits. If a task wants to be able to clear bits, ie. |
| 610 | * unfreeze counters, it should not set exclude_xxx in its events and |
| 611 | * instead manage the MMCR2 entirely by itself. |
| 612 | */ |
| 613 | mtspr(SPRN_MMCR2, cpuhw->mmcr[3] | current->thread.mmcr2); |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 614 | out: |
| 615 | return mmcr0; |
| 616 | } |
Michael Ellerman | c2e37a2 | 2014-03-14 16:00:29 +1100 | [diff] [blame] | 617 | |
| 618 | static void pmao_restore_workaround(bool ebb) |
| 619 | { |
| 620 | unsigned pmcs[6]; |
| 621 | |
| 622 | if (!cpu_has_feature(CPU_FTR_PMAO_BUG)) |
| 623 | return; |
| 624 | |
| 625 | /* |
| 626 | * On POWER8E there is a hardware defect which affects the PMU context |
| 627 | * switch logic, ie. power_pmu_disable/enable(). |
| 628 | * |
| 629 | * When a counter overflows PMXE is cleared and FC/PMAO is set in MMCR0 |
| 630 | * by the hardware. Sometime later the actual PMU exception is |
| 631 | * delivered. |
| 632 | * |
| 633 | * If we context switch, or simply disable/enable, the PMU prior to the |
| 634 | * exception arriving, the exception will be lost when we clear PMAO. |
| 635 | * |
| 636 | * When we reenable the PMU, we will write the saved MMCR0 with PMAO |
| 637 | * set, and this _should_ generate an exception. However because of the |
| 638 | * defect no exception is generated when we write PMAO, and we get |
| 639 | * stuck with no counters counting but no exception delivered. |
| 640 | * |
| 641 | * The workaround is to detect this case and tweak the hardware to |
| 642 | * create another pending PMU exception. |
| 643 | * |
| 644 | * We do that by setting up PMC6 (cycles) for an imminent overflow and |
| 645 | * enabling the PMU. That causes a new exception to be generated in the |
| 646 | * chip, but we don't take it yet because we have interrupts hard |
| 647 | * disabled. We then write back the PMU state as we want it to be seen |
| 648 | * by the exception handler. When we reenable interrupts the exception |
| 649 | * handler will be called and see the correct state. |
| 650 | * |
| 651 | * The logic is the same for EBB, except that the exception is gated by |
| 652 | * us having interrupts hard disabled as well as the fact that we are |
| 653 | * not in userspace. The exception is finally delivered when we return |
| 654 | * to userspace. |
| 655 | */ |
| 656 | |
| 657 | /* Only if PMAO is set and PMAO_SYNC is clear */ |
| 658 | if ((current->thread.mmcr0 & (MMCR0_PMAO | MMCR0_PMAO_SYNC)) != MMCR0_PMAO) |
| 659 | return; |
| 660 | |
| 661 | /* If we're doing EBB, only if BESCR[GE] is set */ |
| 662 | if (ebb && !(current->thread.bescr & BESCR_GE)) |
| 663 | return; |
| 664 | |
| 665 | /* |
| 666 | * We are already soft-disabled in power_pmu_enable(). We need to hard |
Madhavan Srinivasan | 58bffb5 | 2016-03-04 10:31:49 +0530 | [diff] [blame] | 667 | * disable to actually prevent the PMU exception from firing. |
Michael Ellerman | c2e37a2 | 2014-03-14 16:00:29 +1100 | [diff] [blame] | 668 | */ |
| 669 | hard_irq_disable(); |
| 670 | |
| 671 | /* |
| 672 | * This is a bit gross, but we know we're on POWER8E and have 6 PMCs. |
| 673 | * Using read/write_pmc() in a for loop adds 12 function calls and |
| 674 | * almost doubles our code size. |
| 675 | */ |
| 676 | pmcs[0] = mfspr(SPRN_PMC1); |
| 677 | pmcs[1] = mfspr(SPRN_PMC2); |
| 678 | pmcs[2] = mfspr(SPRN_PMC3); |
| 679 | pmcs[3] = mfspr(SPRN_PMC4); |
| 680 | pmcs[4] = mfspr(SPRN_PMC5); |
| 681 | pmcs[5] = mfspr(SPRN_PMC6); |
| 682 | |
| 683 | /* Ensure all freeze bits are unset */ |
| 684 | mtspr(SPRN_MMCR2, 0); |
| 685 | |
| 686 | /* Set up PMC6 to overflow in one cycle */ |
| 687 | mtspr(SPRN_PMC6, 0x7FFFFFFE); |
| 688 | |
| 689 | /* Enable exceptions and unfreeze PMC6 */ |
| 690 | mtspr(SPRN_MMCR0, MMCR0_PMXE | MMCR0_PMCjCE | MMCR0_PMAO); |
| 691 | |
| 692 | /* Now we need to refreeze and restore the PMCs */ |
| 693 | mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMAO); |
| 694 | |
| 695 | mtspr(SPRN_PMC1, pmcs[0]); |
| 696 | mtspr(SPRN_PMC2, pmcs[1]); |
| 697 | mtspr(SPRN_PMC3, pmcs[2]); |
| 698 | mtspr(SPRN_PMC4, pmcs[3]); |
| 699 | mtspr(SPRN_PMC5, pmcs[4]); |
| 700 | mtspr(SPRN_PMC6, pmcs[5]); |
| 701 | } |
Madhavan Srinivasan | 356d8ce | 2017-02-12 22:33:14 +0530 | [diff] [blame] | 702 | |
| 703 | static bool use_ic(u64 event) |
| 704 | { |
| 705 | if (cpu_has_feature(CPU_FTR_POWER9_DD1) && |
| 706 | (event == 0x200f2 || event == 0x300f2)) |
| 707 | return true; |
| 708 | |
| 709 | return false; |
| 710 | } |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 711 | #endif /* CONFIG_PPC64 */ |
| 712 | |
| 713 | static void perf_event_interrupt(struct pt_regs *regs); |
| 714 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 715 | /* |
Ingo Molnar | 57c0c15 | 2009-09-21 12:20:38 +0200 | [diff] [blame] | 716 | * Read one performance monitor counter (PMC). |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 717 | */ |
| 718 | static unsigned long read_pmc(int idx) |
| 719 | { |
| 720 | unsigned long val; |
| 721 | |
| 722 | switch (idx) { |
| 723 | case 1: |
| 724 | val = mfspr(SPRN_PMC1); |
| 725 | break; |
| 726 | case 2: |
| 727 | val = mfspr(SPRN_PMC2); |
| 728 | break; |
| 729 | case 3: |
| 730 | val = mfspr(SPRN_PMC3); |
| 731 | break; |
| 732 | case 4: |
| 733 | val = mfspr(SPRN_PMC4); |
| 734 | break; |
| 735 | case 5: |
| 736 | val = mfspr(SPRN_PMC5); |
| 737 | break; |
| 738 | case 6: |
| 739 | val = mfspr(SPRN_PMC6); |
| 740 | break; |
| 741 | #ifdef CONFIG_PPC64 |
| 742 | case 7: |
| 743 | val = mfspr(SPRN_PMC7); |
| 744 | break; |
| 745 | case 8: |
| 746 | val = mfspr(SPRN_PMC8); |
| 747 | break; |
| 748 | #endif /* CONFIG_PPC64 */ |
| 749 | default: |
| 750 | printk(KERN_ERR "oops trying to read PMC%d\n", idx); |
| 751 | val = 0; |
| 752 | } |
| 753 | return val; |
| 754 | } |
| 755 | |
| 756 | /* |
| 757 | * Write one PMC. |
| 758 | */ |
| 759 | static void write_pmc(int idx, unsigned long val) |
| 760 | { |
| 761 | switch (idx) { |
| 762 | case 1: |
| 763 | mtspr(SPRN_PMC1, val); |
| 764 | break; |
| 765 | case 2: |
| 766 | mtspr(SPRN_PMC2, val); |
| 767 | break; |
| 768 | case 3: |
| 769 | mtspr(SPRN_PMC3, val); |
| 770 | break; |
| 771 | case 4: |
| 772 | mtspr(SPRN_PMC4, val); |
| 773 | break; |
| 774 | case 5: |
| 775 | mtspr(SPRN_PMC5, val); |
| 776 | break; |
| 777 | case 6: |
| 778 | mtspr(SPRN_PMC6, val); |
| 779 | break; |
| 780 | #ifdef CONFIG_PPC64 |
| 781 | case 7: |
| 782 | mtspr(SPRN_PMC7, val); |
| 783 | break; |
| 784 | case 8: |
| 785 | mtspr(SPRN_PMC8, val); |
| 786 | break; |
| 787 | #endif /* CONFIG_PPC64 */ |
| 788 | default: |
| 789 | printk(KERN_ERR "oops trying to write PMC%d\n", idx); |
| 790 | } |
| 791 | } |
| 792 | |
Anshuman Khandual | 5f6d038 | 2014-03-14 16:00:27 +1100 | [diff] [blame] | 793 | /* Called from sysrq_handle_showregs() */ |
| 794 | void perf_event_print_debug(void) |
| 795 | { |
| 796 | unsigned long sdar, sier, flags; |
| 797 | u32 pmcs[MAX_HWEVENTS]; |
| 798 | int i; |
| 799 | |
Ravi Bangoria | 4917fcb | 2017-09-19 11:47:06 +0530 | [diff] [blame] | 800 | if (!ppmu) { |
| 801 | pr_info("Performance monitor hardware not registered.\n"); |
| 802 | return; |
| 803 | } |
| 804 | |
Anshuman Khandual | 5f6d038 | 2014-03-14 16:00:27 +1100 | [diff] [blame] | 805 | if (!ppmu->n_counter) |
| 806 | return; |
| 807 | |
| 808 | local_irq_save(flags); |
| 809 | |
| 810 | pr_info("CPU: %d PMU registers, ppmu = %s n_counters = %d", |
| 811 | smp_processor_id(), ppmu->name, ppmu->n_counter); |
| 812 | |
| 813 | for (i = 0; i < ppmu->n_counter; i++) |
| 814 | pmcs[i] = read_pmc(i + 1); |
| 815 | |
| 816 | for (; i < MAX_HWEVENTS; i++) |
| 817 | pmcs[i] = 0xdeadbeef; |
| 818 | |
| 819 | pr_info("PMC1: %08x PMC2: %08x PMC3: %08x PMC4: %08x\n", |
| 820 | pmcs[0], pmcs[1], pmcs[2], pmcs[3]); |
| 821 | |
| 822 | if (ppmu->n_counter > 4) |
| 823 | pr_info("PMC5: %08x PMC6: %08x PMC7: %08x PMC8: %08x\n", |
| 824 | pmcs[4], pmcs[5], pmcs[6], pmcs[7]); |
| 825 | |
| 826 | pr_info("MMCR0: %016lx MMCR1: %016lx MMCRA: %016lx\n", |
| 827 | mfspr(SPRN_MMCR0), mfspr(SPRN_MMCR1), mfspr(SPRN_MMCRA)); |
| 828 | |
| 829 | sdar = sier = 0; |
| 830 | #ifdef CONFIG_PPC64 |
| 831 | sdar = mfspr(SPRN_SDAR); |
| 832 | |
| 833 | if (ppmu->flags & PPMU_HAS_SIER) |
| 834 | sier = mfspr(SPRN_SIER); |
| 835 | |
Joel Stanley | 4d9690d | 2014-07-08 16:08:21 +0930 | [diff] [blame] | 836 | if (ppmu->flags & PPMU_ARCH_207S) { |
Anshuman Khandual | 5f6d038 | 2014-03-14 16:00:27 +1100 | [diff] [blame] | 837 | pr_info("MMCR2: %016lx EBBHR: %016lx\n", |
| 838 | mfspr(SPRN_MMCR2), mfspr(SPRN_EBBHR)); |
| 839 | pr_info("EBBRR: %016lx BESCR: %016lx\n", |
| 840 | mfspr(SPRN_EBBRR), mfspr(SPRN_BESCR)); |
| 841 | } |
| 842 | #endif |
| 843 | pr_info("SIAR: %016lx SDAR: %016lx SIER: %016lx\n", |
| 844 | mfspr(SPRN_SIAR), sdar, sier); |
| 845 | |
| 846 | local_irq_restore(flags); |
| 847 | } |
| 848 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 849 | /* |
| 850 | * Check if a set of events can all go on the PMU at once. |
| 851 | * If they can't, this will look at alternative codes for the events |
| 852 | * and see if any combination of alternative codes is feasible. |
| 853 | * The feasible set is returned in event_id[]. |
| 854 | */ |
| 855 | static int power_check_constraints(struct cpu_hw_events *cpuhw, |
| 856 | u64 event_id[], unsigned int cflags[], |
| 857 | int n_ev) |
| 858 | { |
| 859 | unsigned long mask, value, nv; |
| 860 | unsigned long smasks[MAX_HWEVENTS], svalues[MAX_HWEVENTS]; |
| 861 | int n_alt[MAX_HWEVENTS], choice[MAX_HWEVENTS]; |
| 862 | int i, j; |
| 863 | unsigned long addf = ppmu->add_fields; |
| 864 | unsigned long tadd = ppmu->test_adder; |
| 865 | |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 866 | if (n_ev > ppmu->n_counter) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 867 | return -1; |
| 868 | |
| 869 | /* First see if the events will go on as-is */ |
| 870 | for (i = 0; i < n_ev; ++i) { |
| 871 | if ((cflags[i] & PPMU_LIMITED_PMC_REQD) |
| 872 | && !ppmu->limited_pmc_event(event_id[i])) { |
| 873 | ppmu->get_alternatives(event_id[i], cflags[i], |
| 874 | cpuhw->alternatives[i]); |
| 875 | event_id[i] = cpuhw->alternatives[i][0]; |
| 876 | } |
| 877 | if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0], |
| 878 | &cpuhw->avalues[i][0])) |
| 879 | return -1; |
| 880 | } |
| 881 | value = mask = 0; |
| 882 | for (i = 0; i < n_ev; ++i) { |
| 883 | nv = (value | cpuhw->avalues[i][0]) + |
| 884 | (value & cpuhw->avalues[i][0] & addf); |
| 885 | if ((((nv + tadd) ^ value) & mask) != 0 || |
| 886 | (((nv + tadd) ^ cpuhw->avalues[i][0]) & |
| 887 | cpuhw->amasks[i][0]) != 0) |
| 888 | break; |
| 889 | value = nv; |
| 890 | mask |= cpuhw->amasks[i][0]; |
| 891 | } |
| 892 | if (i == n_ev) |
| 893 | return 0; /* all OK */ |
| 894 | |
| 895 | /* doesn't work, gather alternatives... */ |
| 896 | if (!ppmu->get_alternatives) |
| 897 | return -1; |
| 898 | for (i = 0; i < n_ev; ++i) { |
| 899 | choice[i] = 0; |
| 900 | n_alt[i] = ppmu->get_alternatives(event_id[i], cflags[i], |
| 901 | cpuhw->alternatives[i]); |
| 902 | for (j = 1; j < n_alt[i]; ++j) |
| 903 | ppmu->get_constraint(cpuhw->alternatives[i][j], |
| 904 | &cpuhw->amasks[i][j], |
| 905 | &cpuhw->avalues[i][j]); |
| 906 | } |
| 907 | |
| 908 | /* enumerate all possibilities and see if any will work */ |
| 909 | i = 0; |
| 910 | j = -1; |
| 911 | value = mask = nv = 0; |
| 912 | while (i < n_ev) { |
| 913 | if (j >= 0) { |
| 914 | /* we're backtracking, restore context */ |
| 915 | value = svalues[i]; |
| 916 | mask = smasks[i]; |
| 917 | j = choice[i]; |
| 918 | } |
| 919 | /* |
| 920 | * See if any alternative k for event_id i, |
| 921 | * where k > j, will satisfy the constraints. |
| 922 | */ |
| 923 | while (++j < n_alt[i]) { |
| 924 | nv = (value | cpuhw->avalues[i][j]) + |
| 925 | (value & cpuhw->avalues[i][j] & addf); |
| 926 | if ((((nv + tadd) ^ value) & mask) == 0 && |
| 927 | (((nv + tadd) ^ cpuhw->avalues[i][j]) |
| 928 | & cpuhw->amasks[i][j]) == 0) |
| 929 | break; |
| 930 | } |
| 931 | if (j >= n_alt[i]) { |
| 932 | /* |
| 933 | * No feasible alternative, backtrack |
| 934 | * to event_id i-1 and continue enumerating its |
| 935 | * alternatives from where we got up to. |
| 936 | */ |
| 937 | if (--i < 0) |
| 938 | return -1; |
| 939 | } else { |
| 940 | /* |
| 941 | * Found a feasible alternative for event_id i, |
| 942 | * remember where we got up to with this event_id, |
| 943 | * go on to the next event_id, and start with |
| 944 | * the first alternative for it. |
| 945 | */ |
| 946 | choice[i] = j; |
| 947 | svalues[i] = value; |
| 948 | smasks[i] = mask; |
| 949 | value = nv; |
| 950 | mask |= cpuhw->amasks[i][j]; |
| 951 | ++i; |
| 952 | j = -1; |
| 953 | } |
| 954 | } |
| 955 | |
| 956 | /* OK, we have a feasible combination, tell the caller the solution */ |
| 957 | for (i = 0; i < n_ev; ++i) |
| 958 | event_id[i] = cpuhw->alternatives[i][choice[i]]; |
| 959 | return 0; |
| 960 | } |
| 961 | |
| 962 | /* |
| 963 | * Check if newly-added events have consistent settings for |
| 964 | * exclude_{user,kernel,hv} with each other and any previously |
| 965 | * added events. |
| 966 | */ |
| 967 | static int check_excludes(struct perf_event **ctrs, unsigned int cflags[], |
| 968 | int n_prev, int n_new) |
| 969 | { |
| 970 | int eu = 0, ek = 0, eh = 0; |
| 971 | int i, n, first; |
| 972 | struct perf_event *event; |
| 973 | |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 974 | /* |
| 975 | * If the PMU we're on supports per event exclude settings then we |
| 976 | * don't need to do any of this logic. NB. This assumes no PMU has both |
| 977 | * per event exclude and limited PMCs. |
| 978 | */ |
| 979 | if (ppmu->flags & PPMU_ARCH_207S) |
| 980 | return 0; |
| 981 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 982 | n = n_prev + n_new; |
| 983 | if (n <= 1) |
| 984 | return 0; |
| 985 | |
| 986 | first = 1; |
| 987 | for (i = 0; i < n; ++i) { |
| 988 | if (cflags[i] & PPMU_LIMITED_PMC_OK) { |
| 989 | cflags[i] &= ~PPMU_LIMITED_PMC_REQD; |
| 990 | continue; |
| 991 | } |
| 992 | event = ctrs[i]; |
| 993 | if (first) { |
| 994 | eu = event->attr.exclude_user; |
| 995 | ek = event->attr.exclude_kernel; |
| 996 | eh = event->attr.exclude_hv; |
| 997 | first = 0; |
| 998 | } else if (event->attr.exclude_user != eu || |
| 999 | event->attr.exclude_kernel != ek || |
| 1000 | event->attr.exclude_hv != eh) { |
| 1001 | return -EAGAIN; |
| 1002 | } |
| 1003 | } |
| 1004 | |
| 1005 | if (eu || ek || eh) |
| 1006 | for (i = 0; i < n; ++i) |
| 1007 | if (cflags[i] & PPMU_LIMITED_PMC_OK) |
| 1008 | cflags[i] |= PPMU_LIMITED_PMC_REQD; |
| 1009 | |
| 1010 | return 0; |
| 1011 | } |
| 1012 | |
Eric B Munson | 86c74ab | 2011-04-15 08:12:30 +0000 | [diff] [blame] | 1013 | static u64 check_and_compute_delta(u64 prev, u64 val) |
| 1014 | { |
| 1015 | u64 delta = (val - prev) & 0xfffffffful; |
| 1016 | |
| 1017 | /* |
| 1018 | * POWER7 can roll back counter values, if the new value is smaller |
| 1019 | * than the previous value it will cause the delta and the counter to |
| 1020 | * have bogus values unless we rolled a counter over. If a coutner is |
| 1021 | * rolled back, it will be smaller, but within 256, which is the maximum |
Michael Ellerman | 027dfac | 2016-06-01 16:34:37 +1000 | [diff] [blame] | 1022 | * number of events to rollback at once. If we detect a rollback |
Eric B Munson | 86c74ab | 2011-04-15 08:12:30 +0000 | [diff] [blame] | 1023 | * return 0. This can lead to a small lack of precision in the |
| 1024 | * counters. |
| 1025 | */ |
| 1026 | if (prev > val && (prev - val) < 256) |
| 1027 | delta = 0; |
| 1028 | |
| 1029 | return delta; |
| 1030 | } |
| 1031 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1032 | static void power_pmu_read(struct perf_event *event) |
| 1033 | { |
| 1034 | s64 val, delta, prev; |
Madhavan Srinivasan | 356d8ce | 2017-02-12 22:33:14 +0530 | [diff] [blame] | 1035 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1036 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1037 | if (event->hw.state & PERF_HES_STOPPED) |
| 1038 | return; |
| 1039 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1040 | if (!event->hw.idx) |
| 1041 | return; |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1042 | |
| 1043 | if (is_ebb_event(event)) { |
| 1044 | val = read_pmc(event->hw.idx); |
Madhavan Srinivasan | 356d8ce | 2017-02-12 22:33:14 +0530 | [diff] [blame] | 1045 | if (use_ic(event->attr.config)) { |
| 1046 | val = mfspr(SPRN_IC); |
| 1047 | if (val > cpuhw->ic_init) |
| 1048 | val = val - cpuhw->ic_init; |
| 1049 | else |
| 1050 | val = val + (0 - cpuhw->ic_init); |
| 1051 | } |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1052 | local64_set(&event->hw.prev_count, val); |
| 1053 | return; |
| 1054 | } |
| 1055 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1056 | /* |
| 1057 | * Performance monitor interrupts come even when interrupts |
| 1058 | * are soft-disabled, as long as interrupts are hard-enabled. |
| 1059 | * Therefore we treat them like NMIs. |
| 1060 | */ |
| 1061 | do { |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 1062 | prev = local64_read(&event->hw.prev_count); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1063 | barrier(); |
| 1064 | val = read_pmc(event->hw.idx); |
Madhavan Srinivasan | 356d8ce | 2017-02-12 22:33:14 +0530 | [diff] [blame] | 1065 | if (use_ic(event->attr.config)) { |
| 1066 | val = mfspr(SPRN_IC); |
| 1067 | if (val > cpuhw->ic_init) |
| 1068 | val = val - cpuhw->ic_init; |
| 1069 | else |
| 1070 | val = val + (0 - cpuhw->ic_init); |
| 1071 | } |
Eric B Munson | 86c74ab | 2011-04-15 08:12:30 +0000 | [diff] [blame] | 1072 | delta = check_and_compute_delta(prev, val); |
| 1073 | if (!delta) |
| 1074 | return; |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 1075 | } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1076 | |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 1077 | local64_add(delta, &event->count); |
Anton Blanchard | f560294 | 2014-05-29 08:15:38 +1000 | [diff] [blame] | 1078 | |
| 1079 | /* |
| 1080 | * A number of places program the PMC with (0x80000000 - period_left). |
| 1081 | * We never want period_left to be less than 1 because we will program |
| 1082 | * the PMC with a value >= 0x800000000 and an edge detected PMC will |
| 1083 | * roll around to 0 before taking an exception. We have seen this |
| 1084 | * on POWER8. |
| 1085 | * |
| 1086 | * To fix this, clamp the minimum value of period_left to 1. |
| 1087 | */ |
| 1088 | do { |
| 1089 | prev = local64_read(&event->hw.period_left); |
| 1090 | val = prev - delta; |
| 1091 | if (val < 1) |
| 1092 | val = 1; |
| 1093 | } while (local64_cmpxchg(&event->hw.period_left, prev, val) != prev); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1094 | } |
| 1095 | |
| 1096 | /* |
| 1097 | * On some machines, PMC5 and PMC6 can't be written, don't respect |
| 1098 | * the freeze conditions, and don't generate interrupts. This tells |
| 1099 | * us if `event' is using such a PMC. |
| 1100 | */ |
| 1101 | static int is_limited_pmc(int pmcnum) |
| 1102 | { |
| 1103 | return (ppmu->flags & PPMU_LIMITED_PMC5_6) |
| 1104 | && (pmcnum == 5 || pmcnum == 6); |
| 1105 | } |
| 1106 | |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1107 | static void freeze_limited_counters(struct cpu_hw_events *cpuhw, |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1108 | unsigned long pmc5, unsigned long pmc6) |
| 1109 | { |
| 1110 | struct perf_event *event; |
| 1111 | u64 val, prev, delta; |
| 1112 | int i; |
| 1113 | |
| 1114 | for (i = 0; i < cpuhw->n_limited; ++i) { |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1115 | event = cpuhw->limited_counter[i]; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1116 | if (!event->hw.idx) |
| 1117 | continue; |
| 1118 | val = (event->hw.idx == 5) ? pmc5 : pmc6; |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 1119 | prev = local64_read(&event->hw.prev_count); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1120 | event->hw.idx = 0; |
Eric B Munson | 86c74ab | 2011-04-15 08:12:30 +0000 | [diff] [blame] | 1121 | delta = check_and_compute_delta(prev, val); |
| 1122 | if (delta) |
| 1123 | local64_add(delta, &event->count); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1124 | } |
| 1125 | } |
| 1126 | |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1127 | static void thaw_limited_counters(struct cpu_hw_events *cpuhw, |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1128 | unsigned long pmc5, unsigned long pmc6) |
| 1129 | { |
| 1130 | struct perf_event *event; |
Eric B Munson | 86c74ab | 2011-04-15 08:12:30 +0000 | [diff] [blame] | 1131 | u64 val, prev; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1132 | int i; |
| 1133 | |
| 1134 | for (i = 0; i < cpuhw->n_limited; ++i) { |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1135 | event = cpuhw->limited_counter[i]; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1136 | event->hw.idx = cpuhw->limited_hwidx[i]; |
| 1137 | val = (event->hw.idx == 5) ? pmc5 : pmc6; |
Eric B Munson | 86c74ab | 2011-04-15 08:12:30 +0000 | [diff] [blame] | 1138 | prev = local64_read(&event->hw.prev_count); |
| 1139 | if (check_and_compute_delta(prev, val)) |
| 1140 | local64_set(&event->hw.prev_count, val); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1141 | perf_event_update_userpage(event); |
| 1142 | } |
| 1143 | } |
| 1144 | |
| 1145 | /* |
| 1146 | * Since limited events don't respect the freeze conditions, we |
| 1147 | * have to read them immediately after freezing or unfreezing the |
| 1148 | * other events. We try to keep the values from the limited |
| 1149 | * events as consistent as possible by keeping the delay (in |
| 1150 | * cycles and instructions) between freezing/unfreezing and reading |
| 1151 | * the limited events as small and consistent as possible. |
| 1152 | * Therefore, if any limited events are in use, we read them |
| 1153 | * both, and always in the same order, to minimize variability, |
| 1154 | * and do it inside the same asm that writes MMCR0. |
| 1155 | */ |
| 1156 | static void write_mmcr0(struct cpu_hw_events *cpuhw, unsigned long mmcr0) |
| 1157 | { |
| 1158 | unsigned long pmc5, pmc6; |
| 1159 | |
| 1160 | if (!cpuhw->n_limited) { |
| 1161 | mtspr(SPRN_MMCR0, mmcr0); |
| 1162 | return; |
| 1163 | } |
| 1164 | |
| 1165 | /* |
| 1166 | * Write MMCR0, then read PMC5 and PMC6 immediately. |
| 1167 | * To ensure we don't get a performance monitor interrupt |
| 1168 | * between writing MMCR0 and freezing/thawing the limited |
| 1169 | * events, we first write MMCR0 with the event overflow |
| 1170 | * interrupt enable bits turned off. |
| 1171 | */ |
| 1172 | asm volatile("mtspr %3,%2; mfspr %0,%4; mfspr %1,%5" |
| 1173 | : "=&r" (pmc5), "=&r" (pmc6) |
| 1174 | : "r" (mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)), |
| 1175 | "i" (SPRN_MMCR0), |
| 1176 | "i" (SPRN_PMC5), "i" (SPRN_PMC6)); |
| 1177 | |
| 1178 | if (mmcr0 & MMCR0_FC) |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1179 | freeze_limited_counters(cpuhw, pmc5, pmc6); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1180 | else |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1181 | thaw_limited_counters(cpuhw, pmc5, pmc6); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1182 | |
| 1183 | /* |
| 1184 | * Write the full MMCR0 including the event overflow interrupt |
| 1185 | * enable bits, if necessary. |
| 1186 | */ |
| 1187 | if (mmcr0 & (MMCR0_PMC1CE | MMCR0_PMCjCE)) |
| 1188 | mtspr(SPRN_MMCR0, mmcr0); |
| 1189 | } |
| 1190 | |
| 1191 | /* |
| 1192 | * Disable all events to prevent PMU interrupts and to allow |
| 1193 | * events to be added or removed. |
| 1194 | */ |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1195 | static void power_pmu_disable(struct pmu *pmu) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1196 | { |
| 1197 | struct cpu_hw_events *cpuhw; |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1198 | unsigned long flags, mmcr0, val; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1199 | |
| 1200 | if (!ppmu) |
| 1201 | return; |
| 1202 | local_irq_save(flags); |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 1203 | cpuhw = this_cpu_ptr(&cpu_hw_events); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1204 | |
| 1205 | if (!cpuhw->disabled) { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1206 | /* |
| 1207 | * Check if we ever enabled the PMU on this cpu. |
| 1208 | */ |
| 1209 | if (!cpuhw->pmcs_enabled) { |
| 1210 | ppc_enable_pmcs(); |
| 1211 | cpuhw->pmcs_enabled = 1; |
| 1212 | } |
| 1213 | |
| 1214 | /* |
Michael Ellerman | 76cb8a7 | 2014-03-14 16:00:34 +1100 | [diff] [blame] | 1215 | * Set the 'freeze counters' bit, clear EBE/BHRBA/PMCC/PMAO/FC56 |
Michael Ellerman | 378a6ee | 2013-06-28 18:15:11 +1000 | [diff] [blame] | 1216 | */ |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1217 | val = mmcr0 = mfspr(SPRN_MMCR0); |
Michael Ellerman | 378a6ee | 2013-06-28 18:15:11 +1000 | [diff] [blame] | 1218 | val |= MMCR0_FC; |
Michael Ellerman | 76cb8a7 | 2014-03-14 16:00:34 +1100 | [diff] [blame] | 1219 | val &= ~(MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC | MMCR0_PMAO | |
| 1220 | MMCR0_FC56); |
Michael Ellerman | 378a6ee | 2013-06-28 18:15:11 +1000 | [diff] [blame] | 1221 | |
| 1222 | /* |
| 1223 | * The barrier is to make sure the mtspr has been |
| 1224 | * executed and the PMU has frozen the events etc. |
| 1225 | * before we return. |
| 1226 | */ |
| 1227 | write_mmcr0(cpuhw, val); |
| 1228 | mb(); |
| 1229 | |
| 1230 | /* |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1231 | * Disable instruction sampling if it was enabled |
| 1232 | */ |
| 1233 | if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) { |
| 1234 | mtspr(SPRN_MMCRA, |
| 1235 | cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE); |
| 1236 | mb(); |
| 1237 | } |
| 1238 | |
Michael Ellerman | 378a6ee | 2013-06-28 18:15:11 +1000 | [diff] [blame] | 1239 | cpuhw->disabled = 1; |
| 1240 | cpuhw->n_added = 0; |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1241 | |
| 1242 | ebb_switch_out(mmcr0); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1243 | } |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1244 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1245 | local_irq_restore(flags); |
| 1246 | } |
| 1247 | |
| 1248 | /* |
| 1249 | * Re-enable all events if disable == 0. |
| 1250 | * If we were previously disabled and events were added, then |
| 1251 | * put the new config on the PMU. |
| 1252 | */ |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1253 | static void power_pmu_enable(struct pmu *pmu) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1254 | { |
| 1255 | struct perf_event *event; |
| 1256 | struct cpu_hw_events *cpuhw; |
| 1257 | unsigned long flags; |
| 1258 | long i; |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1259 | unsigned long val, mmcr0; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1260 | s64 left; |
| 1261 | unsigned int hwc_index[MAX_HWEVENTS]; |
| 1262 | int n_lim; |
| 1263 | int idx; |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1264 | bool ebb; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1265 | |
| 1266 | if (!ppmu) |
| 1267 | return; |
| 1268 | local_irq_save(flags); |
Michael Ellerman | 0a48843 | 2013-06-28 18:15:13 +1000 | [diff] [blame] | 1269 | |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 1270 | cpuhw = this_cpu_ptr(&cpu_hw_events); |
Michael Ellerman | 0a48843 | 2013-06-28 18:15:13 +1000 | [diff] [blame] | 1271 | if (!cpuhw->disabled) |
| 1272 | goto out; |
| 1273 | |
Michael Ellerman | 4ea355b | 2013-06-28 18:15:14 +1000 | [diff] [blame] | 1274 | if (cpuhw->n_events == 0) { |
| 1275 | ppc_set_pmu_inuse(0); |
| 1276 | goto out; |
| 1277 | } |
| 1278 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1279 | cpuhw->disabled = 0; |
| 1280 | |
| 1281 | /* |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1282 | * EBB requires an exclusive group and all events must have the EBB |
| 1283 | * flag set, or not set, so we can just check a single event. Also we |
| 1284 | * know we have at least one event. |
| 1285 | */ |
| 1286 | ebb = is_ebb_event(cpuhw->event[0]); |
| 1287 | |
| 1288 | /* |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1289 | * If we didn't change anything, or only removed events, |
| 1290 | * no need to recalculate MMCR* settings and reset the PMCs. |
| 1291 | * Just reenable the PMU with the current MMCR* settings |
| 1292 | * (possibly updated for removal of events). |
| 1293 | */ |
| 1294 | if (!cpuhw->n_added) { |
| 1295 | mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE); |
| 1296 | mtspr(SPRN_MMCR1, cpuhw->mmcr[1]); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1297 | goto out_enable; |
| 1298 | } |
| 1299 | |
| 1300 | /* |
Michael Ellerman | 79a4cb2 | 2014-07-23 21:12:36 +1000 | [diff] [blame] | 1301 | * Clear all MMCR settings and recompute them for the new set of events. |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1302 | */ |
Michael Ellerman | 79a4cb2 | 2014-07-23 21:12:36 +1000 | [diff] [blame] | 1303 | memset(cpuhw->mmcr, 0, sizeof(cpuhw->mmcr)); |
| 1304 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1305 | if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index, |
Michael Ellerman | 8abd818 | 2014-07-23 21:12:37 +1000 | [diff] [blame] | 1306 | cpuhw->mmcr, cpuhw->event)) { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1307 | /* shouldn't ever get here */ |
| 1308 | printk(KERN_ERR "oops compute_mmcr failed\n"); |
| 1309 | goto out; |
| 1310 | } |
| 1311 | |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 1312 | if (!(ppmu->flags & PPMU_ARCH_207S)) { |
| 1313 | /* |
| 1314 | * Add in MMCR0 freeze bits corresponding to the attr.exclude_* |
| 1315 | * bits for the first event. We have already checked that all |
| 1316 | * events have the same value for these bits as the first event. |
| 1317 | */ |
| 1318 | event = cpuhw->event[0]; |
| 1319 | if (event->attr.exclude_user) |
| 1320 | cpuhw->mmcr[0] |= MMCR0_FCP; |
| 1321 | if (event->attr.exclude_kernel) |
| 1322 | cpuhw->mmcr[0] |= freeze_events_kernel; |
| 1323 | if (event->attr.exclude_hv) |
| 1324 | cpuhw->mmcr[0] |= MMCR0_FCHV; |
| 1325 | } |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1326 | |
| 1327 | /* |
| 1328 | * Write the new configuration to MMCR* with the freeze |
| 1329 | * bit set and set the hardware events to their initial values. |
| 1330 | * Then unfreeze the events. |
| 1331 | */ |
| 1332 | ppc_set_pmu_inuse(1); |
| 1333 | mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE); |
| 1334 | mtspr(SPRN_MMCR1, cpuhw->mmcr[1]); |
| 1335 | mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)) |
| 1336 | | MMCR0_FC); |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 1337 | if (ppmu->flags & PPMU_ARCH_207S) |
| 1338 | mtspr(SPRN_MMCR2, cpuhw->mmcr[3]); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1339 | |
| 1340 | /* |
| 1341 | * Read off any pre-existing events that need to move |
| 1342 | * to another PMC. |
| 1343 | */ |
| 1344 | for (i = 0; i < cpuhw->n_events; ++i) { |
| 1345 | event = cpuhw->event[i]; |
| 1346 | if (event->hw.idx && event->hw.idx != hwc_index[i] + 1) { |
| 1347 | power_pmu_read(event); |
| 1348 | write_pmc(event->hw.idx, 0); |
| 1349 | event->hw.idx = 0; |
| 1350 | } |
| 1351 | } |
| 1352 | |
| 1353 | /* |
| 1354 | * Initialize the PMCs for all the new and moved events. |
| 1355 | */ |
| 1356 | cpuhw->n_limited = n_lim = 0; |
| 1357 | for (i = 0; i < cpuhw->n_events; ++i) { |
| 1358 | event = cpuhw->event[i]; |
| 1359 | if (event->hw.idx) |
| 1360 | continue; |
| 1361 | idx = hwc_index[i] + 1; |
| 1362 | if (is_limited_pmc(idx)) { |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1363 | cpuhw->limited_counter[n_lim] = event; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1364 | cpuhw->limited_hwidx[n_lim] = idx; |
| 1365 | ++n_lim; |
| 1366 | continue; |
| 1367 | } |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1368 | |
| 1369 | if (ebb) |
| 1370 | val = local64_read(&event->hw.prev_count); |
| 1371 | else { |
| 1372 | val = 0; |
| 1373 | if (event->hw.sample_period) { |
| 1374 | left = local64_read(&event->hw.period_left); |
| 1375 | if (left < 0x80000000L) |
| 1376 | val = 0x80000000L - left; |
| 1377 | } |
| 1378 | local64_set(&event->hw.prev_count, val); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1379 | } |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1380 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1381 | event->hw.idx = idx; |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1382 | if (event->hw.state & PERF_HES_STOPPED) |
| 1383 | val = 0; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1384 | write_pmc(idx, val); |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1385 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1386 | perf_event_update_userpage(event); |
| 1387 | } |
| 1388 | cpuhw->n_limited = n_lim; |
| 1389 | cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE; |
| 1390 | |
| 1391 | out_enable: |
Michael Ellerman | c2e37a2 | 2014-03-14 16:00:29 +1100 | [diff] [blame] | 1392 | pmao_restore_workaround(ebb); |
| 1393 | |
Michael Ellerman | 9de5cb0 | 2014-07-23 21:12:38 +1000 | [diff] [blame] | 1394 | mmcr0 = ebb_switch_in(ebb, cpuhw); |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1395 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1396 | mb(); |
Anshuman Khandual | b4d6c06 | 2013-12-18 13:14:53 +1100 | [diff] [blame] | 1397 | if (cpuhw->bhrb_users) |
| 1398 | ppmu->config_bhrb(cpuhw->bhrb_filter); |
| 1399 | |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1400 | write_mmcr0(cpuhw, mmcr0); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1401 | |
| 1402 | /* |
| 1403 | * Enable instruction sampling if necessary |
| 1404 | */ |
| 1405 | if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) { |
| 1406 | mb(); |
| 1407 | mtspr(SPRN_MMCRA, cpuhw->mmcr[2]); |
| 1408 | } |
| 1409 | |
| 1410 | out: |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1411 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1412 | local_irq_restore(flags); |
| 1413 | } |
| 1414 | |
| 1415 | static int collect_events(struct perf_event *group, int max_count, |
| 1416 | struct perf_event *ctrs[], u64 *events, |
| 1417 | unsigned int *flags) |
| 1418 | { |
| 1419 | int n = 0; |
| 1420 | struct perf_event *event; |
| 1421 | |
Ravi Bangoria | 5aa04b3 | 2017-11-30 14:03:22 +0530 | [diff] [blame] | 1422 | if (group->pmu->task_ctx_nr == perf_hw_context) { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1423 | if (n >= max_count) |
| 1424 | return -1; |
| 1425 | ctrs[n] = group; |
| 1426 | flags[n] = group->hw.event_base; |
| 1427 | events[n++] = group->hw.config; |
| 1428 | } |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1429 | list_for_each_entry(event, &group->sibling_list, group_entry) { |
Ravi Bangoria | 5aa04b3 | 2017-11-30 14:03:22 +0530 | [diff] [blame] | 1430 | if (event->pmu->task_ctx_nr == perf_hw_context && |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1431 | event->state != PERF_EVENT_STATE_OFF) { |
| 1432 | if (n >= max_count) |
| 1433 | return -1; |
| 1434 | ctrs[n] = event; |
| 1435 | flags[n] = event->hw.event_base; |
| 1436 | events[n++] = event->hw.config; |
| 1437 | } |
| 1438 | } |
| 1439 | return n; |
| 1440 | } |
| 1441 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1442 | /* |
| 1443 | * Add a event to the PMU. |
| 1444 | * If all events are not already frozen, then we disable and |
| 1445 | * re-enable the PMU in order to get hw_perf_enable to do the |
| 1446 | * actual work of reconfiguring the PMU. |
| 1447 | */ |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1448 | static int power_pmu_add(struct perf_event *event, int ef_flags) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1449 | { |
| 1450 | struct cpu_hw_events *cpuhw; |
| 1451 | unsigned long flags; |
| 1452 | int n0; |
| 1453 | int ret = -EAGAIN; |
| 1454 | |
| 1455 | local_irq_save(flags); |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1456 | perf_pmu_disable(event->pmu); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1457 | |
| 1458 | /* |
| 1459 | * Add the event to the list (if there is room) |
| 1460 | * and check whether the total set is still feasible. |
| 1461 | */ |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 1462 | cpuhw = this_cpu_ptr(&cpu_hw_events); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1463 | n0 = cpuhw->n_events; |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1464 | if (n0 >= ppmu->n_counter) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1465 | goto out; |
| 1466 | cpuhw->event[n0] = event; |
| 1467 | cpuhw->events[n0] = event->hw.config; |
| 1468 | cpuhw->flags[n0] = event->hw.event_base; |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1469 | |
sukadev@linux.vnet.ibm.com | f53d168 | 2013-01-24 13:25:23 +0000 | [diff] [blame] | 1470 | /* |
| 1471 | * This event may have been disabled/stopped in record_and_restart() |
| 1472 | * because we exceeded the ->event_limit. If re-starting the event, |
| 1473 | * clear the ->hw.state (STOPPED and UPTODATE flags), so the user |
| 1474 | * notification is re-enabled. |
| 1475 | */ |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1476 | if (!(ef_flags & PERF_EF_START)) |
| 1477 | event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
sukadev@linux.vnet.ibm.com | f53d168 | 2013-01-24 13:25:23 +0000 | [diff] [blame] | 1478 | else |
| 1479 | event->hw.state = 0; |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1480 | |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1481 | /* |
| 1482 | * If group events scheduling transaction was started, |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 1483 | * skip the schedulability test here, it will be performed |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1484 | * at commit time(->commit_txn) as a whole |
| 1485 | */ |
Sukadev Bhattiprolu | 8f3e568 | 2015-09-03 20:07:53 -0700 | [diff] [blame] | 1486 | if (cpuhw->txn_flags & PERF_PMU_TXN_ADD) |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1487 | goto nocheck; |
| 1488 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1489 | if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1)) |
| 1490 | goto out; |
| 1491 | if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1)) |
| 1492 | goto out; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1493 | event->hw.config = cpuhw->events[n0]; |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1494 | |
| 1495 | nocheck: |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1496 | ebb_event_add(event); |
| 1497 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1498 | ++cpuhw->n_events; |
| 1499 | ++cpuhw->n_added; |
| 1500 | |
| 1501 | ret = 0; |
| 1502 | out: |
Anshuman Khandual | ff3d79d | 2013-06-10 11:23:29 +0530 | [diff] [blame] | 1503 | if (has_branch_stack(event)) { |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1504 | power_pmu_bhrb_enable(event); |
Anshuman Khandual | ff3d79d | 2013-06-10 11:23:29 +0530 | [diff] [blame] | 1505 | cpuhw->bhrb_filter = ppmu->bhrb_filter_map( |
| 1506 | event->attr.branch_sample_type); |
| 1507 | } |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1508 | |
Madhavan Srinivasan | 356d8ce | 2017-02-12 22:33:14 +0530 | [diff] [blame] | 1509 | /* |
| 1510 | * Workaround for POWER9 DD1 to use the Instruction Counter |
| 1511 | * register value for instruction counting |
| 1512 | */ |
| 1513 | if (use_ic(event->attr.config)) |
| 1514 | cpuhw->ic_init = mfspr(SPRN_IC); |
| 1515 | |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1516 | perf_pmu_enable(event->pmu); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1517 | local_irq_restore(flags); |
| 1518 | return ret; |
| 1519 | } |
| 1520 | |
| 1521 | /* |
| 1522 | * Remove a event from the PMU. |
| 1523 | */ |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1524 | static void power_pmu_del(struct perf_event *event, int ef_flags) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1525 | { |
| 1526 | struct cpu_hw_events *cpuhw; |
| 1527 | long i; |
| 1528 | unsigned long flags; |
| 1529 | |
| 1530 | local_irq_save(flags); |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1531 | perf_pmu_disable(event->pmu); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1532 | |
| 1533 | power_pmu_read(event); |
| 1534 | |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 1535 | cpuhw = this_cpu_ptr(&cpu_hw_events); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1536 | for (i = 0; i < cpuhw->n_events; ++i) { |
| 1537 | if (event == cpuhw->event[i]) { |
Matt Evans | 219a92a | 2010-07-05 17:36:32 +0000 | [diff] [blame] | 1538 | while (++i < cpuhw->n_events) { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1539 | cpuhw->event[i-1] = cpuhw->event[i]; |
Matt Evans | 219a92a | 2010-07-05 17:36:32 +0000 | [diff] [blame] | 1540 | cpuhw->events[i-1] = cpuhw->events[i]; |
| 1541 | cpuhw->flags[i-1] = cpuhw->flags[i]; |
| 1542 | } |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1543 | --cpuhw->n_events; |
| 1544 | ppmu->disable_pmc(event->hw.idx - 1, cpuhw->mmcr); |
| 1545 | if (event->hw.idx) { |
| 1546 | write_pmc(event->hw.idx, 0); |
| 1547 | event->hw.idx = 0; |
| 1548 | } |
| 1549 | perf_event_update_userpage(event); |
| 1550 | break; |
| 1551 | } |
| 1552 | } |
| 1553 | for (i = 0; i < cpuhw->n_limited; ++i) |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1554 | if (event == cpuhw->limited_counter[i]) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1555 | break; |
| 1556 | if (i < cpuhw->n_limited) { |
| 1557 | while (++i < cpuhw->n_limited) { |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1558 | cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i]; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1559 | cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i]; |
| 1560 | } |
| 1561 | --cpuhw->n_limited; |
| 1562 | } |
| 1563 | if (cpuhw->n_events == 0) { |
| 1564 | /* disable exceptions if no events are running */ |
| 1565 | cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE); |
| 1566 | } |
| 1567 | |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1568 | if (has_branch_stack(event)) |
| 1569 | power_pmu_bhrb_disable(event); |
| 1570 | |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1571 | perf_pmu_enable(event->pmu); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1572 | local_irq_restore(flags); |
| 1573 | } |
| 1574 | |
| 1575 | /* |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1576 | * POWER-PMU does not support disabling individual counters, hence |
| 1577 | * program their cycle counter to their max value and ignore the interrupts. |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1578 | */ |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1579 | |
| 1580 | static void power_pmu_start(struct perf_event *event, int ef_flags) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1581 | { |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1582 | unsigned long flags; |
| 1583 | s64 left; |
Anton Blanchard | 9a45a94 | 2012-02-15 18:48:22 +0000 | [diff] [blame] | 1584 | unsigned long val; |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1585 | |
| 1586 | if (!event->hw.idx || !event->hw.sample_period) |
| 1587 | return; |
| 1588 | |
| 1589 | if (!(event->hw.state & PERF_HES_STOPPED)) |
| 1590 | return; |
| 1591 | |
| 1592 | if (ef_flags & PERF_EF_RELOAD) |
| 1593 | WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); |
| 1594 | |
| 1595 | local_irq_save(flags); |
| 1596 | perf_pmu_disable(event->pmu); |
| 1597 | |
| 1598 | event->hw.state = 0; |
| 1599 | left = local64_read(&event->hw.period_left); |
Anton Blanchard | 9a45a94 | 2012-02-15 18:48:22 +0000 | [diff] [blame] | 1600 | |
| 1601 | val = 0; |
| 1602 | if (left < 0x80000000L) |
| 1603 | val = 0x80000000L - left; |
| 1604 | |
| 1605 | write_pmc(event->hw.idx, val); |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1606 | |
| 1607 | perf_event_update_userpage(event); |
| 1608 | perf_pmu_enable(event->pmu); |
| 1609 | local_irq_restore(flags); |
| 1610 | } |
| 1611 | |
| 1612 | static void power_pmu_stop(struct perf_event *event, int ef_flags) |
| 1613 | { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1614 | unsigned long flags; |
| 1615 | |
| 1616 | if (!event->hw.idx || !event->hw.sample_period) |
| 1617 | return; |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1618 | |
| 1619 | if (event->hw.state & PERF_HES_STOPPED) |
| 1620 | return; |
| 1621 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1622 | local_irq_save(flags); |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1623 | perf_pmu_disable(event->pmu); |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1624 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1625 | power_pmu_read(event); |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1626 | event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 1627 | write_pmc(event->hw.idx, 0); |
| 1628 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1629 | perf_event_update_userpage(event); |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1630 | perf_pmu_enable(event->pmu); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1631 | local_irq_restore(flags); |
| 1632 | } |
| 1633 | |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1634 | /* |
| 1635 | * Start group events scheduling transaction |
| 1636 | * Set the flag to make pmu::enable() not perform the |
| 1637 | * schedulability test, it will be performed at commit time |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 1638 | * |
| 1639 | * We only support PERF_PMU_TXN_ADD transactions. Save the |
| 1640 | * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD |
| 1641 | * transactions. |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1642 | */ |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 1643 | static void power_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1644 | { |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 1645 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1646 | |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 1647 | WARN_ON_ONCE(cpuhw->txn_flags); /* txn already in flight */ |
| 1648 | |
| 1649 | cpuhw->txn_flags = txn_flags; |
| 1650 | if (txn_flags & ~PERF_PMU_TXN_ADD) |
| 1651 | return; |
| 1652 | |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1653 | perf_pmu_disable(pmu); |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1654 | cpuhw->n_txn_start = cpuhw->n_events; |
| 1655 | } |
| 1656 | |
| 1657 | /* |
| 1658 | * Stop group events scheduling transaction |
| 1659 | * Clear the flag and pmu::enable() will perform the |
| 1660 | * schedulability test. |
| 1661 | */ |
Anton Blanchard | e51df2c | 2014-08-20 08:55:18 +1000 | [diff] [blame] | 1662 | static void power_pmu_cancel_txn(struct pmu *pmu) |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1663 | { |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 1664 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 1665 | unsigned int txn_flags; |
| 1666 | |
| 1667 | WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ |
| 1668 | |
| 1669 | txn_flags = cpuhw->txn_flags; |
| 1670 | cpuhw->txn_flags = 0; |
| 1671 | if (txn_flags & ~PERF_PMU_TXN_ADD) |
| 1672 | return; |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1673 | |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1674 | perf_pmu_enable(pmu); |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1675 | } |
| 1676 | |
| 1677 | /* |
| 1678 | * Commit group events scheduling transaction |
| 1679 | * Perform the group schedulability test as a whole |
| 1680 | * Return 0 if success |
| 1681 | */ |
Anton Blanchard | e51df2c | 2014-08-20 08:55:18 +1000 | [diff] [blame] | 1682 | static int power_pmu_commit_txn(struct pmu *pmu) |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1683 | { |
| 1684 | struct cpu_hw_events *cpuhw; |
| 1685 | long i, n; |
| 1686 | |
| 1687 | if (!ppmu) |
| 1688 | return -EAGAIN; |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 1689 | |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 1690 | cpuhw = this_cpu_ptr(&cpu_hw_events); |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 1691 | WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ |
| 1692 | |
| 1693 | if (cpuhw->txn_flags & ~PERF_PMU_TXN_ADD) { |
| 1694 | cpuhw->txn_flags = 0; |
| 1695 | return 0; |
| 1696 | } |
| 1697 | |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1698 | n = cpuhw->n_events; |
| 1699 | if (check_excludes(cpuhw->event, cpuhw->flags, 0, n)) |
| 1700 | return -EAGAIN; |
| 1701 | i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n); |
| 1702 | if (i < 0) |
| 1703 | return -EAGAIN; |
| 1704 | |
| 1705 | for (i = cpuhw->n_txn_start; i < n; ++i) |
| 1706 | cpuhw->event[i]->hw.config = cpuhw->events[i]; |
| 1707 | |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 1708 | cpuhw->txn_flags = 0; |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1709 | perf_pmu_enable(pmu); |
Lin Ming | 8e6d557 | 2010-05-08 20:28:41 +1000 | [diff] [blame] | 1710 | return 0; |
| 1711 | } |
| 1712 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1713 | /* |
| 1714 | * Return 1 if we might be able to put event on a limited PMC, |
| 1715 | * or 0 if not. |
| 1716 | * A event can only go on a limited PMC if it counts something |
| 1717 | * that a limited PMC can count, doesn't require interrupts, and |
| 1718 | * doesn't exclude any processor mode. |
| 1719 | */ |
| 1720 | static int can_go_on_limited_pmc(struct perf_event *event, u64 ev, |
| 1721 | unsigned int flags) |
| 1722 | { |
| 1723 | int n; |
| 1724 | u64 alt[MAX_EVENT_ALTERNATIVES]; |
| 1725 | |
| 1726 | if (event->attr.exclude_user |
| 1727 | || event->attr.exclude_kernel |
| 1728 | || event->attr.exclude_hv |
| 1729 | || event->attr.sample_period) |
| 1730 | return 0; |
| 1731 | |
| 1732 | if (ppmu->limited_pmc_event(ev)) |
| 1733 | return 1; |
| 1734 | |
| 1735 | /* |
| 1736 | * The requested event_id isn't on a limited PMC already; |
| 1737 | * see if any alternative code goes on a limited PMC. |
| 1738 | */ |
| 1739 | if (!ppmu->get_alternatives) |
| 1740 | return 0; |
| 1741 | |
| 1742 | flags |= PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD; |
| 1743 | n = ppmu->get_alternatives(ev, flags, alt); |
| 1744 | |
| 1745 | return n > 0; |
| 1746 | } |
| 1747 | |
| 1748 | /* |
| 1749 | * Find an alternative event_id that goes on a normal PMC, if possible, |
| 1750 | * and return the event_id code, or 0 if there is no such alternative. |
| 1751 | * (Note: event_id code 0 is "don't count" on all machines.) |
| 1752 | */ |
| 1753 | static u64 normal_pmc_alternative(u64 ev, unsigned long flags) |
| 1754 | { |
| 1755 | u64 alt[MAX_EVENT_ALTERNATIVES]; |
| 1756 | int n; |
| 1757 | |
| 1758 | flags &= ~(PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD); |
| 1759 | n = ppmu->get_alternatives(ev, flags, alt); |
| 1760 | if (!n) |
| 1761 | return 0; |
| 1762 | return alt[0]; |
| 1763 | } |
| 1764 | |
| 1765 | /* Number of perf_events counting hardware events */ |
| 1766 | static atomic_t num_events; |
| 1767 | /* Used to avoid races in calling reserve/release_pmc_hardware */ |
| 1768 | static DEFINE_MUTEX(pmc_reserve_mutex); |
| 1769 | |
| 1770 | /* |
| 1771 | * Release the PMU if this is the last perf_event. |
| 1772 | */ |
| 1773 | static void hw_perf_event_destroy(struct perf_event *event) |
| 1774 | { |
| 1775 | if (!atomic_add_unless(&num_events, -1, 1)) { |
| 1776 | mutex_lock(&pmc_reserve_mutex); |
| 1777 | if (atomic_dec_return(&num_events) == 0) |
| 1778 | release_pmc_hardware(); |
| 1779 | mutex_unlock(&pmc_reserve_mutex); |
| 1780 | } |
| 1781 | } |
| 1782 | |
| 1783 | /* |
| 1784 | * Translate a generic cache event_id config to a raw event_id code. |
| 1785 | */ |
| 1786 | static int hw_perf_cache_event(u64 config, u64 *eventp) |
| 1787 | { |
| 1788 | unsigned long type, op, result; |
| 1789 | int ev; |
| 1790 | |
| 1791 | if (!ppmu->cache_events) |
| 1792 | return -EINVAL; |
| 1793 | |
| 1794 | /* unpack config */ |
| 1795 | type = config & 0xff; |
| 1796 | op = (config >> 8) & 0xff; |
| 1797 | result = (config >> 16) & 0xff; |
| 1798 | |
| 1799 | if (type >= PERF_COUNT_HW_CACHE_MAX || |
| 1800 | op >= PERF_COUNT_HW_CACHE_OP_MAX || |
| 1801 | result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
| 1802 | return -EINVAL; |
| 1803 | |
| 1804 | ev = (*ppmu->cache_events)[type][op][result]; |
| 1805 | if (ev == 0) |
| 1806 | return -EOPNOTSUPP; |
| 1807 | if (ev == -1) |
| 1808 | return -EINVAL; |
| 1809 | *eventp = ev; |
| 1810 | return 0; |
| 1811 | } |
| 1812 | |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1813 | static int power_pmu_event_init(struct perf_event *event) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1814 | { |
| 1815 | u64 ev; |
| 1816 | unsigned long flags; |
| 1817 | struct perf_event *ctrs[MAX_HWEVENTS]; |
| 1818 | u64 events[MAX_HWEVENTS]; |
| 1819 | unsigned int cflags[MAX_HWEVENTS]; |
| 1820 | int n; |
| 1821 | int err; |
| 1822 | struct cpu_hw_events *cpuhw; |
| 1823 | |
| 1824 | if (!ppmu) |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1825 | return -ENOENT; |
| 1826 | |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1827 | if (has_branch_stack(event)) { |
| 1828 | /* PMU has BHRB enabled */ |
Joel Stanley | 4d9690d | 2014-07-08 16:08:21 +0930 | [diff] [blame] | 1829 | if (!(ppmu->flags & PPMU_ARCH_207S)) |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1830 | return -EOPNOTSUPP; |
| 1831 | } |
Stephane Eranian | 2481c5f | 2012-02-09 23:20:59 +0100 | [diff] [blame] | 1832 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1833 | switch (event->attr.type) { |
| 1834 | case PERF_TYPE_HARDWARE: |
| 1835 | ev = event->attr.config; |
| 1836 | if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0) |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1837 | return -EOPNOTSUPP; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1838 | ev = ppmu->generic_events[ev]; |
| 1839 | break; |
| 1840 | case PERF_TYPE_HW_CACHE: |
| 1841 | err = hw_perf_cache_event(event->attr.config, &ev); |
| 1842 | if (err) |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1843 | return err; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1844 | break; |
| 1845 | case PERF_TYPE_RAW: |
| 1846 | ev = event->attr.config; |
| 1847 | break; |
| 1848 | default: |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1849 | return -ENOENT; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1850 | } |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1851 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1852 | event->hw.config_base = ev; |
| 1853 | event->hw.idx = 0; |
| 1854 | |
| 1855 | /* |
| 1856 | * If we are not running on a hypervisor, force the |
| 1857 | * exclude_hv bit to 0 so that we don't care what |
| 1858 | * the user set it to. |
| 1859 | */ |
| 1860 | if (!firmware_has_feature(FW_FEATURE_LPAR)) |
| 1861 | event->attr.exclude_hv = 0; |
| 1862 | |
| 1863 | /* |
| 1864 | * If this is a per-task event, then we can use |
| 1865 | * PM_RUN_* events interchangeably with their non RUN_* |
| 1866 | * equivalents, e.g. PM_RUN_CYC instead of PM_CYC. |
| 1867 | * XXX we should check if the task is an idle task. |
| 1868 | */ |
| 1869 | flags = 0; |
Paul Mackerras | 57fa721 | 2010-10-19 16:55:35 +1100 | [diff] [blame] | 1870 | if (event->attach_state & PERF_ATTACH_TASK) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1871 | flags |= PPMU_ONLY_COUNT_RUN; |
| 1872 | |
| 1873 | /* |
| 1874 | * If this machine has limited events, check whether this |
| 1875 | * event_id could go on a limited event. |
| 1876 | */ |
| 1877 | if (ppmu->flags & PPMU_LIMITED_PMC5_6) { |
| 1878 | if (can_go_on_limited_pmc(event, ev, flags)) { |
| 1879 | flags |= PPMU_LIMITED_PMC_OK; |
| 1880 | } else if (ppmu->limited_pmc_event(ev)) { |
| 1881 | /* |
| 1882 | * The requested event_id is on a limited PMC, |
| 1883 | * but we can't use a limited PMC; see if any |
| 1884 | * alternative goes on a normal PMC. |
| 1885 | */ |
| 1886 | ev = normal_pmc_alternative(ev, flags); |
| 1887 | if (!ev) |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1888 | return -EINVAL; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1889 | } |
| 1890 | } |
| 1891 | |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1892 | /* Extra checks for EBB */ |
| 1893 | err = ebb_event_check(event); |
| 1894 | if (err) |
| 1895 | return err; |
| 1896 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1897 | /* |
| 1898 | * If this is in a group, check if it can go on with all the |
| 1899 | * other hardware events in the group. We assume the event |
| 1900 | * hasn't been linked into its leader's sibling list at this point. |
| 1901 | */ |
| 1902 | n = 0; |
| 1903 | if (event->group_leader != event) { |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 1904 | n = collect_events(event->group_leader, ppmu->n_counter - 1, |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1905 | ctrs, events, cflags); |
| 1906 | if (n < 0) |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1907 | return -EINVAL; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1908 | } |
| 1909 | events[n] = ev; |
| 1910 | ctrs[n] = event; |
| 1911 | cflags[n] = flags; |
| 1912 | if (check_excludes(ctrs, cflags, n, 1)) |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1913 | return -EINVAL; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1914 | |
| 1915 | cpuhw = &get_cpu_var(cpu_hw_events); |
| 1916 | err = power_check_constraints(cpuhw, events, cflags, n + 1); |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1917 | |
| 1918 | if (has_branch_stack(event)) { |
| 1919 | cpuhw->bhrb_filter = ppmu->bhrb_filter_map( |
| 1920 | event->attr.branch_sample_type); |
| 1921 | |
Jan Stancek | 68de886 | 2015-03-24 08:33:22 -0400 | [diff] [blame] | 1922 | if (cpuhw->bhrb_filter == -1) { |
| 1923 | put_cpu_var(cpu_hw_events); |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1924 | return -EOPNOTSUPP; |
Jan Stancek | 68de886 | 2015-03-24 08:33:22 -0400 | [diff] [blame] | 1925 | } |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 1926 | } |
| 1927 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1928 | put_cpu_var(cpu_hw_events); |
| 1929 | if (err) |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1930 | return -EINVAL; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1931 | |
| 1932 | event->hw.config = events[n]; |
| 1933 | event->hw.event_base = cflags[n]; |
| 1934 | event->hw.last_period = event->hw.sample_period; |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 1935 | local64_set(&event->hw.period_left, event->hw.last_period); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1936 | |
| 1937 | /* |
Michael Ellerman | 330a1eb | 2013-06-28 18:15:16 +1000 | [diff] [blame] | 1938 | * For EBB events we just context switch the PMC value, we don't do any |
| 1939 | * of the sample_period logic. We use hw.prev_count for this. |
| 1940 | */ |
| 1941 | if (is_ebb_event(event)) |
| 1942 | local64_set(&event->hw.prev_count, 0); |
| 1943 | |
| 1944 | /* |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1945 | * See if we need to reserve the PMU. |
| 1946 | * If no events are currently in use, then we have to take a |
| 1947 | * mutex to ensure that we don't race with another task doing |
| 1948 | * reserve_pmc_hardware or release_pmc_hardware. |
| 1949 | */ |
| 1950 | err = 0; |
| 1951 | if (!atomic_inc_not_zero(&num_events)) { |
| 1952 | mutex_lock(&pmc_reserve_mutex); |
| 1953 | if (atomic_read(&num_events) == 0 && |
| 1954 | reserve_pmc_hardware(perf_event_interrupt)) |
| 1955 | err = -EBUSY; |
| 1956 | else |
| 1957 | atomic_inc(&num_events); |
| 1958 | mutex_unlock(&pmc_reserve_mutex); |
| 1959 | } |
| 1960 | event->destroy = hw_perf_event_destroy; |
| 1961 | |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1962 | return err; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1963 | } |
| 1964 | |
Peter Zijlstra | 35edc2a | 2011-11-20 20:36:02 +0100 | [diff] [blame] | 1965 | static int power_pmu_event_idx(struct perf_event *event) |
| 1966 | { |
| 1967 | return event->hw.idx; |
| 1968 | } |
| 1969 | |
Sukadev Bhattiprolu | 1c53a27 | 2013-01-22 22:24:54 -0800 | [diff] [blame] | 1970 | ssize_t power_events_sysfs_show(struct device *dev, |
| 1971 | struct device_attribute *attr, char *page) |
| 1972 | { |
| 1973 | struct perf_pmu_events_attr *pmu_attr; |
| 1974 | |
| 1975 | pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); |
| 1976 | |
| 1977 | return sprintf(page, "event=0x%02llx\n", pmu_attr->id); |
| 1978 | } |
| 1979 | |
Anton Blanchard | e51df2c | 2014-08-20 08:55:18 +1000 | [diff] [blame] | 1980 | static struct pmu power_pmu = { |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1981 | .pmu_enable = power_pmu_enable, |
| 1982 | .pmu_disable = power_pmu_disable, |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1983 | .event_init = power_pmu_event_init, |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1984 | .add = power_pmu_add, |
| 1985 | .del = power_pmu_del, |
| 1986 | .start = power_pmu_start, |
| 1987 | .stop = power_pmu_stop, |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1988 | .read = power_pmu_read, |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1989 | .start_txn = power_pmu_start_txn, |
| 1990 | .cancel_txn = power_pmu_cancel_txn, |
| 1991 | .commit_txn = power_pmu_commit_txn, |
Peter Zijlstra | 35edc2a | 2011-11-20 20:36:02 +0100 | [diff] [blame] | 1992 | .event_idx = power_pmu_event_idx, |
Peter Zijlstra | acba3c7 | 2015-01-14 14:15:39 +0100 | [diff] [blame] | 1993 | .sched_task = power_pmu_sched_task, |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1994 | }; |
| 1995 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1996 | /* |
Ingo Molnar | 57c0c15 | 2009-09-21 12:20:38 +0200 | [diff] [blame] | 1997 | * A counter has overflowed; update its count and record |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1998 | * things if requested. Note that interrupts are hard-disabled |
| 1999 | * here so there is no possibility of being interrupted. |
| 2000 | */ |
| 2001 | static void record_and_restart(struct perf_event *event, unsigned long val, |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 2002 | struct pt_regs *regs) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2003 | { |
| 2004 | u64 period = event->hw.sample_period; |
| 2005 | s64 prev, delta, left; |
| 2006 | int record = 0; |
| 2007 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 2008 | if (event->hw.state & PERF_HES_STOPPED) { |
| 2009 | write_pmc(event->hw.idx, 0); |
| 2010 | return; |
| 2011 | } |
| 2012 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2013 | /* we don't have to worry about interrupts here */ |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 2014 | prev = local64_read(&event->hw.prev_count); |
Eric B Munson | 86c74ab | 2011-04-15 08:12:30 +0000 | [diff] [blame] | 2015 | delta = check_and_compute_delta(prev, val); |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 2016 | local64_add(delta, &event->count); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2017 | |
| 2018 | /* |
| 2019 | * See if the total period for this event has expired, |
| 2020 | * and update for the next period. |
| 2021 | */ |
| 2022 | val = 0; |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 2023 | left = local64_read(&event->hw.period_left) - delta; |
Michael Neuling | e13e895 | 2012-11-05 15:08:38 +0000 | [diff] [blame] | 2024 | if (delta == 0) |
| 2025 | left++; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2026 | if (period) { |
| 2027 | if (left <= 0) { |
| 2028 | left += period; |
| 2029 | if (left <= 0) |
| 2030 | left = period; |
sukadev@linux.vnet.ibm.com | e687883 | 2012-09-18 20:56:11 +0000 | [diff] [blame] | 2031 | record = siar_valid(regs); |
Anton Blanchard | 4bca770 | 2011-01-17 16:17:42 +1100 | [diff] [blame] | 2032 | event->hw.last_period = event->hw.sample_period; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2033 | } |
| 2034 | if (left < 0x80000000LL) |
| 2035 | val = 0x80000000LL - left; |
| 2036 | } |
| 2037 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 2038 | write_pmc(event->hw.idx, val); |
| 2039 | local64_set(&event->hw.prev_count, val); |
| 2040 | local64_set(&event->hw.period_left, left); |
| 2041 | perf_event_update_userpage(event); |
| 2042 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2043 | /* |
| 2044 | * Finally record data if requested. |
| 2045 | */ |
| 2046 | if (record) { |
Peter Zijlstra | dc1d628 | 2010-03-03 15:55:04 +0100 | [diff] [blame] | 2047 | struct perf_sample_data data; |
| 2048 | |
Robert Richter | fd0d000 | 2012-04-02 20:19:08 +0200 | [diff] [blame] | 2049 | perf_sample_data_init(&data, ~0ULL, event->hw.last_period); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2050 | |
Kan Liang | fc7ce9c | 2017-08-28 20:52:49 -0400 | [diff] [blame] | 2051 | if (event->attr.sample_type & |
| 2052 | (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR)) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2053 | perf_get_data_addr(regs, &data.addr); |
| 2054 | |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 2055 | if (event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK) { |
| 2056 | struct cpu_hw_events *cpuhw; |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 2057 | cpuhw = this_cpu_ptr(&cpu_hw_events); |
Anshuman Khandual | 3925f46 | 2013-04-22 19:42:44 +0000 | [diff] [blame] | 2058 | power_pmu_bhrb_read(cpuhw); |
| 2059 | data.br_stack = &cpuhw->bhrb_stack; |
| 2060 | } |
| 2061 | |
Madhavan Srinivasan | 79e96f8 | 2017-04-11 07:21:06 +0530 | [diff] [blame] | 2062 | if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC && |
| 2063 | ppmu->get_mem_data_src) |
| 2064 | ppmu->get_mem_data_src(&data.data_src, ppmu->flags, regs); |
| 2065 | |
Madhavan Srinivasan | 170a315 | 2017-04-11 07:21:07 +0530 | [diff] [blame] | 2066 | if (event->attr.sample_type & PERF_SAMPLE_WEIGHT && |
| 2067 | ppmu->get_mem_weight) |
| 2068 | ppmu->get_mem_weight(&data.weight); |
| 2069 | |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 2070 | if (perf_event_overflow(event, &data, regs)) |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 2071 | power_pmu_stop(event, 0); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2072 | } |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2073 | } |
| 2074 | |
| 2075 | /* |
| 2076 | * Called from generic code to get the misc flags (i.e. processor mode) |
| 2077 | * for an event_id. |
| 2078 | */ |
| 2079 | unsigned long perf_misc_flags(struct pt_regs *regs) |
| 2080 | { |
| 2081 | u32 flags = perf_get_misc_flags(regs); |
| 2082 | |
| 2083 | if (flags) |
| 2084 | return flags; |
| 2085 | return user_mode(regs) ? PERF_RECORD_MISC_USER : |
| 2086 | PERF_RECORD_MISC_KERNEL; |
| 2087 | } |
| 2088 | |
| 2089 | /* |
| 2090 | * Called from generic code to get the instruction pointer |
| 2091 | * for an event_id. |
| 2092 | */ |
| 2093 | unsigned long perf_instruction_pointer(struct pt_regs *regs) |
| 2094 | { |
Michael Ellerman | 3390405 | 2013-04-25 19:28:25 +0000 | [diff] [blame] | 2095 | bool use_siar = regs_use_siar(regs); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2096 | |
sukadev@linux.vnet.ibm.com | e687883 | 2012-09-18 20:56:11 +0000 | [diff] [blame] | 2097 | if (use_siar && siar_valid(regs)) |
Anton Blanchard | 75382aa | 2012-06-26 01:01:36 +0000 | [diff] [blame] | 2098 | return mfspr(SPRN_SIAR) + perf_ip_adjust(regs); |
sukadev@linux.vnet.ibm.com | e687883 | 2012-09-18 20:56:11 +0000 | [diff] [blame] | 2099 | else if (use_siar) |
| 2100 | return 0; // no valid instruction pointer |
Anton Blanchard | 75382aa | 2012-06-26 01:01:36 +0000 | [diff] [blame] | 2101 | else |
Benjamin Herrenschmidt | 1ce447b | 2012-03-26 20:47:34 +0000 | [diff] [blame] | 2102 | return regs->nip; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2103 | } |
| 2104 | |
Michael Neuling | bc09c21 | 2012-11-05 15:53:54 +0000 | [diff] [blame] | 2105 | static bool pmc_overflow_power7(unsigned long val) |
Anton Blanchard | 0837e32 | 2011-03-09 14:38:42 +1100 | [diff] [blame] | 2106 | { |
Anton Blanchard | 0837e32 | 2011-03-09 14:38:42 +1100 | [diff] [blame] | 2107 | /* |
| 2108 | * Events on POWER7 can roll back if a speculative event doesn't |
| 2109 | * eventually complete. Unfortunately in some rare cases they will |
| 2110 | * raise a performance monitor exception. We need to catch this to |
| 2111 | * ensure we reset the PMC. In all cases the PMC will be 256 or less |
| 2112 | * cycles from overflow. |
| 2113 | * |
| 2114 | * We only do this if the first pass fails to find any overflowing |
| 2115 | * PMCs because a user might set a period of less than 256 and we |
| 2116 | * don't want to mistakenly reset them. |
| 2117 | */ |
Michael Neuling | bc09c21 | 2012-11-05 15:53:54 +0000 | [diff] [blame] | 2118 | if ((0x80000000 - val) <= 256) |
| 2119 | return true; |
| 2120 | |
| 2121 | return false; |
| 2122 | } |
| 2123 | |
| 2124 | static bool pmc_overflow(unsigned long val) |
| 2125 | { |
| 2126 | if ((int)val < 0) |
Anton Blanchard | 0837e32 | 2011-03-09 14:38:42 +1100 | [diff] [blame] | 2127 | return true; |
| 2128 | |
| 2129 | return false; |
| 2130 | } |
| 2131 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2132 | /* |
| 2133 | * Performance monitor interrupt stuff |
| 2134 | */ |
| 2135 | static void perf_event_interrupt(struct pt_regs *regs) |
| 2136 | { |
Michael Neuling | bc09c21 | 2012-11-05 15:53:54 +0000 | [diff] [blame] | 2137 | int i, j; |
Christoph Lameter | 69111ba | 2014-10-21 15:23:25 -0500 | [diff] [blame] | 2138 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2139 | struct perf_event *event; |
Michael Neuling | bc09c21 | 2012-11-05 15:53:54 +0000 | [diff] [blame] | 2140 | unsigned long val[8]; |
| 2141 | int found, active; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2142 | int nmi; |
| 2143 | |
| 2144 | if (cpuhw->n_limited) |
Paul Mackerras | a8f90e9 | 2009-09-22 09:48:08 +1000 | [diff] [blame] | 2145 | freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5), |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2146 | mfspr(SPRN_PMC6)); |
| 2147 | |
| 2148 | perf_read_regs(regs); |
| 2149 | |
| 2150 | nmi = perf_intr_is_nmi(regs); |
| 2151 | if (nmi) |
| 2152 | nmi_enter(); |
| 2153 | else |
| 2154 | irq_enter(); |
| 2155 | |
Michael Neuling | bc09c21 | 2012-11-05 15:53:54 +0000 | [diff] [blame] | 2156 | /* Read all the PMCs since we'll need them a bunch of times */ |
| 2157 | for (i = 0; i < ppmu->n_counter; ++i) |
| 2158 | val[i] = read_pmc(i + 1); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2159 | |
Michael Neuling | bc09c21 | 2012-11-05 15:53:54 +0000 | [diff] [blame] | 2160 | /* Try to find what caused the IRQ */ |
| 2161 | found = 0; |
| 2162 | for (i = 0; i < ppmu->n_counter; ++i) { |
| 2163 | if (!pmc_overflow(val[i])) |
| 2164 | continue; |
| 2165 | if (is_limited_pmc(i + 1)) |
| 2166 | continue; /* these won't generate IRQs */ |
| 2167 | /* |
| 2168 | * We've found one that's overflowed. For active |
| 2169 | * counters we need to log this. For inactive |
| 2170 | * counters, we need to reset it anyway |
| 2171 | */ |
| 2172 | found = 1; |
| 2173 | active = 0; |
| 2174 | for (j = 0; j < cpuhw->n_events; ++j) { |
| 2175 | event = cpuhw->event[j]; |
| 2176 | if (event->hw.idx == (i + 1)) { |
| 2177 | active = 1; |
| 2178 | record_and_restart(event, val[i], regs); |
| 2179 | break; |
| 2180 | } |
| 2181 | } |
| 2182 | if (!active) |
| 2183 | /* reset non active counters that have overflowed */ |
| 2184 | write_pmc(i + 1, 0); |
| 2185 | } |
| 2186 | if (!found && pvr_version_is(PVR_POWER7)) { |
| 2187 | /* check active counters for special buggy p7 overflow */ |
| 2188 | for (i = 0; i < cpuhw->n_events; ++i) { |
| 2189 | event = cpuhw->event[i]; |
| 2190 | if (!event->hw.idx || is_limited_pmc(event->hw.idx)) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2191 | continue; |
Michael Neuling | bc09c21 | 2012-11-05 15:53:54 +0000 | [diff] [blame] | 2192 | if (pmc_overflow_power7(val[event->hw.idx - 1])) { |
| 2193 | /* event has overflowed in a buggy way*/ |
| 2194 | found = 1; |
| 2195 | record_and_restart(event, |
| 2196 | val[event->hw.idx - 1], |
| 2197 | regs); |
| 2198 | } |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2199 | } |
| 2200 | } |
Michael Ellerman | 6772faa | 2013-06-05 17:58:20 +0000 | [diff] [blame] | 2201 | if (!found && !nmi && printk_ratelimit()) |
Michael Neuling | bc09c21 | 2012-11-05 15:53:54 +0000 | [diff] [blame] | 2202 | printk(KERN_WARNING "Can't find PMC that caused IRQ\n"); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2203 | |
| 2204 | /* |
| 2205 | * Reset MMCR0 to its normal value. This will set PMXE and |
Ingo Molnar | 57c0c15 | 2009-09-21 12:20:38 +0200 | [diff] [blame] | 2206 | * clear FC (freeze counters) and PMAO (perf mon alert occurred) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2207 | * and thus allow interrupts to occur again. |
| 2208 | * XXX might want to use MSR.PM to keep the events frozen until |
| 2209 | * we get back out of this interrupt. |
| 2210 | */ |
| 2211 | write_mmcr0(cpuhw, cpuhw->mmcr[0]); |
| 2212 | |
| 2213 | if (nmi) |
| 2214 | nmi_exit(); |
| 2215 | else |
| 2216 | irq_exit(); |
| 2217 | } |
| 2218 | |
Daniel Axtens | 7c98bd7 | 2016-09-06 15:32:40 +1000 | [diff] [blame] | 2219 | static int power_pmu_prepare_cpu(unsigned int cpu) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2220 | { |
| 2221 | struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu); |
| 2222 | |
Thomas Gleixner | 57ecde4 | 2016-07-13 17:16:20 +0000 | [diff] [blame] | 2223 | if (ppmu) { |
| 2224 | memset(cpuhw, 0, sizeof(*cpuhw)); |
| 2225 | cpuhw->mmcr[0] = MMCR0_FC; |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 2226 | } |
Thomas Gleixner | 57ecde4 | 2016-07-13 17:16:20 +0000 | [diff] [blame] | 2227 | return 0; |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 2228 | } |
| 2229 | |
Paul Gortmaker | 061d19f | 2013-06-24 15:30:09 -0400 | [diff] [blame] | 2230 | int register_power_pmu(struct power_pmu *pmu) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2231 | { |
| 2232 | if (ppmu) |
| 2233 | return -EBUSY; /* something's already registered */ |
| 2234 | |
| 2235 | ppmu = pmu; |
| 2236 | pr_info("%s performance monitor hardware support registered\n", |
| 2237 | pmu->name); |
| 2238 | |
Sukadev Bhattiprolu | 1c53a27 | 2013-01-22 22:24:54 -0800 | [diff] [blame] | 2239 | power_pmu.attr_groups = ppmu->attr_groups; |
| 2240 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2241 | #ifdef MSR_HV |
| 2242 | /* |
| 2243 | * Use FCHV to ignore kernel events if MSR.HV is set. |
| 2244 | */ |
| 2245 | if (mfmsr() & MSR_HV) |
| 2246 | freeze_events_kernel = MMCR0_FCHV; |
| 2247 | #endif /* CONFIG_PPC64 */ |
| 2248 | |
Peter Zijlstra | 2e80a82 | 2010-11-17 23:17:36 +0100 | [diff] [blame] | 2249 | perf_pmu_register(&power_pmu, "cpu", PERF_TYPE_RAW); |
Thomas Gleixner | 73c1b41 | 2016-12-21 20:19:54 +0100 | [diff] [blame] | 2250 | cpuhp_setup_state(CPUHP_PERF_POWER, "perf/powerpc:prepare", |
Thomas Gleixner | 57ecde4 | 2016-07-13 17:16:20 +0000 | [diff] [blame] | 2251 | power_pmu_prepare_cpu, NULL); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2252 | return 0; |
| 2253 | } |