| #undef DEBUG |
| |
| /* |
| * ARM performance counter support. |
| * |
| * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles |
| * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com> |
| * |
| * This code is based on the sparc64 perf event code, which is in turn based |
| * on the x86 code. |
| */ |
| #define pr_fmt(fmt) "hw perfevents: " fmt |
| |
| #include <linux/kernel.h> |
| #include <linux/platform_device.h> |
| #include <linux/pm_runtime.h> |
| #include <linux/irq.h> |
| #include <linux/irqdesc.h> |
| |
| #include <asm/irq_regs.h> |
| #include <asm/pmu.h> |
| |
| static int |
| armpmu_map_cache_event(const unsigned (*cache_map) |
| [PERF_COUNT_HW_CACHE_MAX] |
| [PERF_COUNT_HW_CACHE_OP_MAX] |
| [PERF_COUNT_HW_CACHE_RESULT_MAX], |
| u64 config) |
| { |
| unsigned int cache_type, cache_op, cache_result, ret; |
| |
| cache_type = (config >> 0) & 0xff; |
| if (cache_type >= PERF_COUNT_HW_CACHE_MAX) |
| return -EINVAL; |
| |
| cache_op = (config >> 8) & 0xff; |
| if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) |
| return -EINVAL; |
| |
| cache_result = (config >> 16) & 0xff; |
| if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
| return -EINVAL; |
| |
| ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; |
| |
| if (ret == CACHE_OP_UNSUPPORTED) |
| return -ENOENT; |
| |
| return ret; |
| } |
| |
| static int |
| armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) |
| { |
| int mapping; |
| |
| if (config >= PERF_COUNT_HW_MAX) |
| return -EINVAL; |
| |
| mapping = (*event_map)[config]; |
| return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; |
| } |
| |
| static int |
| armpmu_map_raw_event(u32 raw_event_mask, u64 config) |
| { |
| return (int)(config & raw_event_mask); |
| } |
| |
| int |
| armpmu_map_event(struct perf_event *event, |
| const unsigned (*event_map)[PERF_COUNT_HW_MAX], |
| const unsigned (*cache_map) |
| [PERF_COUNT_HW_CACHE_MAX] |
| [PERF_COUNT_HW_CACHE_OP_MAX] |
| [PERF_COUNT_HW_CACHE_RESULT_MAX], |
| u32 raw_event_mask) |
| { |
| u64 config = event->attr.config; |
| int type = event->attr.type; |
| |
| if (type == event->pmu->type) |
| return armpmu_map_raw_event(raw_event_mask, config); |
| |
| switch (type) { |
| case PERF_TYPE_HARDWARE: |
| return armpmu_map_hw_event(event_map, config); |
| case PERF_TYPE_HW_CACHE: |
| return armpmu_map_cache_event(cache_map, config); |
| case PERF_TYPE_RAW: |
| return armpmu_map_raw_event(raw_event_mask, config); |
| } |
| |
| return -ENOENT; |
| } |
| |
| int armpmu_event_set_period(struct perf_event *event) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| struct hw_perf_event *hwc = &event->hw; |
| s64 left = local64_read(&hwc->period_left); |
| s64 period = hwc->sample_period; |
| int ret = 0; |
| |
| if (unlikely(left <= -period)) { |
| left = period; |
| local64_set(&hwc->period_left, left); |
| hwc->last_period = period; |
| ret = 1; |
| } |
| |
| if (unlikely(left <= 0)) { |
| left += period; |
| local64_set(&hwc->period_left, left); |
| hwc->last_period = period; |
| ret = 1; |
| } |
| |
| if (left > (s64)armpmu->max_period) |
| left = armpmu->max_period; |
| |
| local64_set(&hwc->prev_count, (u64)-left); |
| |
| armpmu->write_counter(event, (u64)(-left) & 0xffffffff); |
| |
| perf_event_update_userpage(event); |
| |
| return ret; |
| } |
| |
| u64 armpmu_event_update(struct perf_event *event) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| struct hw_perf_event *hwc = &event->hw; |
| u64 delta, prev_raw_count, new_raw_count; |
| |
| again: |
| prev_raw_count = local64_read(&hwc->prev_count); |
| new_raw_count = armpmu->read_counter(event); |
| |
| if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, |
| new_raw_count) != prev_raw_count) |
| goto again; |
| |
| delta = (new_raw_count - prev_raw_count) & armpmu->max_period; |
| |
| local64_add(delta, &event->count); |
| local64_sub(delta, &hwc->period_left); |
| |
| return new_raw_count; |
| } |
| |
| static void |
| armpmu_read(struct perf_event *event) |
| { |
| armpmu_event_update(event); |
| } |
| |
| static void |
| armpmu_stop(struct perf_event *event, int flags) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| struct hw_perf_event *hwc = &event->hw; |
| |
| /* |
| * ARM pmu always has to update the counter, so ignore |
| * PERF_EF_UPDATE, see comments in armpmu_start(). |
| */ |
| if (!(hwc->state & PERF_HES_STOPPED)) { |
| armpmu->disable(event); |
| armpmu_event_update(event); |
| hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| } |
| } |
| |
| static void armpmu_start(struct perf_event *event, int flags) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| struct hw_perf_event *hwc = &event->hw; |
| |
| /* |
| * ARM pmu always has to reprogram the period, so ignore |
| * PERF_EF_RELOAD, see the comment below. |
| */ |
| if (flags & PERF_EF_RELOAD) |
| WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
| |
| hwc->state = 0; |
| /* |
| * Set the period again. Some counters can't be stopped, so when we |
| * were stopped we simply disabled the IRQ source and the counter |
| * may have been left counting. If we don't do this step then we may |
| * get an interrupt too soon or *way* too late if the overflow has |
| * happened since disabling. |
| */ |
| armpmu_event_set_period(event); |
| armpmu->enable(event); |
| } |
| |
| static void |
| armpmu_del(struct perf_event *event, int flags) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
| struct hw_perf_event *hwc = &event->hw; |
| int idx = hwc->idx; |
| |
| armpmu_stop(event, PERF_EF_UPDATE); |
| hw_events->events[idx] = NULL; |
| clear_bit(idx, hw_events->used_mask); |
| if (armpmu->clear_event_idx) |
| armpmu->clear_event_idx(hw_events, event); |
| |
| perf_event_update_userpage(event); |
| } |
| |
| static int |
| armpmu_add(struct perf_event *event, int flags) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
| struct hw_perf_event *hwc = &event->hw; |
| int idx; |
| int err = 0; |
| |
| perf_pmu_disable(event->pmu); |
| |
| /* If we don't have a space for the counter then finish early. */ |
| idx = armpmu->get_event_idx(hw_events, event); |
| if (idx < 0) { |
| err = idx; |
| goto out; |
| } |
| |
| /* |
| * If there is an event in the counter we are going to use then make |
| * sure it is disabled. |
| */ |
| event->hw.idx = idx; |
| armpmu->disable(event); |
| hw_events->events[idx] = event; |
| |
| hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| if (flags & PERF_EF_START) |
| armpmu_start(event, PERF_EF_RELOAD); |
| |
| /* Propagate our changes to the userspace mapping. */ |
| perf_event_update_userpage(event); |
| |
| out: |
| perf_pmu_enable(event->pmu); |
| return err; |
| } |
| |
| static int |
| validate_event(struct pmu_hw_events *hw_events, |
| struct perf_event *event) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| |
| if (is_software_event(event)) |
| return 1; |
| |
| if (event->state < PERF_EVENT_STATE_OFF) |
| return 1; |
| |
| if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) |
| return 1; |
| |
| return armpmu->get_event_idx(hw_events, event) >= 0; |
| } |
| |
| static int |
| validate_group(struct perf_event *event) |
| { |
| struct perf_event *sibling, *leader = event->group_leader; |
| struct pmu_hw_events fake_pmu; |
| |
| /* |
| * Initialise the fake PMU. We only need to populate the |
| * used_mask for the purposes of validation. |
| */ |
| memset(&fake_pmu.used_mask, 0, sizeof(fake_pmu.used_mask)); |
| |
| if (!validate_event(&fake_pmu, leader)) |
| return -EINVAL; |
| |
| list_for_each_entry(sibling, &leader->sibling_list, group_entry) { |
| if (!validate_event(&fake_pmu, sibling)) |
| return -EINVAL; |
| } |
| |
| if (!validate_event(&fake_pmu, event)) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) |
| { |
| struct arm_pmu *armpmu; |
| struct platform_device *plat_device; |
| struct arm_pmu_platdata *plat; |
| int ret; |
| u64 start_clock, finish_clock; |
| |
| /* |
| * we request the IRQ with a (possibly percpu) struct arm_pmu**, but |
| * the handlers expect a struct arm_pmu*. The percpu_irq framework will |
| * do any necessary shifting, we just need to perform the first |
| * dereference. |
| */ |
| armpmu = *(void **)dev; |
| plat_device = armpmu->plat_device; |
| plat = dev_get_platdata(&plat_device->dev); |
| |
| start_clock = sched_clock(); |
| if (plat && plat->handle_irq) |
| ret = plat->handle_irq(irq, armpmu, armpmu->handle_irq); |
| else |
| ret = armpmu->handle_irq(irq, armpmu); |
| finish_clock = sched_clock(); |
| |
| perf_sample_event_took(finish_clock - start_clock); |
| return ret; |
| } |
| |
| static void |
| armpmu_release_hardware(struct arm_pmu *armpmu) |
| { |
| armpmu->free_irq(armpmu); |
| pm_runtime_put_sync(&armpmu->plat_device->dev); |
| } |
| |
| static int |
| armpmu_reserve_hardware(struct arm_pmu *armpmu) |
| { |
| int err; |
| struct platform_device *pmu_device = armpmu->plat_device; |
| |
| if (!pmu_device) |
| return -ENODEV; |
| |
| pm_runtime_get_sync(&pmu_device->dev); |
| err = armpmu->request_irq(armpmu, armpmu_dispatch_irq); |
| if (err) { |
| armpmu_release_hardware(armpmu); |
| return err; |
| } |
| |
| return 0; |
| } |
| |
| static void |
| hw_perf_event_destroy(struct perf_event *event) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| atomic_t *active_events = &armpmu->active_events; |
| struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex; |
| |
| if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) { |
| armpmu_release_hardware(armpmu); |
| mutex_unlock(pmu_reserve_mutex); |
| } |
| } |
| |
| static int |
| event_requires_mode_exclusion(struct perf_event_attr *attr) |
| { |
| return attr->exclude_idle || attr->exclude_user || |
| attr->exclude_kernel || attr->exclude_hv; |
| } |
| |
| static int |
| __hw_perf_event_init(struct perf_event *event) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| struct hw_perf_event *hwc = &event->hw; |
| int mapping; |
| |
| mapping = armpmu->map_event(event); |
| |
| if (mapping < 0) { |
| pr_debug("event %x:%llx not supported\n", event->attr.type, |
| event->attr.config); |
| return mapping; |
| } |
| |
| /* |
| * We don't assign an index until we actually place the event onto |
| * hardware. Use -1 to signify that we haven't decided where to put it |
| * yet. For SMP systems, each core has it's own PMU so we can't do any |
| * clever allocation or constraints checking at this point. |
| */ |
| hwc->idx = -1; |
| hwc->config_base = 0; |
| hwc->config = 0; |
| hwc->event_base = 0; |
| |
| /* |
| * Check whether we need to exclude the counter from certain modes. |
| */ |
| if ((!armpmu->set_event_filter || |
| armpmu->set_event_filter(hwc, &event->attr)) && |
| event_requires_mode_exclusion(&event->attr)) { |
| pr_debug("ARM performance counters do not support " |
| "mode exclusion\n"); |
| return -EOPNOTSUPP; |
| } |
| |
| /* |
| * Store the event encoding into the config_base field. |
| */ |
| hwc->config_base |= (unsigned long)mapping; |
| |
| if (!is_sampling_event(event)) { |
| /* |
| * For non-sampling runs, limit the sample_period to half |
| * of the counter width. That way, the new counter value |
| * is far less likely to overtake the previous one unless |
| * you have some serious IRQ latency issues. |
| */ |
| hwc->sample_period = armpmu->max_period >> 1; |
| hwc->last_period = hwc->sample_period; |
| local64_set(&hwc->period_left, hwc->sample_period); |
| } |
| |
| if (event->group_leader != event) { |
| if (validate_group(event) != 0) |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| |
| static int armpmu_event_init(struct perf_event *event) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| int err = 0; |
| atomic_t *active_events = &armpmu->active_events; |
| |
| /* does not support taken branch sampling */ |
| if (has_branch_stack(event)) |
| return -EOPNOTSUPP; |
| |
| if (armpmu->map_event(event) == -ENOENT) |
| return -ENOENT; |
| |
| event->destroy = hw_perf_event_destroy; |
| |
| if (!atomic_inc_not_zero(active_events)) { |
| mutex_lock(&armpmu->reserve_mutex); |
| if (atomic_read(active_events) == 0) |
| err = armpmu_reserve_hardware(armpmu); |
| |
| if (!err) |
| atomic_inc(active_events); |
| mutex_unlock(&armpmu->reserve_mutex); |
| } |
| |
| if (err) |
| return err; |
| |
| err = __hw_perf_event_init(event); |
| if (err) |
| hw_perf_event_destroy(event); |
| |
| return err; |
| } |
| |
| static void armpmu_enable(struct pmu *pmu) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(pmu); |
| struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
| int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events); |
| |
| if (enabled) |
| armpmu->start(armpmu); |
| } |
| |
| static void armpmu_disable(struct pmu *pmu) |
| { |
| struct arm_pmu *armpmu = to_arm_pmu(pmu); |
| armpmu->stop(armpmu); |
| } |
| |
| #ifdef CONFIG_PM_RUNTIME |
| static int armpmu_runtime_resume(struct device *dev) |
| { |
| struct arm_pmu_platdata *plat = dev_get_platdata(dev); |
| |
| if (plat && plat->runtime_resume) |
| return plat->runtime_resume(dev); |
| |
| return 0; |
| } |
| |
| static int armpmu_runtime_suspend(struct device *dev) |
| { |
| struct arm_pmu_platdata *plat = dev_get_platdata(dev); |
| |
| if (plat && plat->runtime_suspend) |
| return plat->runtime_suspend(dev); |
| |
| return 0; |
| } |
| #endif |
| |
| const struct dev_pm_ops armpmu_dev_pm_ops = { |
| SET_RUNTIME_PM_OPS(armpmu_runtime_suspend, armpmu_runtime_resume, NULL) |
| }; |
| |
| static void armpmu_init(struct arm_pmu *armpmu) |
| { |
| atomic_set(&armpmu->active_events, 0); |
| mutex_init(&armpmu->reserve_mutex); |
| |
| armpmu->pmu = (struct pmu) { |
| .pmu_enable = armpmu_enable, |
| .pmu_disable = armpmu_disable, |
| .event_init = armpmu_event_init, |
| .add = armpmu_add, |
| .del = armpmu_del, |
| .start = armpmu_start, |
| .stop = armpmu_stop, |
| .read = armpmu_read, |
| }; |
| } |
| |
| int armpmu_register(struct arm_pmu *armpmu, int type) |
| { |
| armpmu_init(armpmu); |
| pm_runtime_enable(&armpmu->plat_device->dev); |
| pr_info("enabled with %s PMU driver, %d counters available\n", |
| armpmu->name, armpmu->num_events); |
| return perf_pmu_register(&armpmu->pmu, armpmu->name, type); |
| } |
| |