blob: f45621aba7b5a9083d00b6696b0ed49db5d70bb2 [file] [log] [blame] [edit]
#include "x86/msr.h"
#include "x86/processor.h"
#include "x86/apic-defs.h"
#include "x86/apic.h"
#include "x86/desc.h"
#include "x86/isr.h"
#include "alloc.h"
#include "libcflat.h"
#include <stdint.h>
#define FIXED_CNT_INDEX 32
#define PC_VECTOR 32
#define EVNSEL_EVENT_SHIFT 0
#define EVNTSEL_UMASK_SHIFT 8
#define EVNTSEL_USR_SHIFT 16
#define EVNTSEL_OS_SHIFT 17
#define EVNTSEL_EDGE_SHIFT 18
#define EVNTSEL_PC_SHIFT 19
#define EVNTSEL_INT_SHIFT 20
#define EVNTSEL_EN_SHIF 22
#define EVNTSEL_INV_SHIF 23
#define EVNTSEL_CMASK_SHIFT 24
#define EVNTSEL_EN (1 << EVNTSEL_EN_SHIF)
#define EVNTSEL_USR (1 << EVNTSEL_USR_SHIFT)
#define EVNTSEL_OS (1 << EVNTSEL_OS_SHIFT)
#define EVNTSEL_PC (1 << EVNTSEL_PC_SHIFT)
#define EVNTSEL_INT (1 << EVNTSEL_INT_SHIFT)
#define EVNTSEL_INV (1 << EVNTSEL_INV_SHIF)
#define N 1000000
typedef struct {
uint32_t ctr;
uint32_t config;
uint64_t count;
int idx;
} pmu_counter_t;
union cpuid10_eax {
struct {
unsigned int version_id:8;
unsigned int num_counters:8;
unsigned int bit_width:8;
unsigned int mask_length:8;
} split;
unsigned int full;
} eax;
union cpuid10_ebx {
struct {
unsigned int no_unhalted_core_cycles:1;
unsigned int no_instructions_retired:1;
unsigned int no_unhalted_reference_cycles:1;
unsigned int no_llc_reference:1;
unsigned int no_llc_misses:1;
unsigned int no_branch_instruction_retired:1;
unsigned int no_branch_misses_retired:1;
} split;
unsigned int full;
} ebx;
union cpuid10_edx {
struct {
unsigned int num_counters_fixed:5;
unsigned int bit_width_fixed:8;
unsigned int reserved:19;
} split;
unsigned int full;
} edx;
struct pmu_event {
const char *name;
uint32_t unit_sel;
int min;
int max;
} gp_events[] = {
{"core cycles", 0x003c, 1*N, 50*N},
{"instructions", 0x00c0, 10*N, 10.2*N},
{"ref cycles", 0x013c, 0.1*N, 30*N},
{"llc refference", 0x4f2e, 1, 2*N},
{"llc misses", 0x412e, 1, 1*N},
{"branches", 0x00c4, 1*N, 1.1*N},
{"branch misses", 0x00c5, 0, 0.1*N},
}, fixed_events[] = {
{"fixed 1", MSR_CORE_PERF_FIXED_CTR0, 10*N, 10.2*N},
{"fixed 2", MSR_CORE_PERF_FIXED_CTR0 + 1, 1*N, 30*N},
{"fixed 3", MSR_CORE_PERF_FIXED_CTR0 + 2, 0.1*N, 30*N}
};
static int num_counters;
char *buf;
static inline void loop(void)
{
unsigned long tmp, tmp2, tmp3;
asm volatile("1: mov (%1), %2; add $64, %1; nop; nop; nop; nop; nop; nop; nop; loop 1b"
: "=c"(tmp), "=r"(tmp2), "=r"(tmp3): "0"(N), "1"(buf));
}
volatile uint64_t irq_received;
static void cnt_overflow(isr_regs_t *regs)
{
irq_received++;
apic_write(APIC_EOI, 0);
}
static bool check_irq(void)
{
int i;
irq_received = 0;
irq_enable();
for (i = 0; i < 100000 && !irq_received; i++)
asm volatile("pause");
irq_disable();
return irq_received;
}
static bool is_gp(pmu_counter_t *evt)
{
return evt->ctr < MSR_CORE_PERF_FIXED_CTR0;
}
static int event_to_global_idx(pmu_counter_t *cnt)
{
return cnt->ctr - (is_gp(cnt) ? MSR_IA32_PERFCTR0 :
(MSR_CORE_PERF_FIXED_CTR0 - FIXED_CNT_INDEX));
}
static struct pmu_event* get_counter_event(pmu_counter_t *cnt)
{
if (is_gp(cnt)) {
int i;
for (i = 0; i < sizeof(gp_events)/sizeof(gp_events[0]); i++)
if (gp_events[i].unit_sel == (cnt->config & 0xffff))
return &gp_events[i];
} else
return &fixed_events[cnt->ctr - MSR_CORE_PERF_FIXED_CTR0];
return (void*)0;
}
static void global_enable(pmu_counter_t *cnt)
{
cnt->idx = event_to_global_idx(cnt);
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, rdmsr(MSR_CORE_PERF_GLOBAL_CTRL) |
(1ull << cnt->idx));
}
static void global_disable(pmu_counter_t *cnt)
{
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, rdmsr(MSR_CORE_PERF_GLOBAL_CTRL) &
~(1ull << cnt->idx));
}
static void start_event(pmu_counter_t *evt)
{
wrmsr(evt->ctr, evt->count);
if (is_gp(evt))
wrmsr(MSR_P6_EVNTSEL0 + event_to_global_idx(evt),
evt->config | EVNTSEL_EN);
else {
uint32_t ctrl = rdmsr(MSR_CORE_PERF_FIXED_CTR_CTRL);
int shift = (evt->ctr - MSR_CORE_PERF_FIXED_CTR0) * 4;
uint32_t usrospmi = 0;
if (evt->config & EVNTSEL_OS)
usrospmi |= (1 << 0);
if (evt->config & EVNTSEL_USR)
usrospmi |= (1 << 1);
if (evt->config & EVNTSEL_INT)
usrospmi |= (1 << 3); // PMI on overflow
ctrl = (ctrl & ~(0xf << shift)) | (usrospmi << shift);
wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, ctrl);
}
global_enable(evt);
apic_write(APIC_LVTPC, PC_VECTOR);
}
static void stop_event(pmu_counter_t *evt)
{
global_disable(evt);
if (is_gp(evt))
wrmsr(MSR_P6_EVNTSEL0 + event_to_global_idx(evt),
evt->config & ~EVNTSEL_EN);
else {
uint32_t ctrl = rdmsr(MSR_CORE_PERF_FIXED_CTR_CTRL);
int shift = (evt->ctr - MSR_CORE_PERF_FIXED_CTR0) * 4;
wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, ctrl & ~(0xf << shift));
}
evt->count = rdmsr(evt->ctr);
}
static void measure(pmu_counter_t *evt, int count)
{
int i;
for (i = 0; i < count; i++)
start_event(&evt[i]);
loop();
for (i = 0; i < count; i++)
stop_event(&evt[i]);
}
static bool verify_event(uint64_t count, struct pmu_event *e)
{
// printf("%lld >= %lld <= %lld\n", e->min, count, e->max);
return count >= e->min && count <= e->max;
}
static bool verify_counter(pmu_counter_t *cnt)
{
return verify_event(cnt->count, get_counter_event(cnt));
}
static void check_gp_counter(struct pmu_event *evt)
{
pmu_counter_t cnt = {
.ctr = MSR_IA32_PERFCTR0,
.config = EVNTSEL_OS | EVNTSEL_USR | evt->unit_sel,
};
int i;
for (i = 0; i < num_counters; i++, cnt.ctr++) {
cnt.count = 0;
measure(&cnt, 1);
report(verify_event(cnt.count, evt), "%s-%d", evt->name, i);
}
}
static void check_gp_counters(void)
{
int i;
for (i = 0; i < sizeof(gp_events)/sizeof(gp_events[0]); i++)
if (!(ebx.full & (1 << i)))
check_gp_counter(&gp_events[i]);
else
printf("GP event '%s' is disabled\n",
gp_events[i].name);
}
static void check_fixed_counters(void)
{
pmu_counter_t cnt = {
.config = EVNTSEL_OS | EVNTSEL_USR,
};
int i;
for (i = 0; i < edx.split.num_counters_fixed; i++) {
cnt.count = 0;
cnt.ctr = fixed_events[i].unit_sel;
measure(&cnt, 1);
report(verify_event(cnt.count, &fixed_events[i]), "fixed-%d",
i);
}
}
static void check_counters_many(void)
{
pmu_counter_t cnt[10];
int i, n;
for (i = 0, n = 0; n < num_counters; i++) {
if (ebx.full & (1 << i))
continue;
cnt[n].count = 0;
cnt[n].ctr = MSR_IA32_PERFCTR0 + n;
cnt[n].config = EVNTSEL_OS | EVNTSEL_USR |
gp_events[i % ARRAY_SIZE(gp_events)].unit_sel;
n++;
}
for (i = 0; i < edx.split.num_counters_fixed; i++) {
cnt[n].count = 0;
cnt[n].ctr = fixed_events[i].unit_sel;
cnt[n].config = EVNTSEL_OS | EVNTSEL_USR;
n++;
}
measure(cnt, n);
for (i = 0; i < n; i++)
if (!verify_counter(&cnt[i]))
break;
report(i == n, "all counters");
}
static void check_counter_overflow(void)
{
uint64_t count;
int i;
pmu_counter_t cnt = {
.ctr = MSR_IA32_PERFCTR0,
.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[1].unit_sel /* instructions */,
.count = 0,
};
measure(&cnt, 1);
count = cnt.count;
/* clear status before test */
wrmsr(MSR_CORE_PERF_GLOBAL_OVF_CTRL, rdmsr(MSR_CORE_PERF_GLOBAL_STATUS));
report_prefix_push("overflow");
for (i = 0; i < num_counters + 1; i++, cnt.ctr++) {
uint64_t status;
int idx;
cnt.count = 1 - count;
if (i == num_counters) {
cnt.ctr = fixed_events[0].unit_sel;
cnt.count &= (1ul << edx.split.bit_width_fixed) - 1;
}
if (i % 2)
cnt.config |= EVNTSEL_INT;
else
cnt.config &= ~EVNTSEL_INT;
idx = event_to_global_idx(&cnt);
measure(&cnt, 1);
report(cnt.count == 1, "cntr-%d", i);
status = rdmsr(MSR_CORE_PERF_GLOBAL_STATUS);
report(status & (1ull << idx), "status-%d", i);
wrmsr(MSR_CORE_PERF_GLOBAL_OVF_CTRL, status);
status = rdmsr(MSR_CORE_PERF_GLOBAL_STATUS);
report(!(status & (1ull << idx)), "status clear-%d", i);
report(check_irq() == (i % 2), "irq-%d", i);
}
report_prefix_pop();
}
static void check_gp_counter_cmask(void)
{
pmu_counter_t cnt = {
.ctr = MSR_IA32_PERFCTR0,
.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[1].unit_sel /* instructions */,
.count = 0,
};
cnt.config |= (0x2 << EVNTSEL_CMASK_SHIFT);
measure(&cnt, 1);
report(cnt.count < gp_events[1].min, "cmask");
}
static void do_rdpmc_fast(void *ptr)
{
pmu_counter_t *cnt = ptr;
uint32_t idx = (uint32_t)cnt->idx | (1u << 31);
if (!is_gp(cnt))
idx |= 1 << 30;
cnt->count = rdpmc(idx);
}
static void check_rdpmc(void)
{
uint64_t val = 0x1f3456789ull;
bool exc;
int i;
report_prefix_push("rdpmc");
for (i = 0; i < num_counters; i++) {
uint64_t x;
pmu_counter_t cnt = {
.ctr = MSR_IA32_PERFCTR0 + i,
.idx = i
};
/*
* Only the low 32 bits are writable, and the value is
* sign-extended.
*/
x = (uint64_t)(int64_t)(int32_t)val;
/* Mask according to the number of supported bits */
x &= (1ull << eax.split.bit_width) - 1;
wrmsr(MSR_IA32_PERFCTR0 + i, val);
report(rdpmc(i) == x, "cntr-%d", i);
exc = test_for_exception(GP_VECTOR, do_rdpmc_fast, &cnt);
if (exc)
report_skip("fast-%d", i);
else
report(cnt.count == (u32)val, "fast-%d", i);
}
for (i = 0; i < edx.split.num_counters_fixed; i++) {
uint64_t x = val & ((1ull << edx.split.bit_width_fixed) - 1);
pmu_counter_t cnt = {
.ctr = MSR_CORE_PERF_FIXED_CTR0 + i,
.idx = i
};
wrmsr(MSR_CORE_PERF_FIXED_CTR0 + i, x);
report(rdpmc(i | (1 << 30)) == x, "fixed cntr-%d", i);
exc = test_for_exception(GP_VECTOR, do_rdpmc_fast, &cnt);
if (exc)
report_skip("fixed fast-%d", i);
else
report(cnt.count == (u32)x, "fixed fast-%d", i);
}
report_prefix_pop();
}
static void check_running_counter_wrmsr(void)
{
uint64_t status;
pmu_counter_t evt = {
.ctr = MSR_IA32_PERFCTR0,
.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[1].unit_sel,
.count = 0,
};
report_prefix_push("running counter wrmsr");
start_event(&evt);
loop();
wrmsr(MSR_IA32_PERFCTR0, 0);
stop_event(&evt);
report(evt.count < gp_events[1].min, "cntr");
/* clear status before overflow test */
wrmsr(MSR_CORE_PERF_GLOBAL_OVF_CTRL,
rdmsr(MSR_CORE_PERF_GLOBAL_STATUS));
evt.count = 0;
start_event(&evt);
wrmsr(MSR_IA32_PERFCTR0, -1);
loop();
stop_event(&evt);
status = rdmsr(MSR_CORE_PERF_GLOBAL_STATUS);
report(status & 1, "status");
report_prefix_pop();
}
int main(int ac, char **av)
{
struct cpuid id = cpuid(10);
setup_vm();
setup_idt();
handle_irq(PC_VECTOR, cnt_overflow);
buf = malloc(N*64);
eax.full = id.a;
ebx.full = id.b;
edx.full = id.d;
if (!eax.split.version_id) {
printf("No pmu is detected!\n");
return report_summary();
}
printf("PMU version: %d\n", eax.split.version_id);
printf("GP counters: %d\n", eax.split.num_counters);
printf("GP counter width: %d\n", eax.split.bit_width);
printf("Mask length: %d\n", eax.split.mask_length);
printf("Fixed counters: %d\n", edx.split.num_counters_fixed);
printf("Fixed counter width: %d\n", edx.split.bit_width_fixed);
num_counters = eax.split.num_counters;
apic_write(APIC_LVTPC, PC_VECTOR);
check_gp_counters();
check_fixed_counters();
check_rdpmc();
check_counters_many();
check_counter_overflow();
check_gp_counter_cmask();
check_running_counter_wrmsr();
return report_summary();
}