Author | Tokens | Token Proportion | Commits | Commit Proportion |
---|---|---|---|---|
Marc Zyngier | 2368 | 98.42% | 3 | 75.00% |
Janne Grunau | 38 | 1.58% | 1 | 25.00% |
Total | 2406 | 4 |
// SPDX-License-Identifier: GPL-2.0 /* * CPU PMU driver for the Apple M1 and derivatives * * Copyright (C) 2021 Google LLC * * Author: Marc Zyngier <maz@kernel.org> * * Most of the information used in this driver was provided by the * Asahi Linux project. The rest was experimentally discovered. */ #include <linux/of.h> #include <linux/perf/arm_pmu.h> #include <linux/platform_device.h> #include <asm/apple_m1_pmu.h> #include <asm/irq_regs.h> #include <asm/perf_event.h> #define M1_PMU_NR_COUNTERS 10 #define M1_PMU_CFG_EVENT GENMASK(7, 0) #define ANY_BUT_0_1 GENMASK(9, 2) #define ONLY_2_TO_7 GENMASK(7, 2) #define ONLY_2_4_6 (BIT(2) | BIT(4) | BIT(6)) #define ONLY_5_6_7 (BIT(5) | BIT(6) | BIT(7)) /* * Description of the events we actually know about, as well as those with * a specific counter affinity. Yes, this is a grand total of two known * counters, and the rest is anybody's guess. * * Not all counters can count all events. Counters #0 and #1 are wired to * count cycles and instructions respectively, and some events have * bizarre mappings (every other counter, or even *one* counter). These * restrictions equally apply to both P and E cores. * * It is worth noting that the PMUs attached to P and E cores are likely * to be different because the underlying uarches are different. At the * moment, we don't really need to distinguish between the two because we * know next to nothing about the events themselves, and we already have * per cpu-type PMU abstractions. * * If we eventually find out that the events are different across * implementations, we'll have to introduce per cpu-type tables. */ enum m1_pmu_events { M1_PMU_PERFCTR_UNKNOWN_01 = 0x01, M1_PMU_PERFCTR_CPU_CYCLES = 0x02, M1_PMU_PERFCTR_INSTRUCTIONS = 0x8c, M1_PMU_PERFCTR_UNKNOWN_8d = 0x8d, M1_PMU_PERFCTR_UNKNOWN_8e = 0x8e, M1_PMU_PERFCTR_UNKNOWN_8f = 0x8f, M1_PMU_PERFCTR_UNKNOWN_90 = 0x90, M1_PMU_PERFCTR_UNKNOWN_93 = 0x93, M1_PMU_PERFCTR_UNKNOWN_94 = 0x94, M1_PMU_PERFCTR_UNKNOWN_95 = 0x95, M1_PMU_PERFCTR_UNKNOWN_96 = 0x96, M1_PMU_PERFCTR_UNKNOWN_97 = 0x97, M1_PMU_PERFCTR_UNKNOWN_98 = 0x98, M1_PMU_PERFCTR_UNKNOWN_99 = 0x99, M1_PMU_PERFCTR_UNKNOWN_9a = 0x9a, M1_PMU_PERFCTR_UNKNOWN_9b = 0x9b, M1_PMU_PERFCTR_UNKNOWN_9c = 0x9c, M1_PMU_PERFCTR_UNKNOWN_9f = 0x9f, M1_PMU_PERFCTR_UNKNOWN_bf = 0xbf, M1_PMU_PERFCTR_UNKNOWN_c0 = 0xc0, M1_PMU_PERFCTR_UNKNOWN_c1 = 0xc1, M1_PMU_PERFCTR_UNKNOWN_c4 = 0xc4, M1_PMU_PERFCTR_UNKNOWN_c5 = 0xc5, M1_PMU_PERFCTR_UNKNOWN_c6 = 0xc6, M1_PMU_PERFCTR_UNKNOWN_c8 = 0xc8, M1_PMU_PERFCTR_UNKNOWN_ca = 0xca, M1_PMU_PERFCTR_UNKNOWN_cb = 0xcb, M1_PMU_PERFCTR_UNKNOWN_f5 = 0xf5, M1_PMU_PERFCTR_UNKNOWN_f6 = 0xf6, M1_PMU_PERFCTR_UNKNOWN_f7 = 0xf7, M1_PMU_PERFCTR_UNKNOWN_f8 = 0xf8, M1_PMU_PERFCTR_UNKNOWN_fd = 0xfd, M1_PMU_PERFCTR_LAST = M1_PMU_CFG_EVENT, /* * From this point onwards, these are not actual HW events, * but attributes that get stored in hw->config_base. */ M1_PMU_CFG_COUNT_USER = BIT(8), M1_PMU_CFG_COUNT_KERNEL = BIT(9), }; /* * Per-event affinity table. Most events can be installed on counter * 2-9, but there are a number of exceptions. Note that this table * has been created experimentally, and I wouldn't be surprised if more * counters had strange affinities. */ static const u16 m1_pmu_event_affinity[M1_PMU_PERFCTR_LAST + 1] = { [0 ... M1_PMU_PERFCTR_LAST] = ANY_BUT_0_1, [M1_PMU_PERFCTR_UNKNOWN_01] = BIT(7), [M1_PMU_PERFCTR_CPU_CYCLES] = ANY_BUT_0_1 | BIT(0), [M1_PMU_PERFCTR_INSTRUCTIONS] = BIT(7) | BIT(1), [M1_PMU_PERFCTR_UNKNOWN_8d] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_8e] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_8f] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_90] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_93] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_94] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_95] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_96] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_97] = BIT(7), [M1_PMU_PERFCTR_UNKNOWN_98] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_99] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_9a] = BIT(7), [M1_PMU_PERFCTR_UNKNOWN_9b] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_9c] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_9f] = BIT(7), [M1_PMU_PERFCTR_UNKNOWN_bf] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_c0] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_c1] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_c4] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_c5] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_c6] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_c8] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_ca] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_cb] = ONLY_5_6_7, [M1_PMU_PERFCTR_UNKNOWN_f5] = ONLY_2_4_6, [M1_PMU_PERFCTR_UNKNOWN_f6] = ONLY_2_4_6, [M1_PMU_PERFCTR_UNKNOWN_f7] = ONLY_2_4_6, [M1_PMU_PERFCTR_UNKNOWN_f8] = ONLY_2_TO_7, [M1_PMU_PERFCTR_UNKNOWN_fd] = ONLY_2_4_6, }; static const unsigned m1_pmu_perf_map[PERF_COUNT_HW_MAX] = { PERF_MAP_ALL_UNSUPPORTED, [PERF_COUNT_HW_CPU_CYCLES] = M1_PMU_PERFCTR_CPU_CYCLES, [PERF_COUNT_HW_INSTRUCTIONS] = M1_PMU_PERFCTR_INSTRUCTIONS, /* No idea about the rest yet */ }; /* sysfs definitions */ static ssize_t m1_pmu_events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) { struct perf_pmu_events_attr *pmu_attr; pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); return sprintf(page, "event=0x%04llx\n", pmu_attr->id); } #define M1_PMU_EVENT_ATTR(name, config) \ PMU_EVENT_ATTR_ID(name, m1_pmu_events_sysfs_show, config) static struct attribute *m1_pmu_event_attrs[] = { M1_PMU_EVENT_ATTR(cycles, M1_PMU_PERFCTR_CPU_CYCLES), M1_PMU_EVENT_ATTR(instructions, M1_PMU_PERFCTR_INSTRUCTIONS), NULL, }; static const struct attribute_group m1_pmu_events_attr_group = { .name = "events", .attrs = m1_pmu_event_attrs, }; PMU_FORMAT_ATTR(event, "config:0-7"); static struct attribute *m1_pmu_format_attrs[] = { &format_attr_event.attr, NULL, }; static const struct attribute_group m1_pmu_format_attr_group = { .name = "format", .attrs = m1_pmu_format_attrs, }; /* Low level accessors. No synchronisation. */ #define PMU_READ_COUNTER(_idx) \ case _idx: return read_sysreg_s(SYS_IMP_APL_PMC## _idx ##_EL1) #define PMU_WRITE_COUNTER(_val, _idx) \ case _idx: \ write_sysreg_s(_val, SYS_IMP_APL_PMC## _idx ##_EL1); \ return static u64 m1_pmu_read_hw_counter(unsigned int index) { switch (index) { PMU_READ_COUNTER(0); PMU_READ_COUNTER(1); PMU_READ_COUNTER(2); PMU_READ_COUNTER(3); PMU_READ_COUNTER(4); PMU_READ_COUNTER(5); PMU_READ_COUNTER(6); PMU_READ_COUNTER(7); PMU_READ_COUNTER(8); PMU_READ_COUNTER(9); } BUG(); } static void m1_pmu_write_hw_counter(u64 val, unsigned int index) { switch (index) { PMU_WRITE_COUNTER(val, 0); PMU_WRITE_COUNTER(val, 1); PMU_WRITE_COUNTER(val, 2); PMU_WRITE_COUNTER(val, 3); PMU_WRITE_COUNTER(val, 4); PMU_WRITE_COUNTER(val, 5); PMU_WRITE_COUNTER(val, 6); PMU_WRITE_COUNTER(val, 7); PMU_WRITE_COUNTER(val, 8); PMU_WRITE_COUNTER(val, 9); } BUG(); } #define get_bit_offset(index, mask) (__ffs(mask) + (index)) static void __m1_pmu_enable_counter(unsigned int index, bool en) { u64 val, bit; switch (index) { case 0 ... 7: bit = BIT(get_bit_offset(index, PMCR0_CNT_ENABLE_0_7)); break; case 8 ... 9: bit = BIT(get_bit_offset(index - 8, PMCR0_CNT_ENABLE_8_9)); break; default: BUG(); } val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); if (en) val |= bit; else val &= ~bit; write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); } static void m1_pmu_enable_counter(unsigned int index) { __m1_pmu_enable_counter(index, true); } static void m1_pmu_disable_counter(unsigned int index) { __m1_pmu_enable_counter(index, false); } static void __m1_pmu_enable_counter_interrupt(unsigned int index, bool en) { u64 val, bit; switch (index) { case 0 ... 7: bit = BIT(get_bit_offset(index, PMCR0_PMI_ENABLE_0_7)); break; case 8 ... 9: bit = BIT(get_bit_offset(index - 8, PMCR0_PMI_ENABLE_8_9)); break; default: BUG(); } val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); if (en) val |= bit; else val &= ~bit; write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); } static void m1_pmu_enable_counter_interrupt(unsigned int index) { __m1_pmu_enable_counter_interrupt(index, true); } static void m1_pmu_disable_counter_interrupt(unsigned int index) { __m1_pmu_enable_counter_interrupt(index, false); } static void m1_pmu_configure_counter(unsigned int index, u8 event, bool user, bool kernel) { u64 val, user_bit, kernel_bit; int shift; switch (index) { case 0 ... 7: user_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL0_0_7)); kernel_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL1_0_7)); break; case 8 ... 9: user_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL0_8_9)); kernel_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL1_8_9)); break; default: BUG(); } val = read_sysreg_s(SYS_IMP_APL_PMCR1_EL1); if (user) val |= user_bit; else val &= ~user_bit; if (kernel) val |= kernel_bit; else val &= ~kernel_bit; write_sysreg_s(val, SYS_IMP_APL_PMCR1_EL1); /* * Counters 0 and 1 have fixed events. For anything else, * place the event at the expected location in the relevant * register (PMESR0 holds the event configuration for counters * 2-5, resp. PMESR1 for counters 6-9). */ switch (index) { case 0 ... 1: break; case 2 ... 5: shift = (index - 2) * 8; val = read_sysreg_s(SYS_IMP_APL_PMESR0_EL1); val &= ~((u64)0xff << shift); val |= (u64)event << shift; write_sysreg_s(val, SYS_IMP_APL_PMESR0_EL1); break; case 6 ... 9: shift = (index - 6) * 8; val = read_sysreg_s(SYS_IMP_APL_PMESR1_EL1); val &= ~((u64)0xff << shift); val |= (u64)event << shift; write_sysreg_s(val, SYS_IMP_APL_PMESR1_EL1); break; } } /* arm_pmu backend */ static void m1_pmu_enable_event(struct perf_event *event) { bool user, kernel; u8 evt; evt = event->hw.config_base & M1_PMU_CFG_EVENT; user = event->hw.config_base & M1_PMU_CFG_COUNT_USER; kernel = event->hw.config_base & M1_PMU_CFG_COUNT_KERNEL; m1_pmu_disable_counter_interrupt(event->hw.idx); m1_pmu_disable_counter(event->hw.idx); isb(); m1_pmu_configure_counter(event->hw.idx, evt, user, kernel); m1_pmu_enable_counter(event->hw.idx); m1_pmu_enable_counter_interrupt(event->hw.idx); isb(); } static void m1_pmu_disable_event(struct perf_event *event) { m1_pmu_disable_counter_interrupt(event->hw.idx); m1_pmu_disable_counter(event->hw.idx); isb(); } static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu) { struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); struct pt_regs *regs; u64 overflow, state; int idx; overflow = read_sysreg_s(SYS_IMP_APL_PMSR_EL1); if (!overflow) { /* Spurious interrupt? */ state = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); state &= ~PMCR0_IACT; write_sysreg_s(state, SYS_IMP_APL_PMCR0_EL1); isb(); return IRQ_NONE; } cpu_pmu->stop(cpu_pmu); regs = get_irq_regs(); for (idx = 0; idx < cpu_pmu->num_events; idx++) { struct perf_event *event = cpuc->events[idx]; struct perf_sample_data data; if (!event) continue; armpmu_event_update(event); perf_sample_data_init(&data, 0, event->hw.last_period); if (!armpmu_event_set_period(event)) continue; if (perf_event_overflow(event, &data, regs)) m1_pmu_disable_event(event); } cpu_pmu->start(cpu_pmu); return IRQ_HANDLED; } static u64 m1_pmu_read_counter(struct perf_event *event) { return m1_pmu_read_hw_counter(event->hw.idx); } static void m1_pmu_write_counter(struct perf_event *event, u64 value) { m1_pmu_write_hw_counter(value, event->hw.idx); isb(); } static int m1_pmu_get_event_idx(struct pmu_hw_events *cpuc, struct perf_event *event) { unsigned long evtype = event->hw.config_base & M1_PMU_CFG_EVENT; unsigned long affinity = m1_pmu_event_affinity[evtype]; int idx; /* * Place the event on the first free counter that can count * this event. * * We could do a better job if we had a view of all the events * counting on the PMU at any given time, and by placing the * most constraining events first. */ for_each_set_bit(idx, &affinity, M1_PMU_NR_COUNTERS) { if (!test_and_set_bit(idx, cpuc->used_mask)) return idx; } return -EAGAIN; } static void m1_pmu_clear_event_idx(struct pmu_hw_events *cpuc, struct perf_event *event) { clear_bit(event->hw.idx, cpuc->used_mask); } static void __m1_pmu_set_mode(u8 mode) { u64 val; val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); val &= ~(PMCR0_IMODE | PMCR0_IACT); val |= FIELD_PREP(PMCR0_IMODE, mode); write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); isb(); } static void m1_pmu_start(struct arm_pmu *cpu_pmu) { __m1_pmu_set_mode(PMCR0_IMODE_FIQ); } static void m1_pmu_stop(struct arm_pmu *cpu_pmu) { __m1_pmu_set_mode(PMCR0_IMODE_OFF); } static int m1_pmu_map_event(struct perf_event *event) { /* * Although the counters are 48bit wide, bit 47 is what * triggers the overflow interrupt. Advertise the counters * being 47bit wide to mimick the behaviour of the ARM PMU. */ event->hw.flags |= ARMPMU_EVT_47BIT; return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); } static int m2_pmu_map_event(struct perf_event *event) { /* * Same deal as the above, except that M2 has 64bit counters. * Which, as far as we're concerned, actually means 63 bits. * Yes, this is getting awkward. */ event->hw.flags |= ARMPMU_EVT_63BIT; return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); } static void m1_pmu_reset(void *info) { int i; __m1_pmu_set_mode(PMCR0_IMODE_OFF); for (i = 0; i < M1_PMU_NR_COUNTERS; i++) { m1_pmu_disable_counter(i); m1_pmu_disable_counter_interrupt(i); m1_pmu_write_hw_counter(0, i); } isb(); } static int m1_pmu_set_event_filter(struct hw_perf_event *event, struct perf_event_attr *attr) { unsigned long config_base = 0; if (!attr->exclude_guest) return -EINVAL; if (!attr->exclude_kernel) config_base |= M1_PMU_CFG_COUNT_KERNEL; if (!attr->exclude_user) config_base |= M1_PMU_CFG_COUNT_USER; event->config_base = config_base; return 0; } static int m1_pmu_init(struct arm_pmu *cpu_pmu, u32 flags) { cpu_pmu->handle_irq = m1_pmu_handle_irq; cpu_pmu->enable = m1_pmu_enable_event; cpu_pmu->disable = m1_pmu_disable_event; cpu_pmu->read_counter = m1_pmu_read_counter; cpu_pmu->write_counter = m1_pmu_write_counter; cpu_pmu->get_event_idx = m1_pmu_get_event_idx; cpu_pmu->clear_event_idx = m1_pmu_clear_event_idx; cpu_pmu->start = m1_pmu_start; cpu_pmu->stop = m1_pmu_stop; if (flags & ARMPMU_EVT_47BIT) cpu_pmu->map_event = m1_pmu_map_event; else if (flags & ARMPMU_EVT_63BIT) cpu_pmu->map_event = m2_pmu_map_event; else return WARN_ON(-EINVAL); cpu_pmu->reset = m1_pmu_reset; cpu_pmu->set_event_filter = m1_pmu_set_event_filter; cpu_pmu->num_events = M1_PMU_NR_COUNTERS; cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &m1_pmu_events_attr_group; cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &m1_pmu_format_attr_group; return 0; } /* Device driver gunk */ static int m1_pmu_ice_init(struct arm_pmu *cpu_pmu) { cpu_pmu->name = "apple_icestorm_pmu"; return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); } static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu) { cpu_pmu->name = "apple_firestorm_pmu"; return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); } static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu) { cpu_pmu->name = "apple_avalanche_pmu"; return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); } static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu) { cpu_pmu->name = "apple_blizzard_pmu"; return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); } static const struct of_device_id m1_pmu_of_device_ids[] = { { .compatible = "apple,avalanche-pmu", .data = m2_pmu_avalanche_init, }, { .compatible = "apple,blizzard-pmu", .data = m2_pmu_blizzard_init, }, { .compatible = "apple,icestorm-pmu", .data = m1_pmu_ice_init, }, { .compatible = "apple,firestorm-pmu", .data = m1_pmu_fire_init, }, { }, }; MODULE_DEVICE_TABLE(of, m1_pmu_of_device_ids); static int m1_pmu_device_probe(struct platform_device *pdev) { return arm_pmu_device_probe(pdev, m1_pmu_of_device_ids, NULL); } static struct platform_driver m1_pmu_driver = { .driver = { .name = "apple-m1-cpu-pmu", .of_match_table = m1_pmu_of_device_ids, .suppress_bind_attrs = true, }, .probe = m1_pmu_device_probe, }; module_platform_driver(m1_pmu_driver);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1