| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Neil Leeder | 4150 | 99.57% | 3 | 42.86% |
| Peter Zijlstra | 6 | 0.14% | 1 | 14.29% |
| Andrew Murray | 5 | 0.12% | 1 | 14.29% |
| Qi Liu | 5 | 0.12% | 1 | 14.29% |
| Thomas Gleixner | 2 | 0.05% | 1 | 14.29% |
| Total | 4168 | 7 |
// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved. */ #include <linux/acpi.h> #include <linux/bitops.h> #include <linux/bug.h> #include <linux/cpuhotplug.h> #include <linux/cpumask.h> #include <linux/device.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/kernel.h> #include <linux/list.h> #include <linux/percpu.h> #include <linux/perf_event.h> #include <linux/platform_device.h> #include <linux/smp.h> #include <linux/spinlock.h> #include <linux/sysfs.h> #include <linux/types.h> #include <asm/barrier.h> #include <asm/local64.h> #include <asm/sysreg.h> #define MAX_L2_CTRS 9 #define L2PMCR_NUM_EV_SHIFT 11 #define L2PMCR_NUM_EV_MASK 0x1F #define L2PMCR 0x400 #define L2PMCNTENCLR 0x403 #define L2PMCNTENSET 0x404 #define L2PMINTENCLR 0x405 #define L2PMINTENSET 0x406 #define L2PMOVSCLR 0x407 #define L2PMOVSSET 0x408 #define L2PMCCNTCR 0x409 #define L2PMCCNTR 0x40A #define L2PMCCNTSR 0x40C #define L2PMRESR 0x410 #define IA_L2PMXEVCNTCR_BASE 0x420 #define IA_L2PMXEVCNTR_BASE 0x421 #define IA_L2PMXEVFILTER_BASE 0x423 #define IA_L2PMXEVTYPER_BASE 0x424 #define IA_L2_REG_OFFSET 0x10 #define L2PMXEVFILTER_SUFILTER_ALL 0x000E0000 #define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004 #define L2PMXEVFILTER_ORGFILTER_ALL 0x00000003 #define L2EVTYPER_REG_SHIFT 3 #define L2PMRESR_GROUP_BITS 8 #define L2PMRESR_GROUP_MASK GENMASK(7, 0) #define L2CYCLE_CTR_BIT 31 #define L2CYCLE_CTR_RAW_CODE 0xFE #define L2PMCR_RESET_ALL 0x6 #define L2PMCR_COUNTERS_ENABLE 0x1 #define L2PMCR_COUNTERS_DISABLE 0x0 #define L2PMRESR_EN BIT_ULL(63) #define L2_EVT_MASK 0x00000FFF #define L2_EVT_CODE_MASK 0x00000FF0 #define L2_EVT_GRP_MASK 0x0000000F #define L2_EVT_CODE_SHIFT 4 #define L2_EVT_GRP_SHIFT 0 #define L2_EVT_CODE(event) (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT) #define L2_EVT_GROUP(event) (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT) #define L2_EVT_GROUP_MAX 7 #define L2_COUNTER_RELOAD BIT_ULL(31) #define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63) #define L2CPUSRSELR_EL1 sys_reg(3, 3, 15, 0, 6) #define L2CPUSRDR_EL1 sys_reg(3, 3, 15, 0, 7) #define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE) /* * Events */ #define L2_EVENT_CYCLES 0xfe #define L2_EVENT_DCACHE_OPS 0x400 #define L2_EVENT_ICACHE_OPS 0x401 #define L2_EVENT_TLBI 0x402 #define L2_EVENT_BARRIERS 0x403 #define L2_EVENT_TOTAL_READS 0x405 #define L2_EVENT_TOTAL_WRITES 0x406 #define L2_EVENT_TOTAL_REQUESTS 0x407 #define L2_EVENT_LDREX 0x420 #define L2_EVENT_STREX 0x421 #define L2_EVENT_CLREX 0x422 static DEFINE_RAW_SPINLOCK(l2_access_lock); /** * set_l2_indirect_reg: write value to an L2 register * @reg: Address of L2 register. * @value: Value to be written to register. * * Use architecturally required barriers for ordering between system register * accesses */ static void set_l2_indirect_reg(u64 reg, u64 val) { unsigned long flags; raw_spin_lock_irqsave(&l2_access_lock, flags); write_sysreg_s(reg, L2CPUSRSELR_EL1); isb(); write_sysreg_s(val, L2CPUSRDR_EL1); isb(); raw_spin_unlock_irqrestore(&l2_access_lock, flags); } /** * get_l2_indirect_reg: read an L2 register value * @reg: Address of L2 register. * * Use architecturally required barriers for ordering between system register * accesses */ static u64 get_l2_indirect_reg(u64 reg) { u64 val; unsigned long flags; raw_spin_lock_irqsave(&l2_access_lock, flags); write_sysreg_s(reg, L2CPUSRSELR_EL1); isb(); val = read_sysreg_s(L2CPUSRDR_EL1); raw_spin_unlock_irqrestore(&l2_access_lock, flags); return val; } struct cluster_pmu; /* * Aggregate PMU. Implements the core pmu functions and manages * the hardware PMUs. */ struct l2cache_pmu { struct hlist_node node; u32 num_pmus; struct pmu pmu; int num_counters; cpumask_t cpumask; struct platform_device *pdev; struct cluster_pmu * __percpu *pmu_cluster; struct list_head clusters; }; /* * The cache is made up of one or more clusters, each cluster has its own PMU. * Each cluster is associated with one or more CPUs. * This structure represents one of the hardware PMUs. * * Events can be envisioned as a 2-dimensional array. Each column represents * a group of events. There are 8 groups. Only one entry from each * group can be in use at a time. * * Events are specified as 0xCCG, where CC is 2 hex digits specifying * the code (array row) and G specifies the group (column). * * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE * which is outside the above scheme. */ struct cluster_pmu { struct list_head next; struct perf_event *events[MAX_L2_CTRS]; struct l2cache_pmu *l2cache_pmu; DECLARE_BITMAP(used_counters, MAX_L2_CTRS); DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1); int irq; int cluster_id; /* The CPU that is used for collecting events on this cluster */ int on_cpu; /* All the CPUs associated with this cluster */ cpumask_t cluster_cpus; spinlock_t pmu_lock; }; #define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu)) static u32 l2_cycle_ctr_idx; static u32 l2_counter_present_mask; static inline u32 idx_to_reg_bit(u32 idx) { if (idx == l2_cycle_ctr_idx) return BIT(L2CYCLE_CTR_BIT); return BIT(idx); } static inline struct cluster_pmu *get_cluster_pmu( struct l2cache_pmu *l2cache_pmu, int cpu) { return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu); } static void cluster_pmu_reset(void) { /* Reset all counters */ set_l2_indirect_reg(L2PMCR, L2PMCR_RESET_ALL); set_l2_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask); set_l2_indirect_reg(L2PMINTENCLR, l2_counter_present_mask); set_l2_indirect_reg(L2PMOVSCLR, l2_counter_present_mask); } static inline void cluster_pmu_enable(void) { set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE); } static inline void cluster_pmu_disable(void) { set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE); } static inline void cluster_pmu_counter_set_value(u32 idx, u64 value) { if (idx == l2_cycle_ctr_idx) set_l2_indirect_reg(L2PMCCNTR, value); else set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value); } static inline u64 cluster_pmu_counter_get_value(u32 idx) { u64 value; if (idx == l2_cycle_ctr_idx) value = get_l2_indirect_reg(L2PMCCNTR); else value = get_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx)); return value; } static inline void cluster_pmu_counter_enable(u32 idx) { set_l2_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx)); } static inline void cluster_pmu_counter_disable(u32 idx) { set_l2_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx)); } static inline void cluster_pmu_counter_enable_interrupt(u32 idx) { set_l2_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx)); } static inline void cluster_pmu_counter_disable_interrupt(u32 idx) { set_l2_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx)); } static inline void cluster_pmu_set_evccntcr(u32 val) { set_l2_indirect_reg(L2PMCCNTCR, val); } static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val) { set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val); } static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val) { set_l2_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val); } static void cluster_pmu_set_resr(struct cluster_pmu *cluster, u32 event_group, u32 event_cc) { u64 field; u64 resr_val; u32 shift; unsigned long flags; shift = L2PMRESR_GROUP_BITS * event_group; field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift); spin_lock_irqsave(&cluster->pmu_lock, flags); resr_val = get_l2_indirect_reg(L2PMRESR); resr_val &= ~(L2PMRESR_GROUP_MASK << shift); resr_val |= field; resr_val |= L2PMRESR_EN; set_l2_indirect_reg(L2PMRESR, resr_val); spin_unlock_irqrestore(&cluster->pmu_lock, flags); } /* * Hardware allows filtering of events based on the originating * CPU. Turn this off by setting filter bits to allow events from * all CPUS, subunits and ID independent events in this cluster. */ static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr) { u32 val = L2PMXEVFILTER_SUFILTER_ALL | L2PMXEVFILTER_ORGFILTER_IDINDEP | L2PMXEVFILTER_ORGFILTER_ALL; set_l2_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val); } static inline u32 cluster_pmu_getreset_ovsr(void) { u32 result = get_l2_indirect_reg(L2PMOVSSET); set_l2_indirect_reg(L2PMOVSCLR, result); return result; } static inline bool cluster_pmu_has_overflowed(u32 ovsr) { return !!(ovsr & l2_counter_present_mask); } static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx) { return !!(ovsr & idx_to_reg_bit(idx)); } static void l2_cache_event_update(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; u64 delta, prev, now; u32 idx = hwc->idx; do { prev = local64_read(&hwc->prev_count); now = cluster_pmu_counter_get_value(idx); } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); /* * The cycle counter is 64-bit, but all other counters are * 32-bit, and we must handle 32-bit overflow explicitly. */ delta = now - prev; if (idx != l2_cycle_ctr_idx) delta &= 0xffffffff; local64_add(delta, &event->count); } static void l2_cache_cluster_set_period(struct cluster_pmu *cluster, struct hw_perf_event *hwc) { u32 idx = hwc->idx; u64 new; /* * We limit the max period to half the max counter value so * that even in the case of extreme interrupt latency the * counter will (hopefully) not wrap past its initial value. */ if (idx == l2_cycle_ctr_idx) new = L2_CYCLE_COUNTER_RELOAD; else new = L2_COUNTER_RELOAD; local64_set(&hwc->prev_count, new); cluster_pmu_counter_set_value(idx, new); } static int l2_cache_get_event_idx(struct cluster_pmu *cluster, struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; int idx; int num_ctrs = cluster->l2cache_pmu->num_counters - 1; unsigned int group; if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) { if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters)) return -EAGAIN; return l2_cycle_ctr_idx; } idx = find_first_zero_bit(cluster->used_counters, num_ctrs); if (idx == num_ctrs) /* The counters are all in use. */ return -EAGAIN; /* * Check for column exclusion: event column already in use by another * event. This is for events which are not in the same group. * Conflicting events in the same group are detected in event_init. */ group = L2_EVT_GROUP(hwc->config_base); if (test_bit(group, cluster->used_groups)) return -EAGAIN; set_bit(idx, cluster->used_counters); set_bit(group, cluster->used_groups); return idx; } static void l2_cache_clear_event_idx(struct cluster_pmu *cluster, struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; clear_bit(idx, cluster->used_counters); if (hwc->config_base != L2CYCLE_CTR_RAW_CODE) clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups); } static irqreturn_t l2_cache_handle_irq(int irq_num, void *data) { struct cluster_pmu *cluster = data; int num_counters = cluster->l2cache_pmu->num_counters; u32 ovsr; int idx; ovsr = cluster_pmu_getreset_ovsr(); if (!cluster_pmu_has_overflowed(ovsr)) return IRQ_NONE; for_each_set_bit(idx, cluster->used_counters, num_counters) { struct perf_event *event = cluster->events[idx]; struct hw_perf_event *hwc; if (WARN_ON_ONCE(!event)) continue; if (!cluster_pmu_counter_has_overflowed(ovsr, idx)) continue; l2_cache_event_update(event); hwc = &event->hw; l2_cache_cluster_set_period(cluster, hwc); } return IRQ_HANDLED; } /* * Implementation of abstract pmu functionality required by * the core perf events code. */ static void l2_cache_pmu_enable(struct pmu *pmu) { /* * Although there is only one PMU (per socket) controlling multiple * physical PMUs (per cluster), because we do not support per-task mode * each event is associated with a CPU. Each event has pmu_enable * called on its CPU, so here it is only necessary to enable the * counters for the current CPU. */ cluster_pmu_enable(); } static void l2_cache_pmu_disable(struct pmu *pmu) { cluster_pmu_disable(); } static int l2_cache_event_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; struct cluster_pmu *cluster; struct perf_event *sibling; struct l2cache_pmu *l2cache_pmu; if (event->attr.type != event->pmu->type) return -ENOENT; l2cache_pmu = to_l2cache_pmu(event->pmu); if (hwc->sample_period) { dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "Sampling not supported\n"); return -EOPNOTSUPP; } if (event->cpu < 0) { dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "Per-task mode not supported\n"); return -EOPNOTSUPP; } if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) || ((event->attr.config & ~L2_EVT_MASK) != 0)) && (event->attr.config != L2CYCLE_CTR_RAW_CODE)) { dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "Invalid config %llx\n", event->attr.config); return -EINVAL; } /* Don't allow groups with mixed PMUs, except for s/w events */ if (event->group_leader->pmu != event->pmu && !is_software_event(event->group_leader)) { dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "Can't create mixed PMU group\n"); return -EINVAL; } for_each_sibling_event(sibling, event->group_leader) { if (sibling->pmu != event->pmu && !is_software_event(sibling)) { dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "Can't create mixed PMU group\n"); return -EINVAL; } } cluster = get_cluster_pmu(l2cache_pmu, event->cpu); if (!cluster) { /* CPU has not been initialised */ dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "CPU%d not associated with L2 cluster\n", event->cpu); return -EINVAL; } /* Ensure all events in a group are on the same cpu */ if ((event->group_leader != event) && (cluster->on_cpu != event->group_leader->cpu)) { dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "Can't create group on CPUs %d and %d", event->cpu, event->group_leader->cpu); return -EINVAL; } if ((event != event->group_leader) && !is_software_event(event->group_leader) && (L2_EVT_GROUP(event->group_leader->attr.config) == L2_EVT_GROUP(event->attr.config))) { dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "Column exclusion: conflicting events %llx %llx\n", event->group_leader->attr.config, event->attr.config); return -EINVAL; } for_each_sibling_event(sibling, event->group_leader) { if ((sibling != event) && !is_software_event(sibling) && (L2_EVT_GROUP(sibling->attr.config) == L2_EVT_GROUP(event->attr.config))) { dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, "Column exclusion: conflicting events %llx %llx\n", sibling->attr.config, event->attr.config); return -EINVAL; } } hwc->idx = -1; hwc->config_base = event->attr.config; /* * Ensure all events are on the same cpu so all events are in the * same cpu context, to avoid races on pmu_enable etc. */ event->cpu = cluster->on_cpu; return 0; } static void l2_cache_event_start(struct perf_event *event, int flags) { struct cluster_pmu *cluster; struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; u32 config; u32 event_cc, event_group; hwc->state = 0; cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); l2_cache_cluster_set_period(cluster, hwc); if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) { cluster_pmu_set_evccntcr(0); } else { config = hwc->config_base; event_cc = L2_EVT_CODE(config); event_group = L2_EVT_GROUP(config); cluster_pmu_set_evcntcr(idx, 0); cluster_pmu_set_evtyper(idx, event_group); cluster_pmu_set_resr(cluster, event_group, event_cc); cluster_pmu_set_evfilter_sys_mode(idx); } cluster_pmu_counter_enable_interrupt(idx); cluster_pmu_counter_enable(idx); } static void l2_cache_event_stop(struct perf_event *event, int flags) { struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; if (hwc->state & PERF_HES_STOPPED) return; cluster_pmu_counter_disable_interrupt(idx); cluster_pmu_counter_disable(idx); if (flags & PERF_EF_UPDATE) l2_cache_event_update(event); hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } static int l2_cache_event_add(struct perf_event *event, int flags) { struct hw_perf_event *hwc = &event->hw; int idx; int err = 0; struct cluster_pmu *cluster; cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); idx = l2_cache_get_event_idx(cluster, event); if (idx < 0) return idx; hwc->idx = idx; hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; cluster->events[idx] = event; local64_set(&hwc->prev_count, 0); if (flags & PERF_EF_START) l2_cache_event_start(event, flags); /* Propagate changes to the userspace mapping. */ perf_event_update_userpage(event); return err; } static void l2_cache_event_del(struct perf_event *event, int flags) { struct hw_perf_event *hwc = &event->hw; struct cluster_pmu *cluster; int idx = hwc->idx; cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); l2_cache_event_stop(event, flags | PERF_EF_UPDATE); cluster->events[idx] = NULL; l2_cache_clear_event_idx(cluster, event); perf_event_update_userpage(event); } static void l2_cache_event_read(struct perf_event *event) { l2_cache_event_update(event); } static ssize_t l2_cache_pmu_cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) { struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev)); return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask); } static struct device_attribute l2_cache_pmu_cpumask_attr = __ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL); static struct attribute *l2_cache_pmu_cpumask_attrs[] = { &l2_cache_pmu_cpumask_attr.attr, NULL, }; static struct attribute_group l2_cache_pmu_cpumask_group = { .attrs = l2_cache_pmu_cpumask_attrs, }; /* CCG format for perf RAW codes. */ PMU_FORMAT_ATTR(l2_code, "config:4-11"); PMU_FORMAT_ATTR(l2_group, "config:0-3"); PMU_FORMAT_ATTR(event, "config:0-11"); static struct attribute *l2_cache_pmu_formats[] = { &format_attr_l2_code.attr, &format_attr_l2_group.attr, &format_attr_event.attr, NULL, }; static struct attribute_group l2_cache_pmu_format_group = { .name = "format", .attrs = l2_cache_pmu_formats, }; static ssize_t l2cache_pmu_event_show(struct device *dev, struct device_attribute *attr, char *page) { struct perf_pmu_events_attr *pmu_attr; pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); return sprintf(page, "event=0x%02llx\n", pmu_attr->id); } #define L2CACHE_EVENT_ATTR(_name, _id) \ (&((struct perf_pmu_events_attr[]) { \ { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \ .id = _id, } \ })[0].attr.attr) static struct attribute *l2_cache_pmu_events[] = { L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES), L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS), L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS), L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI), L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS), L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS), L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES), L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS), L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX), L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX), L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX), NULL }; static struct attribute_group l2_cache_pmu_events_group = { .name = "events", .attrs = l2_cache_pmu_events, }; static const struct attribute_group *l2_cache_pmu_attr_grps[] = { &l2_cache_pmu_format_group, &l2_cache_pmu_cpumask_group, &l2_cache_pmu_events_group, NULL, }; /* * Generic device handlers */ static const struct acpi_device_id l2_cache_pmu_acpi_match[] = { { "QCOM8130", }, { } }; static int get_num_counters(void) { int val; val = get_l2_indirect_reg(L2PMCR); /* * Read number of counters from L2PMCR and add 1 * for the cycle counter. */ return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1; } static struct cluster_pmu *l2_cache_associate_cpu_with_cluster( struct l2cache_pmu *l2cache_pmu, int cpu) { u64 mpidr; int cpu_cluster_id; struct cluster_pmu *cluster = NULL; /* * This assumes that the cluster_id is in MPIDR[aff1] for * single-threaded cores, and MPIDR[aff2] for multi-threaded * cores. This logic will have to be updated if this changes. */ mpidr = read_cpuid_mpidr(); if (mpidr & MPIDR_MT_BITMASK) cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2); else cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); list_for_each_entry(cluster, &l2cache_pmu->clusters, next) { if (cluster->cluster_id != cpu_cluster_id) continue; dev_info(&l2cache_pmu->pdev->dev, "CPU%d associated with cluster %d\n", cpu, cluster->cluster_id); cpumask_set_cpu(cpu, &cluster->cluster_cpus); *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster; break; } return cluster; } static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) { struct cluster_pmu *cluster; struct l2cache_pmu *l2cache_pmu; l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); cluster = get_cluster_pmu(l2cache_pmu, cpu); if (!cluster) { /* First time this CPU has come online */ cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu); if (!cluster) { /* Only if broken firmware doesn't list every cluster */ WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu); return 0; } } /* If another CPU is managing this cluster, we're done */ if (cluster->on_cpu != -1) return 0; /* * All CPUs on this cluster were down, use this one. * Reset to put it into sane state. */ cluster->on_cpu = cpu; cpumask_set_cpu(cpu, &l2cache_pmu->cpumask); cluster_pmu_reset(); WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu))); enable_irq(cluster->irq); return 0; } static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) { struct cluster_pmu *cluster; struct l2cache_pmu *l2cache_pmu; cpumask_t cluster_online_cpus; unsigned int target; l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); cluster = get_cluster_pmu(l2cache_pmu, cpu); if (!cluster) return 0; /* If this CPU is not managing the cluster, we're done */ if (cluster->on_cpu != cpu) return 0; /* Give up ownership of cluster */ cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask); cluster->on_cpu = -1; /* Any other CPU for this cluster which is still online */ cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus, cpu_online_mask); target = cpumask_any_but(&cluster_online_cpus, cpu); if (target >= nr_cpu_ids) { disable_irq(cluster->irq); return 0; } perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target); cluster->on_cpu = target; cpumask_set_cpu(target, &l2cache_pmu->cpumask); WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target))); return 0; } static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) { struct platform_device *pdev = to_platform_device(dev->parent); struct platform_device *sdev = to_platform_device(dev); struct l2cache_pmu *l2cache_pmu = data; struct cluster_pmu *cluster; struct acpi_device *device; unsigned long fw_cluster_id; int err; int irq; if (acpi_bus_get_device(ACPI_HANDLE(dev), &device)) return -ENODEV; if (kstrtoul(device->pnp.unique_id, 10, &fw_cluster_id) < 0) { dev_err(&pdev->dev, "unable to read ACPI uid\n"); return -ENODEV; } cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL); if (!cluster) return -ENOMEM; INIT_LIST_HEAD(&cluster->next); list_add(&cluster->next, &l2cache_pmu->clusters); cluster->cluster_id = fw_cluster_id; irq = platform_get_irq(sdev, 0); if (irq < 0) return irq; irq_set_status_flags(irq, IRQ_NOAUTOEN); cluster->irq = irq; cluster->l2cache_pmu = l2cache_pmu; cluster->on_cpu = -1; err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, "l2-cache-pmu", cluster); if (err) { dev_err(&pdev->dev, "Unable to request IRQ%d for L2 PMU counters\n", irq); return err; } dev_info(&pdev->dev, "Registered L2 cache PMU cluster %ld\n", fw_cluster_id); spin_lock_init(&cluster->pmu_lock); l2cache_pmu->num_pmus++; return 0; } static int l2_cache_pmu_probe(struct platform_device *pdev) { int err; struct l2cache_pmu *l2cache_pmu; l2cache_pmu = devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL); if (!l2cache_pmu) return -ENOMEM; INIT_LIST_HEAD(&l2cache_pmu->clusters); platform_set_drvdata(pdev, l2cache_pmu); l2cache_pmu->pmu = (struct pmu) { /* suffix is instance id for future use with multiple sockets */ .name = "l2cache_0", .task_ctx_nr = perf_invalid_context, .pmu_enable = l2_cache_pmu_enable, .pmu_disable = l2_cache_pmu_disable, .event_init = l2_cache_event_init, .add = l2_cache_event_add, .del = l2_cache_event_del, .start = l2_cache_event_start, .stop = l2_cache_event_stop, .read = l2_cache_event_read, .attr_groups = l2_cache_pmu_attr_grps, .capabilities = PERF_PMU_CAP_NO_EXCLUDE, }; l2cache_pmu->num_counters = get_num_counters(); l2cache_pmu->pdev = pdev; l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev, struct cluster_pmu *); if (!l2cache_pmu->pmu_cluster) return -ENOMEM; l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1; l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) | BIT(L2CYCLE_CTR_BIT); cpumask_clear(&l2cache_pmu->cpumask); /* Read cluster info and initialize each cluster */ err = device_for_each_child(&pdev->dev, l2cache_pmu, l2_cache_pmu_probe_cluster); if (err) return err; if (l2cache_pmu->num_pmus == 0) { dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n"); return -ENODEV; } err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, &l2cache_pmu->node); if (err) { dev_err(&pdev->dev, "Error %d registering hotplug", err); return err; } err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1); if (err) { dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err); goto out_unregister; } dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n", l2cache_pmu->num_pmus); return err; out_unregister: cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, &l2cache_pmu->node); return err; } static int l2_cache_pmu_remove(struct platform_device *pdev) { struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(platform_get_drvdata(pdev)); perf_pmu_unregister(&l2cache_pmu->pmu); cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, &l2cache_pmu->node); return 0; } static struct platform_driver l2_cache_pmu_driver = { .driver = { .name = "qcom-l2cache-pmu", .acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match), .suppress_bind_attrs = true, }, .probe = l2_cache_pmu_probe, .remove = l2_cache_pmu_remove, }; static int __init register_l2_cache_pmu_driver(void) { int err; err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, "AP_PERF_ARM_QCOM_L2_ONLINE", l2cache_pmu_online_cpu, l2cache_pmu_offline_cpu); if (err) return err; return platform_driver_register(&l2_cache_pmu_driver); } device_initcall(register_l2_cache_pmu_driver);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1