| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Boris Ostrovsky | 2375 | 97.54% | 6 | 54.55% |
| Pu Wen | 37 | 1.52% | 1 | 9.09% |
| Juergen Gross | 18 | 0.74% | 1 | 9.09% |
| Christoph Hellwig | 3 | 0.12% | 1 | 9.09% |
| Greg Kroah-Hartman | 1 | 0.04% | 1 | 9.09% |
| Borislav Petkov | 1 | 0.04% | 1 | 9.09% |
| Total | 2435 | 11 |
// SPDX-License-Identifier: GPL-2.0 #include <linux/types.h> #include <linux/interrupt.h> #include <asm/xen/hypercall.h> #include <xen/xen.h> #include <xen/page.h> #include <xen/interface/xen.h> #include <xen/interface/vcpu.h> #include <xen/interface/xenpmu.h> #include "xen-ops.h" #include "pmu.h" /* x86_pmu.handle_irq definition */ #include "../events/perf_event.h" #define XENPMU_IRQ_PROCESSING 1 struct xenpmu { /* Shared page between hypervisor and domain */ struct xen_pmu_data *xenpmu_data; uint8_t flags; }; static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared); #define get_xenpmu_data() (this_cpu_ptr(&xenpmu_shared)->xenpmu_data) #define get_xenpmu_flags() (this_cpu_ptr(&xenpmu_shared)->flags) /* Macro for computing address of a PMU MSR bank */ #define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \ (uintptr_t)ctxt->field)) /* AMD PMU */ #define F15H_NUM_COUNTERS 6 #define F10H_NUM_COUNTERS 4 static __read_mostly uint32_t amd_counters_base; static __read_mostly uint32_t amd_ctrls_base; static __read_mostly int amd_msr_step; static __read_mostly int k7_counters_mirrored; static __read_mostly int amd_num_counters; /* Intel PMU */ #define MSR_TYPE_COUNTER 0 #define MSR_TYPE_CTRL 1 #define MSR_TYPE_GLOBAL 2 #define MSR_TYPE_ARCH_COUNTER 3 #define MSR_TYPE_ARCH_CTRL 4 /* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */ #define PMU_GENERAL_NR_SHIFT 8 #define PMU_GENERAL_NR_BITS 8 #define PMU_GENERAL_NR_MASK (((1 << PMU_GENERAL_NR_BITS) - 1) \ << PMU_GENERAL_NR_SHIFT) /* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */ #define PMU_FIXED_NR_SHIFT 0 #define PMU_FIXED_NR_BITS 5 #define PMU_FIXED_NR_MASK (((1 << PMU_FIXED_NR_BITS) - 1) \ << PMU_FIXED_NR_SHIFT) /* Alias registers (0x4c1) for full-width writes to PMCs */ #define MSR_PMC_ALIAS_MASK (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0)) #define INTEL_PMC_TYPE_SHIFT 30 static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters; static void xen_pmu_arch_init(void) { if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { switch (boot_cpu_data.x86) { case 0x15: amd_num_counters = F15H_NUM_COUNTERS; amd_counters_base = MSR_F15H_PERF_CTR; amd_ctrls_base = MSR_F15H_PERF_CTL; amd_msr_step = 2; k7_counters_mirrored = 1; break; case 0x10: case 0x12: case 0x14: case 0x16: default: amd_num_counters = F10H_NUM_COUNTERS; amd_counters_base = MSR_K7_PERFCTR0; amd_ctrls_base = MSR_K7_EVNTSEL0; amd_msr_step = 1; k7_counters_mirrored = 0; break; } } else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { amd_num_counters = F10H_NUM_COUNTERS; amd_counters_base = MSR_K7_PERFCTR0; amd_ctrls_base = MSR_K7_EVNTSEL0; amd_msr_step = 1; k7_counters_mirrored = 0; } else { uint32_t eax, ebx, ecx, edx; cpuid(0xa, &eax, &ebx, &ecx, &edx); intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >> PMU_GENERAL_NR_SHIFT; intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >> PMU_FIXED_NR_SHIFT; } } static inline uint32_t get_fam15h_addr(u32 addr) { switch (addr) { case MSR_K7_PERFCTR0: case MSR_K7_PERFCTR1: case MSR_K7_PERFCTR2: case MSR_K7_PERFCTR3: return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0); case MSR_K7_EVNTSEL0: case MSR_K7_EVNTSEL1: case MSR_K7_EVNTSEL2: case MSR_K7_EVNTSEL3: return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0); default: break; } return addr; } static inline bool is_amd_pmu_msr(unsigned int msr) { if ((msr >= MSR_F15H_PERF_CTL && msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) || (msr >= MSR_K7_EVNTSEL0 && msr < MSR_K7_PERFCTR0 + amd_num_counters)) return true; return false; } static int is_intel_pmu_msr(u32 msr_index, int *type, int *index) { u32 msr_index_pmc; switch (msr_index) { case MSR_CORE_PERF_FIXED_CTR_CTRL: case MSR_IA32_DS_AREA: case MSR_IA32_PEBS_ENABLE: *type = MSR_TYPE_CTRL; return true; case MSR_CORE_PERF_GLOBAL_CTRL: case MSR_CORE_PERF_GLOBAL_STATUS: case MSR_CORE_PERF_GLOBAL_OVF_CTRL: *type = MSR_TYPE_GLOBAL; return true; default: if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) && (msr_index < MSR_CORE_PERF_FIXED_CTR0 + intel_num_fixed_counters)) { *index = msr_index - MSR_CORE_PERF_FIXED_CTR0; *type = MSR_TYPE_COUNTER; return true; } if ((msr_index >= MSR_P6_EVNTSEL0) && (msr_index < MSR_P6_EVNTSEL0 + intel_num_arch_counters)) { *index = msr_index - MSR_P6_EVNTSEL0; *type = MSR_TYPE_ARCH_CTRL; return true; } msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK; if ((msr_index_pmc >= MSR_IA32_PERFCTR0) && (msr_index_pmc < MSR_IA32_PERFCTR0 + intel_num_arch_counters)) { *type = MSR_TYPE_ARCH_COUNTER; *index = msr_index_pmc - MSR_IA32_PERFCTR0; return true; } return false; } } static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type, int index, bool is_read) { uint64_t *reg = NULL; struct xen_pmu_intel_ctxt *ctxt; uint64_t *fix_counters; struct xen_pmu_cntr_pair *arch_cntr_pair; struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); uint8_t xenpmu_flags = get_xenpmu_flags(); if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) return false; ctxt = &xenpmu_data->pmu.c.intel; switch (msr) { case MSR_CORE_PERF_GLOBAL_OVF_CTRL: reg = &ctxt->global_ovf_ctrl; break; case MSR_CORE_PERF_GLOBAL_STATUS: reg = &ctxt->global_status; break; case MSR_CORE_PERF_GLOBAL_CTRL: reg = &ctxt->global_ctrl; break; case MSR_CORE_PERF_FIXED_CTR_CTRL: reg = &ctxt->fixed_ctrl; break; default: switch (type) { case MSR_TYPE_COUNTER: fix_counters = field_offset(ctxt, fixed_counters); reg = &fix_counters[index]; break; case MSR_TYPE_ARCH_COUNTER: arch_cntr_pair = field_offset(ctxt, arch_counters); reg = &arch_cntr_pair[index].counter; break; case MSR_TYPE_ARCH_CTRL: arch_cntr_pair = field_offset(ctxt, arch_counters); reg = &arch_cntr_pair[index].control; break; default: return false; } } if (reg) { if (is_read) *val = *reg; else { *reg = *val; if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL) ctxt->global_status &= (~(*val)); } return true; } return false; } static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read) { uint64_t *reg = NULL; int i, off = 0; struct xen_pmu_amd_ctxt *ctxt; uint64_t *counter_regs, *ctrl_regs; struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); uint8_t xenpmu_flags = get_xenpmu_flags(); if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) return false; if (k7_counters_mirrored && ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3))) msr = get_fam15h_addr(msr); ctxt = &xenpmu_data->pmu.c.amd; for (i = 0; i < amd_num_counters; i++) { if (msr == amd_ctrls_base + off) { ctrl_regs = field_offset(ctxt, ctrls); reg = &ctrl_regs[i]; break; } else if (msr == amd_counters_base + off) { counter_regs = field_offset(ctxt, counters); reg = &counter_regs[i]; break; } off += amd_msr_step; } if (reg) { if (is_read) *val = *reg; else *reg = *val; return true; } return false; } bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err) { if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { if (is_amd_pmu_msr(msr)) { if (!xen_amd_pmu_emulate(msr, val, 1)) *val = native_read_msr_safe(msr, err); return true; } } else { int type, index; if (is_intel_pmu_msr(msr, &type, &index)) { if (!xen_intel_pmu_emulate(msr, val, type, index, 1)) *val = native_read_msr_safe(msr, err); return true; } } return false; } bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err) { uint64_t val = ((uint64_t)high << 32) | low; if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { if (is_amd_pmu_msr(msr)) { if (!xen_amd_pmu_emulate(msr, &val, 0)) *err = native_write_msr_safe(msr, low, high); return true; } } else { int type, index; if (is_intel_pmu_msr(msr, &type, &index)) { if (!xen_intel_pmu_emulate(msr, &val, type, index, 0)) *err = native_write_msr_safe(msr, low, high); return true; } } return false; } static unsigned long long xen_amd_read_pmc(int counter) { struct xen_pmu_amd_ctxt *ctxt; uint64_t *counter_regs; struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); uint8_t xenpmu_flags = get_xenpmu_flags(); if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) { uint32_t msr; int err; msr = amd_counters_base + (counter * amd_msr_step); return native_read_msr_safe(msr, &err); } ctxt = &xenpmu_data->pmu.c.amd; counter_regs = field_offset(ctxt, counters); return counter_regs[counter]; } static unsigned long long xen_intel_read_pmc(int counter) { struct xen_pmu_intel_ctxt *ctxt; uint64_t *fixed_counters; struct xen_pmu_cntr_pair *arch_cntr_pair; struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); uint8_t xenpmu_flags = get_xenpmu_flags(); if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) { uint32_t msr; int err; if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff); else msr = MSR_IA32_PERFCTR0 + counter; return native_read_msr_safe(msr, &err); } ctxt = &xenpmu_data->pmu.c.intel; if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) { fixed_counters = field_offset(ctxt, fixed_counters); return fixed_counters[counter & 0xffff]; } arch_cntr_pair = field_offset(ctxt, arch_counters); return arch_cntr_pair[counter].counter; } unsigned long long xen_read_pmc(int counter) { if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) return xen_amd_read_pmc(counter); else return xen_intel_read_pmc(counter); } int pmu_apic_update(uint32_t val) { int ret; struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); return -EINVAL; } xenpmu_data->pmu.l.lapic_lvtpc = val; if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING) return 0; ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL); return ret; } /* perf callbacks */ static int xen_is_in_guest(void) { const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); return 0; } if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF)) return 0; return 1; } static int xen_is_user_mode(void) { const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); return 0; } if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER); else return !!(xenpmu_data->pmu.r.regs.cpl & 3); } static unsigned long xen_get_guest_ip(void) { const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); return 0; } return xenpmu_data->pmu.r.regs.ip; } static struct perf_guest_info_callbacks xen_guest_cbs = { .is_in_guest = xen_is_in_guest, .is_user_mode = xen_is_user_mode, .get_guest_ip = xen_get_guest_ip, }; /* Convert registers from Xen's format to Linux' */ static void xen_convert_regs(const struct xen_pmu_regs *xen_regs, struct pt_regs *regs, uint64_t pmu_flags) { regs->ip = xen_regs->ip; regs->cs = xen_regs->cs; regs->sp = xen_regs->sp; if (pmu_flags & PMU_SAMPLE_PV) { if (pmu_flags & PMU_SAMPLE_USER) regs->cs |= 3; else regs->cs &= ~3; } else { if (xen_regs->cpl) regs->cs |= 3; else regs->cs &= ~3; } } irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id) { int err, ret = IRQ_NONE; struct pt_regs regs = {0}; const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); uint8_t xenpmu_flags = get_xenpmu_flags(); if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); return ret; } this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags | XENPMU_IRQ_PROCESSING; xen_convert_regs(&xenpmu_data->pmu.r.regs, ®s, xenpmu_data->pmu.pmu_flags); if (x86_pmu.handle_irq(®s)) ret = IRQ_HANDLED; /* Write out cached context to HW */ err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL); this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags; if (err) { pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err); return IRQ_NONE; } return ret; } bool is_xen_pmu(int cpu) { return (get_xenpmu_data() != NULL); } void xen_pmu_init(int cpu) { int err; struct xen_pmu_params xp; unsigned long pfn; struct xen_pmu_data *xenpmu_data; BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE); if (xen_hvm_domain()) return; xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL); if (!xenpmu_data) { pr_err("VPMU init: No memory\n"); return; } pfn = virt_to_pfn(xenpmu_data); xp.val = pfn_to_mfn(pfn); xp.vcpu = cpu; xp.version.maj = XENPMU_VER_MAJ; xp.version.min = XENPMU_VER_MIN; err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp); if (err) goto fail; per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data; per_cpu(xenpmu_shared, cpu).flags = 0; if (cpu == 0) { perf_register_guest_info_callbacks(&xen_guest_cbs); xen_pmu_arch_init(); } return; fail: if (err == -EOPNOTSUPP || err == -ENOSYS) pr_info_once("VPMU disabled by hypervisor.\n"); else pr_info_once("Could not initialize VPMU for cpu %d, error %d\n", cpu, err); free_pages((unsigned long)xenpmu_data, 0); } void xen_pmu_finish(int cpu) { struct xen_pmu_params xp; if (xen_hvm_domain()) return; xp.vcpu = cpu; xp.version.maj = XENPMU_VER_MAJ; xp.version.min = XENPMU_VER_MIN; (void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp); free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0); per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL; }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1