// SPDX-License-Identifier: GPL-2.0 /* * Intel specific MCE features. * Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca> * Copyright (C) 2008, 2009 Intel Corporation * Author: Andi Kleen */ #include <linux/gfp.h> #include <linux/interrupt.h> #include <linux/percpu.h> #include <linux/sched.h> #include <linux/cpumask.h> #include <asm/apic.h> #include <asm/cpufeature.h> #include <asm/intel-family.h> #include <asm/processor.h> #include <asm/msr.h> #include <asm/mce.h> #include "internal.h" /* * Support for Intel Correct Machine Check Interrupts. This allows * the CPU to raise an interrupt when a corrected machine check happened. * Normally we pick those up using a regular polling timer. * Also supports reliable discovery of shared banks. */ /* * CMCI can be delivered to multiple cpus that share a machine check bank * so we need to designate a single cpu to process errors logged in each bank * in the interrupt handler (otherwise we would have many races and potential * double reporting of the same error). * Note that this can change when a cpu is offlined or brought online since * some MCA banks are shared across cpus. When a cpu is offlined, cmci_clear() * disables CMCI on all banks owned by the cpu and clears this bitfield. At * this point, cmci_rediscover() kicks in and a different cpu may end up * taking ownership of some of the shared MCA banks that were previously * owned by the offlined cpu. */ static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned); /* * CMCI storm detection backoff counter * * During storm, we reset this counter to INITIAL_CHECK_INTERVAL in case we've * encountered an error. If not, we decrement it by one. We signal the end of * the CMCI storm when it reaches 0. */ static DEFINE_PER_CPU(int, cmci_backoff_cnt); /* * cmci_discover_lock protects against parallel discovery attempts * which could race against each other. */ static DEFINE_RAW_SPINLOCK(cmci_discover_lock); /* * On systems that do support CMCI but it's disabled, polling for MCEs can * cause the same event to be reported multiple times because IA32_MCi_STATUS * is shared by the same package. */ static DEFINE_SPINLOCK(cmci_poll_lock); #define CMCI_THRESHOLD 1 #define CMCI_POLL_INTERVAL (30 * HZ) #define CMCI_STORM_INTERVAL (HZ) #define CMCI_STORM_THRESHOLD 15 static DEFINE_PER_CPU(unsigned long, cmci_time_stamp); static DEFINE_PER_CPU(unsigned int, cmci_storm_cnt); static DEFINE_PER_CPU(unsigned int, cmci_storm_state); enum { CMCI_STORM_NONE, CMCI_STORM_ACTIVE, CMCI_STORM_SUBSIDED, }; static atomic_t cmci_storm_on_cpus; static int cmci_supported(int *banks) { u64 cap; if (mca_cfg.cmci_disabled || mca_cfg.ignore_ce) return 0; /* * Vendor check is not strictly needed, but the initial * initialization is vendor keyed and this * makes sure none of the backdoors are entered otherwise. */ if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL && boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN) return 0; if (!boot_cpu_has(X86_FEATURE_APIC) || lapic_get_maxlvt() < 6) return 0; rdmsrl(MSR_IA32_MCG_CAP, cap); *banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff); return !!(cap & MCG_CMCI_P); } static bool lmce_supported(void) { u64 tmp; if (mca_cfg.lmce_disabled) return false; rdmsrl(MSR_IA32_MCG_CAP, tmp); /* * LMCE depends on recovery support in the processor. Hence both * MCG_SER_P and MCG_LMCE_P should be present in MCG_CAP. */ if ((tmp & (MCG_SER_P | MCG_LMCE_P)) != (MCG_SER_P | MCG_LMCE_P)) return false; /* * BIOS should indicate support for LMCE by setting bit 20 in * IA32_FEAT_CTL without which touching MCG_EXT_CTL will generate a #GP * fault. The MSR must also be locked for LMCE_ENABLED to take effect. * WARN if the MSR isn't locked as init_ia32_feat_ctl() unconditionally * locks the MSR in the event that it wasn't already locked by BIOS. */ rdmsrl(MSR_IA32_FEAT_CTL, tmp); if (WARN_ON_ONCE(!(tmp & FEAT_CTL_LOCKED))) return false; return tmp & FEAT_CTL_LMCE_ENABLED; } bool mce_intel_cmci_poll(void) { if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE) return false; /* * Reset the counter if we've logged an error in the last poll * during the storm. */ if (machine_check_poll(0, this_cpu_ptr(&mce_banks_owned))) this_cpu_write(cmci_backoff_cnt, INITIAL_CHECK_INTERVAL); else this_cpu_dec(cmci_backoff_cnt); return true; } void mce_intel_hcpu_update(unsigned long cpu) { if (per_cpu(cmci_storm_state, cpu) == CMCI_STORM_ACTIVE) atomic_dec(&cmci_storm_on_cpus); per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE; } static void cmci_toggle_interrupt_mode(bool on) { unsigned long flags, *owned; int bank; u64 val; raw_spin_lock_irqsave(&cmci_discover_lock, flags); owned = this_cpu_ptr(mce_banks_owned); for_each_set_bit(bank, owned, MAX_NR_BANKS) { rdmsrl(MSR_IA32_MCx_CTL2(bank), val); if (on) val |= MCI_CTL2_CMCI_EN; else val &= ~MCI_CTL2_CMCI_EN; wrmsrl(MSR_IA32_MCx_CTL2(bank), val); } raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); } unsigned long cmci_intel_adjust_timer(unsigned long interval) { if ((this_cpu_read(cmci_backoff_cnt) > 0) && (__this_cpu_read(cmci_storm_state) == CMCI_STORM_ACTIVE)) { mce_notify_irq(); return CMCI_STORM_INTERVAL; } switch (__this_cpu_read(cmci_storm_state)) { case CMCI_STORM_ACTIVE: /* * We switch back to interrupt mode once the poll timer has * silenced itself. That means no events recorded and the timer * interval is back to our poll interval. */ __this_cpu_write(cmci_storm_state, CMCI_STORM_SUBSIDED); if (!atomic_sub_return(1, &cmci_storm_on_cpus)) pr_notice("CMCI storm subsided: switching to interrupt mode\n"); fallthrough; case CMCI_STORM_SUBSIDED: /* * We wait for all CPUs to go back to SUBSIDED state. When that * happens we switch back to interrupt mode. */ if (!atomic_read(&cmci_storm_on_cpus)) { __this_cpu_write(cmci_storm_state, CMCI_STORM_NONE); cmci_toggle_interrupt_mode(true); cmci_recheck(); } return CMCI_POLL_INTERVAL; default: /* We have shiny weather. Let the poll do whatever it thinks. */ return interval; } } static bool cmci_storm_detect(void) { unsigned int cnt = __this_cpu_read(cmci_storm_cnt); unsigned long ts = __this_cpu_read(cmci_time_stamp); unsigned long now = jiffies; int r; if (__this_cpu_read(cmci_storm_state) != CMCI_STORM_NONE) return true; if (time_before_eq(now, ts + CMCI_STORM_INTERVAL)) { cnt++; } else { cnt = 1; __this_cpu_write(cmci_time_stamp, now); } __this_cpu_write(cmci_storm_cnt, cnt); if (cnt <= CMCI_STORM_THRESHOLD) return false; cmci_toggle_interrupt_mode(false); __this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE); r = atomic_add_return(1, &cmci_storm_on_cpus); mce_timer_kick(CMCI_STORM_INTERVAL); this_cpu_write(cmci_backoff_cnt, INITIAL_CHECK_INTERVAL); if (r == 1) pr_notice("CMCI storm detected: switching to poll mode\n"); return true; } /* * The interrupt handler. This is called on every event. * Just call the poller directly to log any events. * This could in theory increase the threshold under high load, * but doesn't for now. */ static void intel_threshold_interrupt(void) { if (cmci_storm_detect()) return; machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_banks_owned)); } /* * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks * on this CPU. Use the algorithm recommended in the SDM to discover shared * banks. */ static void cmci_discover(int banks) { unsigned long *owned = (void *)this_cpu_ptr(&mce_banks_owned); unsigned long flags; int i; int bios_wrong_thresh = 0; raw_spin_lock_irqsave(&cmci_discover_lock, flags); for (i = 0; i < banks; i++) { u64 val; int bios_zero_thresh = 0; if (test_bit(i, owned)) continue; /* Skip banks in firmware first mode */ if (test_bit(i, mce_banks_ce_disabled)) continue; rdmsrl(MSR_IA32_MCx_CTL2(i), val); /* Already owned by someone else? */ if (val & MCI_CTL2_CMCI_EN) { clear_bit(i, owned); __clear_bit(i, this_cpu_ptr(mce_poll_banks)); continue; } if (!mca_cfg.bios_cmci_threshold) { val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK; val |= CMCI_THRESHOLD; } else if (!(val & MCI_CTL2_CMCI_THRESHOLD_MASK)) { /* * If bios_cmci_threshold boot option was specified * but the threshold is zero, we'll try to initialize * it to 1. */ bios_zero_thresh = 1; val |= CMCI_THRESHOLD; } val |= MCI_CTL2_CMCI_EN; wrmsrl(MSR_IA32_MCx_CTL2(i), val); rdmsrl(MSR_IA32_MCx_CTL2(i), val); /* Did the enable bit stick? -- the bank supports CMCI */ if (val & MCI_CTL2_CMCI_EN) { set_bit(i, owned); __clear_bit(i, this_cpu_ptr(mce_poll_banks)); /* * We are able to set thresholds for some banks that * had a threshold of 0. This means the BIOS has not * set the thresholds properly or does not work with * this boot option. Note down now and report later. */ if (mca_cfg.bios_cmci_threshold && bios_zero_thresh && (val & MCI_CTL2_CMCI_THRESHOLD_MASK)) bios_wrong_thresh = 1; } else { WARN_ON(!test_bit(i, this_cpu_ptr(mce_poll_banks))); } } raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); if (mca_cfg.bios_cmci_threshold && bios_wrong_thresh) { pr_info_once( "bios_cmci_threshold: Some banks do not have valid thresholds set\n"); pr_info_once( "bios_cmci_threshold: Make sure your BIOS supports this boot option\n"); } } /* * Just in case we missed an event during initialization check * all the CMCI owned banks. */ void cmci_recheck(void) { unsigned long flags; int banks; if (!mce_available(raw_cpu_ptr(&cpu_info)) || !cmci_supported(&banks)) return; local_irq_save(flags); machine_check_poll(0, this_cpu_ptr(&mce_banks_owned)); local_irq_restore(flags); } /* Caller must hold the lock on cmci_discover_lock */ static void __cmci_disable_bank(int bank) { u64 val; if (!test_bit(bank, this_cpu_ptr(mce_banks_owned))) return; rdmsrl(MSR_IA32_MCx_CTL2(bank), val); val &= ~MCI_CTL2_CMCI_EN; wrmsrl(MSR_IA32_MCx_CTL2(bank), val); __clear_bit(bank, this_cpu_ptr(mce_banks_owned)); } /* * Disable CMCI on this CPU for all banks it owns when it goes down. * This allows other CPUs to claim the banks on rediscovery. */ void cmci_clear(void) { unsigned long flags; int i; int banks; if (!cmci_supported(&banks)) return; raw_spin_lock_irqsave(&cmci_discover_lock, flags); for (i = 0; i < banks; i++) __cmci_disable_bank(i); raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); } static void cmci_rediscover_work_func(void *arg) { int banks; /* Recheck banks in case CPUs don't all have the same */ if (cmci_supported(&banks)) cmci_discover(banks); } /* After a CPU went down cycle through all the others and rediscover */ void cmci_rediscover(void) { int banks; if (!cmci_supported(&banks)) return; on_each_cpu(cmci_rediscover_work_func, NULL, 1); } /* * Reenable CMCI on this CPU in case a CPU down failed. */ void cmci_reenable(void) { int banks; if (cmci_supported(&banks)) cmci_discover(banks); } void cmci_disable_bank(int bank) { int banks; unsigned long flags; if (!cmci_supported(&banks)) return; raw_spin_lock_irqsave(&cmci_discover_lock, flags); __cmci_disable_bank(bank); raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); } /* Bank polling function when CMCI is disabled. */ static void cmci_mc_poll_banks(void) { spin_lock(&cmci_poll_lock); machine_check_poll(0, this_cpu_ptr(&mce_poll_banks)); spin_unlock(&cmci_poll_lock); } void intel_init_cmci(void) { int banks; if (!cmci_supported(&banks)) { mc_poll_banks = cmci_mc_poll_banks; return; } mce_threshold_vector = intel_threshold_interrupt; cmci_discover(banks); /* * For CPU #0 this runs with still disabled APIC, but that's * ok because only the vector is set up. We still do another * check for the banks later for CPU #0 just to make sure * to not miss any events. */ apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR|APIC_DM_FIXED); cmci_recheck(); } void intel_init_lmce(void) { u64 val; if (!lmce_supported()) return; rdmsrl(MSR_IA32_MCG_EXT_CTL, val); if (!(val & MCG_EXT_CTL_LMCE_EN)) wrmsrl(MSR_IA32_MCG_EXT_CTL, val | MCG_EXT_CTL_LMCE_EN); } void intel_clear_lmce(void) { u64 val; if (!lmce_supported()) return; rdmsrl(MSR_IA32_MCG_EXT_CTL, val); val &= ~MCG_EXT_CTL_LMCE_EN; wrmsrl(MSR_IA32_MCG_EXT_CTL, val); } /* * Enable additional error logs from the integrated * memory controller on processors that support this. */ static void intel_imc_init(struct cpuinfo_x86 *c) { u64 error_control; switch (c->x86_model) { case INTEL_FAM6_SANDYBRIDGE_X: case INTEL_FAM6_IVYBRIDGE_X: case INTEL_FAM6_HASWELL_X: if (rdmsrl_safe(MSR_ERROR_CONTROL, &error_control)) return; error_control |= 2; wrmsrl_safe(MSR_ERROR_CONTROL, error_control); break; } } void mce_intel_feature_init(struct cpuinfo_x86 *c) { intel_init_cmci(); intel_init_lmce(); intel_imc_init(c); } void mce_intel_feature_clear(struct cpuinfo_x86 *c) { intel_clear_lmce(); } bool intel_filter_mce(struct mce *m) { struct cpuinfo_x86 *c = &boot_cpu_data; /* MCE errata HSD131, HSM142, HSW131, BDM48, HSM142 and SKX37 */ if ((c->x86 == 6) && ((c->x86_model == INTEL_FAM6_HASWELL) || (c->x86_model == INTEL_FAM6_HASWELL_L) || (c->x86_model == INTEL_FAM6_BROADWELL) || (c->x86_model == INTEL_FAM6_HASWELL_G) || (c->x86_model == INTEL_FAM6_SKYLAKE_X)) && (m->bank == 0) && ((m->status & 0xa0000000ffffffff) == 0x80000000000f0005)) return true; return false; } /* * Check if the address reported by the CPU is in a format we can parse. * It would be possible to add code for most other cases, but all would * be somewhat complicated (e.g. segment offset would require an instruction * parser). So only support physical addresses up to page granularity for now. */ bool intel_mce_usable_address(struct mce *m) { if (!(m->status & MCI_STATUS_MISCV)) return false; if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT) return false; if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS) return false; return true; }