cregit-Linux how code gets into the kernel

Release 4.11 drivers/edac/pnd2_edac.c

Directory: drivers/edac
/*
 * Driver for Pondicherry2 memory controller.
 *
 * Copyright (c) 2016, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * [Derived from sb_edac.c]
 *
 * Translation of system physical addresses to DIMM addresses
 * is a two stage process:
 *
 * First the Pondicherry 2 memory controller handles slice and channel interleaving
 * in "sys2pmi()". This is (almost) completley common between platforms.
 *
 * Then a platform specific dunit (DIMM unit) completes the process to provide DIMM,
 * rank, bank, row and column using the appropriate "dunit_ops" functions/parameters.
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/edac.h>
#include <linux/mmzone.h>
#include <linux/smp.h>
#include <linux/bitmap.h>
#include <linux/math64.h>
#include <linux/mod_devicetable.h>
#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>
#include <asm/processor.h>
#include <asm/mce.h>

#include "edac_mc.h"
#include "edac_module.h"
#include "pnd2_edac.h"


#define APL_NUM_CHANNELS	4

#define DNV_NUM_CHANNELS	2

#define DNV_MAX_DIMMS		2 
/* Max DIMMs per channel */


enum type {
	
APL,
	
DNV, /* All requests go to PMI CH0 on each slice (CH1 disabled) */
};


struct dram_addr {
	
int chan;
	
int dimm;
	
int rank;
	
int bank;
	
int row;
	
int col;
};


struct pnd2_pvt {
	
int dimm_geom[APL_NUM_CHANNELS];
	

u64 tolm, tohm;
};

/*
 * System address space is divided into multiple regions with
 * different interleave rules in each. The as0/as1 regions
 * have no interleaving at all. The as2 region is interleaved
 * between two channels. The mot region is magic and may overlap
 * other regions, with its interleave rules taking precedence.
 * Addresses not in any of these regions are interleaved across
 * all four channels.
 */

static struct region {
	
u64	base;
	
u64	limit;
	
u8	enabled;
} 



mot, as0, as1, as2;


static struct dunit_ops {
	
char *name;
	
enum type type;
	
int pmiaddr_shift;
	
int pmiidx_shift;
	
int channels;
	
int dimms_per_channel;
	
int (*rd_reg)(int port, int off, int op, void *data, size_t sz, char *name);
	
int (*get_registers)(void);
	
int (*check_ecc)(void);
	
void (*mk_region)(char *name, struct region *rp, void *asym);
	
void (*get_dimm_config)(struct mem_ctl_info *mci);
	
int (*pmi2mem)(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx,
				   struct dram_addr *daddr, char *msg);
} 
*ops;


static struct mem_ctl_info *pnd2_mci;


#define PND2_MSG_SIZE	256

/* Debug macros */

#define pnd2_printk(level, fmt, arg...)			\
	edac_printk(level, "pnd2", fmt, ##arg)


#define pnd2_mc_printk(mci, level, fmt, arg...)	\
	edac_mc_chipset_printk(mci, level, "pnd2", fmt, ##arg)


#define MOT_CHAN_INTLV_BIT_1SLC_2CH 12

#define MOT_CHAN_INTLV_BIT_2SLC_2CH 13

#define SELECTOR_DISABLED (-1)

#define _4GB (1ul << 32)


#define PMI_ADDRESS_WIDTH	31

#define PND_MAX_PHYS_BIT	39


#define APL_ASYMSHIFT		28

#define DNV_ASYMSHIFT		31

#define CH_HASH_MASK_LSB	6

#define SLICE_HASH_MASK_LSB	6

#define MOT_SLC_INTLV_BIT	12

#define LOG2_PMI_ADDR_GRANULARITY	5

#define MOT_SHIFT	24


#define GET_BITFIELD(v, lo, hi)	(((v) & GENMASK_ULL(hi, lo)) >> (lo))

#define U64_LSHIFT(val, s)	((u64)(val) << (s))

#ifdef CONFIG_X86_INTEL_SBI_APL
#include "linux/platform_data/sbi_apl.h"



int sbi_send(int port, int off, int op, u32 *data)
{
	struct sbi_apl_message sbi_arg;
	int ret, read = 0;

	memset(&sbi_arg, 0, sizeof(sbi_arg));

	if (op == 0 || op == 4 || op == 6)
		read = 1;
	else
		sbi_arg.data = *data;

	sbi_arg.opcode = op;
	sbi_arg.port_address = port;
	sbi_arg.register_offset = off;
	ret = sbi_apl_commit(&sbi_arg);
	if (ret || sbi_arg.status)
		edac_dbg(2, "sbi_send status=%d ret=%d data=%x\n",
				 sbi_arg.status, ret, sbi_arg.data);

	if (ret == 0)
		ret = sbi_arg.status;

	if (ret == 0 && read)
		*data = sbi_arg.data;

	return ret;
}

Contributors
PersonTokensPropCommitsCommitProp
Tony Luck149100.00%1100.00%


Total149100.00%1100.00%



#else



int sbi_send(int port, int off, int op, u32 *data)
{
	return -EUNATCH;
}

Contributors
PersonTokensPropCommitsCommitProp
Tony Luck22100.00%1100.00%


Total22100.00%1100.00%



#endif




static int apl_rd_reg(int port, int off, int op, void *data, size_t sz, char *name)
{
	int	ret = 0;

	edac_dbg(2, "Read %s port=%x off=%x op=%x\n", name, port, off, op);
	switch (sz) {
	case 8:
		ret = sbi_send(port, off + 4, op, (u32 *)(data + 4));
	case 4:
		ret = sbi_send(port, off, op, (u32 *)data);
		pnd2_printk(KERN_DEBUG, "%s=%x%08x ret=%d\n", name,
					sz == 8 ? *((u32 *)(data + 4)) : 0, *((u32 *)data), ret);
		break;
	}

	return ret;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck141100.00%1100.00%


Total141100.00%1100.00%






static u64 get_mem_ctrl_hub_base_addr(void)
{
	struct b_cr_mchbar_lo_pci lo;
	struct b_cr_mchbar_hi_pci hi;
	struct pci_dev *pdev;

	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL);
	if (pdev) {
		pci_read_config_dword(pdev, 0x48, (u32 *)&lo);
		pci_read_config_dword(pdev, 0x4c, (u32 *)&hi);
		pci_dev_put(pdev);
	} else {
		return 0;
	}

	if (!lo.enable) {
		edac_dbg(2, "MMIO via memory controller hub base address is disabled!\n");
		return 0;
	}

	return U64_LSHIFT(hi.base, 32) | U64_LSHIFT(lo.base, 15);
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck115100.00%1100.00%


Total115100.00%1100.00%






static u64 get_sideband_reg_base_addr(void)
{
	struct pci_dev *pdev;
	u32 hi, lo;

	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x19dd, NULL);
	if (pdev) {
		pci_read_config_dword(pdev, 0x10, &lo);
		pci_read_config_dword(pdev, 0x14, &hi);
		pci_dev_put(pdev);
		return (U64_LSHIFT(hi, 32) | U64_LSHIFT(lo, 0));
	} else {
		return 0xfd000000;
	}
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck83100.00%1100.00%


Total83100.00%1100.00%






static int dnv_rd_reg(int port, int off, int op, void *data, size_t sz, char *name)
{
	struct pci_dev *pdev;
	char *base;
	u64 addr;

	if (op == 4) {
		pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL);
		if (!pdev)
			return -ENODEV;

		pci_read_config_dword(pdev, off, data);
		pci_dev_put(pdev);
	} else {
		/* MMIO via memory controller hub base address */
		if (op == 0 && port == 0x4c) {
			addr = get_mem_ctrl_hub_base_addr();
			if (!addr)
				return -ENODEV;
		} else {
			/* MMIO via sideband register base address */
			addr = get_sideband_reg_base_addr();
			if (!addr)
				return -ENODEV;
			addr += (port << 16);
		}

		base = ioremap((resource_size_t)addr, 0x10000);
		if (!base)
			return -ENODEV;

		if (sz == 8)
			*(u32 *)(data + 4) = *(u32 *)(base + off + 4);
		*(u32 *)data = *(u32 *)(base + off);

		iounmap(base);
	}

	edac_dbg(2, "Read %s=%.8x_%.8x\n", name,
			(sz == 8) ? *(u32 *)(data + 4) : 0, *(u32 *)data);

	return 0;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck248100.00%1100.00%


Total248100.00%1100.00%




#define RD_REGP(regp, regname, port)	\
	ops->rd_reg(port,                                       \
                regname##_offset,                               \
                regname##_r_opcode,                             \
                regp, sizeof(struct regname),   \
                #regname)


#define RD_REG(regp, regname)			\
	ops->rd_reg(regname ## _port,           \
                regname##_offset,                               \
                regname##_r_opcode,                             \
                regp, sizeof(struct regname),   \
                #regname)



static u64 top_lm, top_hm;

static bool two_slices;

static bool two_channels; 
/* Both PMI channels in one slice enabled */


static u8 sym_chan_mask;

static u8 asym_chan_mask;

static u8 chan_mask;


static int slice_selector = -1;

static int chan_selector = -1;

static u64 slice_hash_mask;

static u64 chan_hash_mask;




static void mk_region(char *name, struct region *rp, u64 base, u64 limit)
{
	rp->enabled = 1;
	rp->base = base;
	rp->limit = limit;
	edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, limit);
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck52100.00%1100.00%


Total52100.00%1100.00%






static void mk_region_mask(char *name, struct region *rp, u64 base, u64 mask)
{
	if (mask == 0) {
		pr_info(FW_BUG "MOT mask cannot be zero\n");
		return;
	}
	if (mask != GENMASK_ULL(PND_MAX_PHYS_BIT, __ffs(mask))) {
		pr_info(FW_BUG "MOT mask not power of two\n");
		return;
	}
	if (base & ~mask) {
		pr_info(FW_BUG "MOT region base/mask alignment error\n");
		return;
	}
	rp->base = base;
	rp->limit = (base | ~mask) & GENMASK_ULL(PND_MAX_PHYS_BIT, 0);
	rp->enabled = 1;
	edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, rp->limit);
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck120100.00%1100.00%


Total120100.00%1100.00%






static bool in_region(struct region *rp, u64 addr)
{
	if (!rp->enabled)
		return false;

	return rp->base <= addr && addr <= rp->limit;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck37100.00%1100.00%


Total37100.00%1100.00%






static int gen_sym_mask(struct b_cr_slice_channel_hash *p)
{
	int mask = 0;

	if (!p->slice_0_mem_disabled)
		mask |= p->sym_slice0_channel_enabled;

	if (!p->slice_1_disabled)
		mask |= p->sym_slice1_channel_enabled << 2;

	if (p->ch_1_disabled || p->enable_pmi_dual_data_mode)
		mask &= 0x5;

	return mask;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck61100.00%1100.00%


Total61100.00%1100.00%






static int gen_asym_mask(struct b_cr_slice_channel_hash *p,
			 struct b_cr_asym_mem_region0_mchbar *as0,
			 struct b_cr_asym_mem_region1_mchbar *as1,
			 struct b_cr_asym_2way_mem_region_mchbar *as2way)
{
	const int intlv[] = { 0x5, 0xA, 0x3, 0xC };
	int mask = 0;

	if (as2way->asym_2way_interleave_enable)
		mask = intlv[as2way->asym_2way_intlv_mode];
	if (as0->slice0_asym_enable)
		mask |= (1 << as0->slice0_asym_channel_select);
	if (as1->slice1_asym_enable)
		mask |= (4 << as1->slice1_asym_channel_select);
	if (p->slice_0_mem_disabled)
		mask &= 0xc;
	if (p->slice_1_disabled)
		mask &= 0x3;
	if (p->ch_1_disabled || p->enable_pmi_dual_data_mode)
		mask &= 0x5;

	return mask;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck130100.00%1100.00%


Total130100.00%1100.00%




static struct b_cr_tolud_pci tolud;

static struct b_cr_touud_lo_pci touud_lo;

static struct b_cr_touud_hi_pci touud_hi;

static struct b_cr_asym_mem_region0_mchbar asym0;

static struct b_cr_asym_mem_region1_mchbar asym1;

static struct b_cr_asym_2way_mem_region_mchbar asym_2way;

static struct b_cr_mot_out_base_mchbar mot_base;

static struct b_cr_mot_out_mask_mchbar mot_mask;

static struct b_cr_slice_channel_hash chash;

/* Apollo Lake dunit */
/*
 * Validated on board with just two DIMMs in the [0] and [2] positions
 * in this array. Other port number matches documentation, but caution
 * advised.
 */

static const int apl_dports[APL_NUM_CHANNELS] = { 0x18, 0x10, 0x11, 0x19 };

static struct d_cr_drp0 drp0[APL_NUM_CHANNELS];

/* Denverton dunit */

static const int dnv_dports[DNV_NUM_CHANNELS] = { 0x10, 0x12 };

static struct d_cr_dsch dsch;

static struct d_cr_ecc_ctrl ecc_ctrl[DNV_NUM_CHANNELS];

static struct d_cr_drp drp[DNV_NUM_CHANNELS];

static struct d_cr_dmap dmap[DNV_NUM_CHANNELS];

static struct d_cr_dmap1 dmap1[DNV_NUM_CHANNELS];

static struct d_cr_dmap2 dmap2[DNV_NUM_CHANNELS];

static struct d_cr_dmap3 dmap3[DNV_NUM_CHANNELS];

static struct d_cr_dmap4 dmap4[DNV_NUM_CHANNELS];

static struct d_cr_dmap5 dmap5[DNV_NUM_CHANNELS];




static void apl_mk_region(char *name, struct region *rp, void *asym)
{
	struct b_cr_asym_mem_region0_mchbar *a = asym;

	mk_region(name, rp,
			  U64_LSHIFT(a->slice0_asym_base, APL_ASYMSHIFT),
			  U64_LSHIFT(a->slice0_asym_limit, APL_ASYMSHIFT) +
			  GENMASK_ULL(APL_ASYMSHIFT - 1, 0));
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck60100.00%1100.00%


Total60100.00%1100.00%






static void dnv_mk_region(char *name, struct region *rp, void *asym)
{
	struct b_cr_asym_mem_region_denverton *a = asym;

	mk_region(name, rp,
			  U64_LSHIFT(a->slice_asym_base, DNV_ASYMSHIFT),
			  U64_LSHIFT(a->slice_asym_limit, DNV_ASYMSHIFT) +
			  GENMASK_ULL(DNV_ASYMSHIFT - 1, 0));
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck60100.00%1100.00%


Total60100.00%1100.00%






static int apl_get_registers(void)
{
	int i;

	if (RD_REG(&asym_2way, b_cr_asym_2way_mem_region_mchbar))
		return -ENODEV;

	for (i = 0; i < APL_NUM_CHANNELS; i++)
		if (RD_REGP(&drp0[i], d_cr_drp0, apl_dports[i]))
			return -ENODEV;

	return 0;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck63100.00%1100.00%


Total63100.00%1100.00%






static int dnv_get_registers(void)
{
	int i;

	if (RD_REG(&dsch, d_cr_dsch))
		return -ENODEV;

	for (i = 0; i < DNV_NUM_CHANNELS; i++)
		if (RD_REGP(&ecc_ctrl[i], d_cr_ecc_ctrl, dnv_dports[i]) ||
			RD_REGP(&drp[i], d_cr_drp, dnv_dports[i]) ||
			RD_REGP(&dmap[i], d_cr_dmap, dnv_dports[i]) ||
			RD_REGP(&dmap1[i], d_cr_dmap1, dnv_dports[i]) ||
			RD_REGP(&dmap2[i], d_cr_dmap2, dnv_dports[i]) ||
			RD_REGP(&dmap3[i], d_cr_dmap3, dnv_dports[i]) ||
			RD_REGP(&dmap4[i], d_cr_dmap4, dnv_dports[i]) ||
			RD_REGP(&dmap5[i], d_cr_dmap5, dnv_dports[i]))
			return -ENODEV;

	return 0;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck175100.00%1100.00%


Total175100.00%1100.00%



/*
 * Read all the h/w config registers once here (they don't
 * change at run time. Figure out which address ranges have
 * which interleave characteristics.
 */



static int get_registers(void)
{
	const int intlv[] = { 10, 11, 12, 12 };

	if (RD_REG(&tolud, b_cr_tolud_pci) ||
		RD_REG(&touud_lo, b_cr_touud_lo_pci) ||
		RD_REG(&touud_hi, b_cr_touud_hi_pci) ||
		RD_REG(&asym0, b_cr_asym_mem_region0_mchbar) ||
		RD_REG(&asym1, b_cr_asym_mem_region1_mchbar) ||
		RD_REG(&mot_base, b_cr_mot_out_base_mchbar) ||
		RD_REG(&mot_mask, b_cr_mot_out_mask_mchbar) ||
		RD_REG(&chash, b_cr_slice_channel_hash))
		return -ENODEV;

	if (ops->get_registers())
		return -ENODEV;

	if (ops->type == DNV) {
		/* PMI channel idx (always 0) for asymmetric region */
		asym0.slice0_asym_channel_select = 0;
		asym1.slice1_asym_channel_select = 0;
		/* PMI channel bitmap (always 1) for symmetric region */
		chash.sym_slice0_channel_enabled = 0x1;
		chash.sym_slice1_channel_enabled = 0x1;
	}

	if (asym0.slice0_asym_enable)
		ops->mk_region("as0", &as0, &asym0);

	if (asym1.slice1_asym_enable)
		ops->mk_region("as1", &as1, &asym1);

	if (asym_2way.asym_2way_interleave_enable) {
		mk_region("as2way", &as2,
				  U64_LSHIFT(asym_2way.asym_2way_base, APL_ASYMSHIFT),
				  U64_LSHIFT(asym_2way.asym_2way_limit, APL_ASYMSHIFT) +
				  GENMASK_ULL(APL_ASYMSHIFT - 1, 0));
	}

	if (mot_base.imr_en) {
		mk_region_mask("mot", &mot,
					   U64_LSHIFT(mot_base.mot_out_base, MOT_SHIFT),
					   U64_LSHIFT(mot_mask.mot_out_mask, MOT_SHIFT));
	}

	top_lm = U64_LSHIFT(tolud.tolud, 20);
	top_hm = U64_LSHIFT(touud_hi.touud, 32) | U64_LSHIFT(touud_lo.touud, 20);

	two_slices = !chash.slice_1_disabled &&
				 !chash.slice_0_mem_disabled &&
				 (chash.sym_slice0_channel_enabled != 0) &&
				 (chash.sym_slice1_channel_enabled != 0);
	two_channels = !chash.ch_1_disabled &&
				 !chash.enable_pmi_dual_data_mode &&
				 ((chash.sym_slice0_channel_enabled == 3) ||
				 (chash.sym_slice1_channel_enabled == 3));

	sym_chan_mask = gen_sym_mask(&chash);
	asym_chan_mask = gen_asym_mask(&chash, &asym0, &asym1, &asym_2way);
	chan_mask = sym_chan_mask | asym_chan_mask;

	if (two_slices && !two_channels) {
		if (chash.hvm_mode)
			slice_selector = 29;
		else
			slice_selector = intlv[chash.interleave_mode];
	} else if (!two_slices && two_channels) {
		if (chash.hvm_mode)
			chan_selector = 29;
		else
			chan_selector = intlv[chash.interleave_mode];
	} else if (two_slices && two_channels) {
		if (chash.hvm_mode) {
			slice_selector = 29;
			chan_selector = 30;
		} else {
			slice_selector = intlv[chash.interleave_mode];
			chan_selector = intlv[chash.interleave_mode] + 1;
		}
	}

	if (two_slices) {
		if (!chash.hvm_mode)
			slice_hash_mask = chash.slice_hash_mask << SLICE_HASH_MASK_LSB;
		if (!two_channels)
			slice_hash_mask |= BIT_ULL(slice_selector);
	}

	if (two_channels) {
		if (!chash.hvm_mode)
			chan_hash_mask = chash.ch_hash_mask << CH_HASH_MASK_LSB;
		if (!two_slices)
			chan_hash_mask |= BIT_ULL(chan_selector);
	}

	return 0;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck551100.00%1100.00%


Total551100.00%1100.00%



/* Get a contiguous memory address (remove the MMIO gap) */



static u64 remove_mmio_gap(u64 sys)
{
	return (sys < _4GB) ? sys : sys - (_4GB - top_lm);
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck26100.00%1100.00%


Total26100.00%1100.00%



/* Squeeze out one address bit, shift upper part down to fill gap */



static void remove_addr_bit(u64 *addr, int bitidx)
{
	u64	mask;

	if (bitidx == -1)
		return;

	mask = (1ull << bitidx) - 1;
	*addr = ((*addr >> 1) & ~mask) | (*addr & mask);
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck56100.00%1100.00%


Total56100.00%1100.00%



/* XOR all the bits from addr specified in mask */



static int hash_by_mask(u64 addr, u64 mask)
{
	u64 result = addr & mask;

	result = (result >> 32) ^ result;
	result = (result >> 16) ^ result;
	result = (result >> 8) ^ result;
	result = (result >> 4) ^ result;
	result = (result >> 2) ^ result;
	result = (result >> 1) ^ result;

	return (int)result & 1;
}


Contributors
PersonTokensPropCommitsCommitProp
Tony Luck87100.00%1100.00%


Total87100.00%1100.00%



/*
 * First stage decode. Take the system address and figure out which
 * second stage will deal with it based on interleave modes.
 */



static int sys2pmi(const u64 addr, u32 *pmiidx, u64 *pmiaddr, char *msg)
{
	u64 contig_addr, contig_base, contig_offset,