| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Vidya Sagar | 7614 | 100.00% | 3 | 100.00% |
| Total | 7614 | 3 |
// SPDX-License-Identifier: GPL-2.0+ /* * PCIe host controller driver for Tegra194 SoC * * Copyright (C) 2019 NVIDIA Corporation. * * Author: Vidya Sagar <vidyas@nvidia.com> */ #include <linux/clk.h> #include <linux/debugfs.h> #include <linux/delay.h> #include <linux/gpio.h> #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_device.h> #include <linux/of_gpio.h> #include <linux/of_irq.h> #include <linux/of_pci.h> #include <linux/pci.h> #include <linux/phy/phy.h> #include <linux/pinctrl/consumer.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/random.h> #include <linux/reset.h> #include <linux/resource.h> #include <linux/types.h> #include "pcie-designware.h" #include <soc/tegra/bpmp.h> #include <soc/tegra/bpmp-abi.h> #include "../../pci.h" #define APPL_PINMUX 0x0 #define APPL_PINMUX_PEX_RST BIT(0) #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3) #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4) #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5) #define APPL_PINMUX_CLKREQ_OUT_OVRD_EN BIT(9) #define APPL_PINMUX_CLKREQ_OUT_OVRD BIT(10) #define APPL_CTRL 0x4 #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6) #define APPL_CTRL_LTSSM_EN BIT(7) #define APPL_CTRL_HW_HOT_RST_EN BIT(20) #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 #define APPL_INTR_EN_L0_0 0x8 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4) #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8) #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19) #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30) #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31) #define APPL_INTR_STATUS_L0 0xC #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0) #define APPL_INTR_STATUS_L0_INT_INT BIT(8) #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18) #define APPL_INTR_EN_L1_0_0 0x1C #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1) #define APPL_INTR_STATUS_L1_0_0 0x20 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1) #define APPL_INTR_STATUS_L1_1 0x2C #define APPL_INTR_STATUS_L1_2 0x30 #define APPL_INTR_STATUS_L1_3 0x34 #define APPL_INTR_STATUS_L1_6 0x3C #define APPL_INTR_STATUS_L1_7 0x40 #define APPL_INTR_EN_L1_8_0 0x44 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2) #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3) #define APPL_INTR_EN_L1_8_INTX_EN BIT(11) #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15) #define APPL_INTR_STATUS_L1_8_0 0x4C #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6) #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2) #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3) #define APPL_INTR_STATUS_L1_9 0x54 #define APPL_INTR_STATUS_L1_10 0x58 #define APPL_INTR_STATUS_L1_11 0x64 #define APPL_INTR_STATUS_L1_13 0x74 #define APPL_INTR_STATUS_L1_14 0x78 #define APPL_INTR_STATUS_L1_15 0x7C #define APPL_INTR_STATUS_L1_17 0x88 #define APPL_INTR_EN_L1_18 0x90 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2) #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) #define APPL_INTR_STATUS_L1_18 0x94 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2) #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) #define APPL_MSI_CTRL_2 0xB0 #define APPL_LTR_MSG_1 0xC4 #define LTR_MSG_REQ BIT(15) #define LTR_MST_NO_SNOOP_SHIFT 16 #define APPL_LTR_MSG_2 0xC8 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) #define APPL_LINK_STATUS 0xCC #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0) #define APPL_DEBUG 0xD0 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21) #define APPL_DEBUG_PM_LINKST_IN_L0 0x11 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) #define APPL_DEBUG_LTSSM_STATE_SHIFT 3 #define LTSSM_STATE_PRE_DETECT 5 #define APPL_RADM_STATUS 0xE4 #define APPL_PM_XMT_TURNOFF_STATE BIT(0) #define APPL_DM_TYPE 0x100 #define APPL_DM_TYPE_MASK GENMASK(3, 0) #define APPL_DM_TYPE_RP 0x4 #define APPL_DM_TYPE_EP 0x0 #define APPL_CFG_BASE_ADDR 0x104 #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12) #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18) #define APPL_CFG_MISC 0x110 #define APPL_CFG_MISC_SLV_EP_MODE BIT(14) #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10) #define APPL_CFG_MISC_ARCACHE_SHIFT 10 #define APPL_CFG_MISC_ARCACHE_VAL 3 #define APPL_CFG_SLCG_OVERRIDE 0x114 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0) #define APPL_CAR_RESET_OVRD 0x12C #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0) #define IO_BASE_IO_DECODE BIT(0) #define IO_BASE_IO_DECODE_BIT8 BIT(8) #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0) #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16) #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) #define EVENT_COUNTER_ALL_CLEAR 0x3 #define EVENT_COUNTER_ENABLE_ALL 0x7 #define EVENT_COUNTER_ENABLE_SHIFT 2 #define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0) #define EVENT_COUNTER_EVENT_SEL_SHIFT 16 #define EVENT_COUNTER_EVENT_Tx_L0S 0x2 #define EVENT_COUNTER_EVENT_Rx_L0S 0x3 #define EVENT_COUNTER_EVENT_L1 0x5 #define EVENT_COUNTER_EVENT_L1_1 0x7 #define EVENT_COUNTER_EVENT_L1_2 0x8 #define EVENT_COUNTER_GROUP_SEL_SHIFT 24 #define EVENT_COUNTER_GROUP_5 0x5 #define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C #define ENTER_ASPM BIT(30) #define L0S_ENTRANCE_LAT_SHIFT 24 #define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) #define L1_ENTRANCE_LAT_SHIFT 27 #define L1_ENTRANCE_LAT_MASK GENMASK(29, 27) #define N_FTS_SHIFT 8 #define N_FTS_MASK GENMASK(7, 0) #define N_FTS_VAL 52 #define PORT_LOGIC_GEN2_CTRL 0x80C #define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17) #define FTS_MASK GENMASK(7, 0) #define FTS_VAL 52 #define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828 #define GEN3_EQ_CONTROL_OFF 0x8a8 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) #define GEN3_RELATED_OFF 0x890 #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0) #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) #define AMBA_ERROR_RESPONSE_CRS_OKAY 0 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 #define PORT_LOGIC_MSIX_DOORBELL 0x948 #define CAP_SPCIE_CAP_OFF 0x154 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0) #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 #define PME_ACK_TIMEOUT 10000 #define LTSSM_TIMEOUT 50000 /* 50ms */ #define GEN3_GEN4_EQ_PRESET_INIT 5 #define GEN1_CORE_CLK_FREQ 62500000 #define GEN2_CORE_CLK_FREQ 125000000 #define GEN3_CORE_CLK_FREQ 250000000 #define GEN4_CORE_CLK_FREQ 500000000 static const unsigned int pcie_gen_freq[] = { GEN1_CORE_CLK_FREQ, GEN2_CORE_CLK_FREQ, GEN3_CORE_CLK_FREQ, GEN4_CORE_CLK_FREQ }; static const u32 event_cntr_ctrl_offset[] = { 0x1d8, 0x1a8, 0x1a8, 0x1a8, 0x1c4, 0x1d8 }; static const u32 event_cntr_data_offset[] = { 0x1dc, 0x1ac, 0x1ac, 0x1ac, 0x1c8, 0x1dc }; struct tegra_pcie_dw { struct device *dev; struct resource *appl_res; struct resource *dbi_res; struct resource *atu_dma_res; void __iomem *appl_base; struct clk *core_clk; struct reset_control *core_apb_rst; struct reset_control *core_rst; struct dw_pcie pci; struct tegra_bpmp *bpmp; bool supports_clkreq; bool enable_cdm_check; bool link_state; bool update_fc_fixup; u8 init_link_width; u32 msi_ctrl_int; u32 num_lanes; u32 max_speed; u32 cid; u32 cfg_link_cap_l1sub; u32 pcie_cap_base; u32 aspm_cmrt; u32 aspm_pwr_on_t; u32 aspm_l0s_enter_lat; struct regulator *pex_ctl_supply; struct regulator *slot_ctl_3v3; struct regulator *slot_ctl_12v; unsigned int phy_count; struct phy **phys; struct dentry *debugfs; }; static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) { return container_of(pci, struct tegra_pcie_dw, pci); } static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value, const u32 reg) { writel_relaxed(value, pcie->appl_base + reg); } static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) { return readl_relaxed(pcie->appl_base + reg); } struct tegra_pcie_soc { enum dw_pcie_device_mode mode; }; static void apply_bad_link_workaround(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 current_link_width; u16 val; /* * NOTE:- Since this scenario is uncommon and link as such is not * stable anyway, not waiting to confirm if link is really * transitioning to Gen-2 speed */ val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); if (val & PCI_EXP_LNKSTA_LBMS) { current_link_width = (val & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT; if (pcie->init_link_width > current_link_width) { dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL2); val &= ~PCI_EXP_LNKCTL2_TLS; val |= PCI_EXP_LNKCTL2_TLS_2_5GT; dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL2, val); val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL); val |= PCI_EXP_LNKCTL_RL; dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, val); } } } static irqreturn_t tegra_pcie_rp_irq_handler(struct tegra_pcie_dw *pcie) { struct dw_pcie *pci = &pcie->pci; struct pcie_port *pp = &pci->pp; u32 val, tmp; u16 val_w; val = appl_readl(pcie, APPL_INTR_STATUS_L0); if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); /* SBR & Surprise Link Down WAR */ val = appl_readl(pcie, APPL_CAR_RESET_OVRD); val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; appl_writel(pcie, val, APPL_CAR_RESET_OVRD); udelay(1); val = appl_readl(pcie, APPL_CAR_RESET_OVRD); val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; appl_writel(pcie, val, APPL_CAR_RESET_OVRD); val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE; dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); } } if (val & APPL_INTR_STATUS_L0_INT_INT) { val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { appl_writel(pcie, APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, APPL_INTR_STATUS_L1_8_0); apply_bad_link_workaround(pp); } if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { appl_writel(pcie, APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, APPL_INTR_STATUS_L1_8_0); val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w & PCI_EXP_LNKSTA_CLS); } } val = appl_readl(pcie, APPL_INTR_STATUS_L0); if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { val = appl_readl(pcie, APPL_INTR_STATUS_L1_18); tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { dev_info(pci->dev, "CDM check complete\n"); tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; } if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { dev_err(pci->dev, "CDM comparison mismatch\n"); tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; } if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { dev_err(pci->dev, "CDM Logic error\n"); tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; } dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp); tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp); } return IRQ_HANDLED; } static irqreturn_t tegra_pcie_irq_handler(int irq, void *arg) { struct tegra_pcie_dw *pcie = arg; return tegra_pcie_rp_irq_handler(pcie); } static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size, u32 *val) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); /* * This is an endpoint mode specific register happen to appear even * when controller is operating in root port mode and system hangs * when it is accessed with link being in ASPM-L1 state. * So skip accessing it altogether */ if (where == PORT_LOGIC_MSIX_DOORBELL) { *val = 0x00000000; return PCIBIOS_SUCCESSFUL; } return dw_pcie_read(pci->dbi_base + where, size, val); } static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size, u32 val) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); /* * This is an endpoint mode specific register happen to appear even * when controller is operating in root port mode and system hangs * when it is accessed with link being in ASPM-L1 state. * So skip accessing it altogether */ if (where == PORT_LOGIC_MSIX_DOORBELL) return PCIBIOS_SUCCESSFUL; return dw_pcie_write(pci->dbi_base + where, size, val); } #if defined(CONFIG_PCIEASPM) static void disable_aspm_l11(struct tegra_pcie_dw *pcie) { u32 val; val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); val &= ~PCI_L1SS_CAP_ASPM_L1_1; dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); } static void disable_aspm_l12(struct tegra_pcie_dw *pcie) { u32 val; val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); val &= ~PCI_L1SS_CAP_ASPM_L1_2; dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); } static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) { u32 val; val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]); val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]); return val; } static int aspm_state_cnt(struct seq_file *s, void *data) { struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *) dev_get_drvdata(s->private); u32 val; seq_printf(s, "Tx L0s entry count : %u\n", event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S)); seq_printf(s, "Rx L0s entry count : %u\n", event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S)); seq_printf(s, "Link L1 entry count : %u\n", event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1)); seq_printf(s, "Link L1.1 entry count : %u\n", event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1)); seq_printf(s, "Link L1.2 entry count : %u\n", event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); /* Clear all counters */ dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], EVENT_COUNTER_ALL_CLEAR); /* Re-enable counting */ val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); return 0; } static void init_host_aspm(struct tegra_pcie_dw *pcie) { struct dw_pcie *pci = &pcie->pci; u32 val; val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP; /* Enable ASPM counters */ val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val); /* Program T_cmrt and T_pwr_on values */ val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); val |= (pcie->aspm_cmrt << 8); val |= (pcie->aspm_pwr_on_t << 19); dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); /* Program L0s and L1 entrance latencies */ val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); val &= ~L0S_ENTRANCE_LAT_MASK; val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT); val |= ENTER_ASPM; dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); } static int init_debugfs(struct tegra_pcie_dw *pcie) { struct dentry *d; d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs, aspm_state_cnt); if (IS_ERR_OR_NULL(d)) dev_err(pcie->dev, "Failed to create debugfs file \"aspm_state_cnt\"\n"); return 0; } #else static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; } #endif static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val; u16 val_w; val = appl_readl(pcie, APPL_INTR_EN_L0_0); val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; appl_writel(pcie, val, APPL_INTR_EN_L0_0); val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); if (pcie->enable_cdm_check) { val = appl_readl(pcie, APPL_INTR_EN_L0_0); val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN; appl_writel(pcie, val, APPL_INTR_EN_L0_0); val = appl_readl(pcie, APPL_INTR_EN_L1_18); val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR; val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR; appl_writel(pcie, val, APPL_INTR_EN_L1_18); } val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT; val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL); val_w |= PCI_EXP_LNKCTL_LBMIE; dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, val_w); } static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val; /* Enable legacy interrupt generation */ val = appl_readl(pcie, APPL_INTR_EN_L0_0); val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; val |= APPL_INTR_EN_L0_0_INT_INT_EN; appl_writel(pcie, val, APPL_INTR_EN_L0_0); val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); val |= APPL_INTR_EN_L1_8_INTX_EN; val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN; val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN; if (IS_ENABLED(CONFIG_PCIEAER)) val |= APPL_INTR_EN_L1_8_AER_INT_EN; appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); } static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val; dw_pcie_msi_init(pp); /* Enable MSI interrupt generation */ val = appl_readl(pcie, APPL_INTR_EN_L0_0); val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN; appl_writel(pcie, val, APPL_INTR_EN_L0_0); } static void tegra_pcie_enable_interrupts(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); /* Clear interrupt statuses before enabling interrupts */ appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); tegra_pcie_enable_system_interrupts(pp); tegra_pcie_enable_legacy_interrupts(pp); if (IS_ENABLED(CONFIG_PCI_MSI)) tegra_pcie_enable_msi_interrupts(pp); } static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) { struct dw_pcie *pci = &pcie->pci; u32 val, offset, i; /* Program init preset */ for (i = 0; i < pcie->num_lanes; i++) { dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF + (i * 2), 2, &val); val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; val |= GEN3_GEN4_EQ_PRESET_INIT; val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; val |= (GEN3_GEN4_EQ_PRESET_INIT << CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF + (i * 2), 2, val); offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_PL_16GT) + PCI_PL_16GT_LE_CTRL; dw_pcie_read(pci->dbi_base + offset + i, 1, &val); val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; val |= GEN3_GEN4_EQ_PRESET_INIT; val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; val |= (GEN3_GEN4_EQ_PRESET_INIT << PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); dw_pcie_write(pci->dbi_base + offset + i, 1, val); } val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT); dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); } static void tegra_pcie_prepare_host(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val; val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); dw_pcie_writel_dbi(pci, PCI_IO_BASE, val); val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE); val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE; val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE; dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val); dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); /* Configure FTS */ val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); val &= ~(N_FTS_MASK << N_FTS_SHIFT); val |= N_FTS_VAL << N_FTS_SHIFT; dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); val &= ~FTS_MASK; val |= FTS_VAL; dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); /* Enable as 0xFFFF0001 response for CRS */ val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << AMBA_ERROR_RESPONSE_CRS_SHIFT); dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); /* Configure Max Speed from DT */ if (pcie->max_speed && pcie->max_speed != -EINVAL) { val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); val &= ~PCI_EXP_LNKCAP_SLS; val |= pcie->max_speed; dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val); } /* Configure Max lane width from DT */ val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); val &= ~PCI_EXP_LNKCAP_MLW; val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT); dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val); config_gen3_gen4_eq_presets(pcie); init_host_aspm(pcie); val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); if (pcie->update_fc_fixup) { val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); } dw_pcie_setup_rc(pp); clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); /* Assert RST */ val = appl_readl(pcie, APPL_PINMUX); val &= ~APPL_PINMUX_PEX_RST; appl_writel(pcie, val, APPL_PINMUX); usleep_range(100, 200); /* Enable LTSSM */ val = appl_readl(pcie, APPL_CTRL); val |= APPL_CTRL_LTSSM_EN; appl_writel(pcie, val, APPL_CTRL); /* De-assert RST */ val = appl_readl(pcie, APPL_PINMUX); val |= APPL_PINMUX_PEX_RST; appl_writel(pcie, val, APPL_PINMUX); msleep(100); } static int tegra_pcie_dw_host_init(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val, tmp, offset, speed; tegra_pcie_prepare_host(pp); if (dw_pcie_wait_for_link(pci)) { /* * There are some endpoints which can't get the link up if * root port has Data Link Feature (DLF) enabled. * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info * on Scaled Flow Control and DLF. * So, need to confirm that is indeed the case here and attempt * link up once again with DLF disabled. */ val = appl_readl(pcie, APPL_DEBUG); val &= APPL_DEBUG_LTSSM_STATE_MASK; val >>= APPL_DEBUG_LTSSM_STATE_SHIFT; tmp = appl_readl(pcie, APPL_LINK_STATUS); tmp &= APPL_LINK_STATUS_RDLH_LINK_UP; if (!(val == 0x11 && !tmp)) { /* Link is down for all good reasons */ return 0; } dev_info(pci->dev, "Link is down in DLL"); dev_info(pci->dev, "Trying again with DLFE disabled\n"); /* Disable LTSSM */ val = appl_readl(pcie, APPL_CTRL); val &= ~APPL_CTRL_LTSSM_EN; appl_writel(pcie, val, APPL_CTRL); reset_control_assert(pcie->core_rst); reset_control_deassert(pcie->core_rst); offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); val &= ~PCI_DLF_EXCHANGE_ENABLE; dw_pcie_writel_dbi(pci, offset, val); tegra_pcie_prepare_host(pp); if (dw_pcie_wait_for_link(pci)) return 0; } speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & PCI_EXP_LNKSTA_CLS; clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); tegra_pcie_enable_interrupts(pp); return 0; } static int tegra_pcie_dw_link_up(struct dw_pcie *pci) { struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); return !!(val & PCI_EXP_LNKSTA_DLLLA); } static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp) { pp->num_vectors = MAX_MSI_IRQS; } static const struct dw_pcie_ops tegra_dw_pcie_ops = { .link_up = tegra_pcie_dw_link_up, }; static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { .rd_own_conf = tegra_pcie_dw_rd_own_conf, .wr_own_conf = tegra_pcie_dw_wr_own_conf, .host_init = tegra_pcie_dw_host_init, .set_num_vectors = tegra_pcie_set_msi_vec_num, }; static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) { unsigned int phy_count = pcie->phy_count; while (phy_count--) { phy_power_off(pcie->phys[phy_count]); phy_exit(pcie->phys[phy_count]); } } static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie) { unsigned int i; int ret; for (i = 0; i < pcie->phy_count; i++) { ret = phy_init(pcie->phys[i]); if (ret < 0) goto phy_power_off; ret = phy_power_on(pcie->phys[i]); if (ret < 0) goto phy_exit; } return 0; phy_power_off: while (i--) { phy_power_off(pcie->phys[i]); phy_exit: phy_exit(pcie->phys[i]); } return ret; } static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) { struct device_node *np = pcie->dev->of_node; int ret; ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); if (ret < 0) { dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); return ret; } ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us", &pcie->aspm_pwr_on_t); if (ret < 0) dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n", ret); ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us", &pcie->aspm_l0s_enter_lat); if (ret < 0) dev_info(pcie->dev, "Failed to read ASPM L0s Entrance latency: %d\n", ret); ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes); if (ret < 0) { dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret); return ret; } pcie->max_speed = of_pci_get_max_link_speed(np); ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); if (ret) { dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); return ret; } ret = of_property_count_strings(np, "phy-names"); if (ret < 0) { dev_err(pcie->dev, "Failed to find PHY entries: %d\n", ret); return ret; } pcie->phy_count = ret; if (of_property_read_bool(np, "nvidia,update-fc-fixup")) pcie->update_fc_fixup = true; pcie->supports_clkreq = of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); pcie->enable_cdm_check = of_property_read_bool(np, "snps,enable-cdm-check"); return 0; } static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, bool enable) { struct mrq_uphy_response resp; struct tegra_bpmp_message msg; struct mrq_uphy_request req; /* Controller-5 doesn't need to have its state set by BPMP-FW */ if (pcie->cid == 5) return 0; memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE; req.controller_state.pcie_controller = pcie->cid; req.controller_state.enable = enable; memset(&msg, 0, sizeof(msg)); msg.mrq = MRQ_UPHY; msg.tx.data = &req; msg.tx.size = sizeof(req); msg.rx.data = &resp; msg.rx.size = sizeof(resp); return tegra_bpmp_transfer(pcie->bpmp, &msg); } static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) { struct pcie_port *pp = &pcie->pci.pp; struct pci_bus *child, *root_bus = NULL; struct pci_dev *pdev; /* * link doesn't go into L2 state with some of the endpoints with Tegra * if they are not in D0 state. So, need to make sure that immediate * downstream devices are in D0 state before sending PME_TurnOff to put * link into L2 state. * This is as per PCI Express Base r4.0 v1.0 September 27-2017, * 5.2 Link State Power Management (Page #428). */ list_for_each_entry(child, &pp->root_bus->children, node) { /* Bring downstream devices to D0 if they are not already in */ if (child->parent == pp->root_bus) { root_bus = child; break; } } if (!root_bus) { dev_err(pcie->dev, "Failed to find downstream devices\n"); return; } list_for_each_entry(pdev, &root_bus->devices, bus_list) { if (PCI_SLOT(pdev->devfn) == 0) { if (pci_set_power_state(pdev, PCI_D0)) dev_err(pcie->dev, "Failed to transition %s to D0 state\n", dev_name(&pdev->dev)); } } } static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) { pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); if (IS_ERR(pcie->slot_ctl_3v3)) { if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV) return PTR_ERR(pcie->slot_ctl_3v3); pcie->slot_ctl_3v3 = NULL; } pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v"); if (IS_ERR(pcie->slot_ctl_12v)) { if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV) return PTR_ERR(pcie->slot_ctl_12v); pcie->slot_ctl_12v = NULL; } return 0; } static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie) { int ret; if (pcie->slot_ctl_3v3) { ret = regulator_enable(pcie->slot_ctl_3v3); if (ret < 0) { dev_err(pcie->dev, "Failed to enable 3.3V slot supply: %d\n", ret); return ret; } } if (pcie->slot_ctl_12v) { ret = regulator_enable(pcie->slot_ctl_12v); if (ret < 0) { dev_err(pcie->dev, "Failed to enable 12V slot supply: %d\n", ret); goto fail_12v_enable; } } /* * According to PCI Express Card Electromechanical Specification * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive) * should be a minimum of 100ms. */ if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v) msleep(100); return 0; fail_12v_enable: if (pcie->slot_ctl_3v3) regulator_disable(pcie->slot_ctl_3v3); return ret; } static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie) { if (pcie->slot_ctl_12v) regulator_disable(pcie->slot_ctl_12v); if (pcie->slot_ctl_3v3) regulator_disable(pcie->slot_ctl_3v3); } static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, bool en_hw_hot_rst) { int ret; u32 val; ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); if (ret) { dev_err(pcie->dev, "Failed to enable controller %u: %d\n", pcie->cid, ret); return ret; } ret = tegra_pcie_enable_slot_regulators(pcie); if (ret < 0) goto fail_slot_reg_en; ret = regulator_enable(pcie->pex_ctl_supply); if (ret < 0) { dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret); goto fail_reg_en; } ret = clk_prepare_enable(pcie->core_clk); if (ret) { dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret); goto fail_core_clk; } ret = reset_control_deassert(pcie->core_apb_rst); if (ret) { dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n", ret); goto fail_core_apb_rst; } if (en_hw_hot_rst) { /* Enable HW_HOT_RST mode */ val = appl_readl(pcie, APPL_CTRL); val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << APPL_CTRL_HW_HOT_RST_MODE_SHIFT); val |= APPL_CTRL_HW_HOT_RST_EN; appl_writel(pcie, val, APPL_CTRL); } ret = tegra_pcie_enable_phy(pcie); if (ret) { dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret); goto fail_phy; } /* Update CFG base address */ appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, APPL_CFG_BASE_ADDR); /* Configure this core for RP mode operation */ appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE); appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); val = appl_readl(pcie, APPL_CTRL); appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL); val = appl_readl(pcie, APPL_CFG_MISC); val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); appl_writel(pcie, val, APPL_CFG_MISC); if (!pcie->supports_clkreq) { val = appl_readl(pcie, APPL_PINMUX); val |= APPL_PINMUX_CLKREQ_OUT_OVRD_EN; val |= APPL_PINMUX_CLKREQ_OUT_OVRD; appl_writel(pcie, val, APPL_PINMUX); } /* Update iATU_DMA base address */ appl_writel(pcie, pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK, APPL_CFG_IATU_DMA_BASE_ADDR); reset_control_deassert(pcie->core_rst); pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, PCI_CAP_ID_EXP); /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */ if (!pcie->supports_clkreq) { disable_aspm_l11(pcie); disable_aspm_l12(pcie); } return ret; fail_phy: reset_control_assert(pcie->core_apb_rst); fail_core_apb_rst: clk_disable_unprepare(pcie->core_clk); fail_core_clk: regulator_disable(pcie->pex_ctl_supply); fail_reg_en: tegra_pcie_disable_slot_regulators(pcie); fail_slot_reg_en: tegra_pcie_bpmp_set_ctrl_state(pcie, false); return ret; } static int __deinit_controller(struct tegra_pcie_dw *pcie) { int ret; ret = reset_control_assert(pcie->core_rst); if (ret) { dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret); return ret; } tegra_pcie_disable_phy(pcie); ret = reset_control_assert(pcie->core_apb_rst); if (ret) { dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); return ret; } clk_disable_unprepare(pcie->core_clk); ret = regulator_disable(pcie->pex_ctl_supply); if (ret) { dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); return ret; } tegra_pcie_disable_slot_regulators(pcie); ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); if (ret) { dev_err(pcie->dev, "Failed to disable controller %d: %d\n", pcie->cid, ret); return ret; } return ret; } static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) { struct dw_pcie *pci = &pcie->pci; struct pcie_port *pp = &pci->pp; int ret; ret = tegra_pcie_config_controller(pcie, false); if (ret < 0) return ret; pp->ops = &tegra_pcie_dw_host_ops; ret = dw_pcie_host_init(pp); if (ret < 0) { dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret); goto fail_host_init; } return 0; fail_host_init: return __deinit_controller(pcie); } static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) { u32 val; if (!tegra_pcie_dw_link_up(&pcie->pci)) return 0; val = appl_readl(pcie, APPL_RADM_STATUS); val |= APPL_PM_XMT_TURNOFF_STATE; appl_writel(pcie, val, APPL_RADM_STATUS); return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val, val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, 1, PME_ACK_TIMEOUT); } static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) { u32 data; int err; if (!tegra_pcie_dw_link_up(&pcie->pci)) { dev_dbg(pcie->dev, "PCIe link is not up...!\n"); return; } if (tegra_pcie_try_link_l2(pcie)) { dev_info(pcie->dev, "Link didn't transition to L2 state\n"); /* * TX lane clock freq will reset to Gen1 only if link is in L2 * or detect state. * So apply pex_rst to end point to force RP to go into detect * state */ data = appl_readl(pcie, APPL_PINMUX); data &= ~APPL_PINMUX_PEX_RST; appl_writel(pcie, data, APPL_PINMUX); err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, data, ((data & APPL_DEBUG_LTSSM_STATE_MASK) >> APPL_DEBUG_LTSSM_STATE_SHIFT) == LTSSM_STATE_PRE_DETECT, 1, LTSSM_TIMEOUT); if (err) { dev_info(pcie->dev, "Link didn't go to detect state\n"); } else { /* Disable LTSSM after link is in detect state */ data = appl_readl(pcie, APPL_CTRL); data &= ~APPL_CTRL_LTSSM_EN; appl_writel(pcie, data, APPL_CTRL); } } /* * DBI registers may not be accessible after this as PLL-E would be * down depending on how CLKREQ is pulled by end point */ data = appl_readl(pcie, APPL_PINMUX); data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE); /* Cut REFCLK to slot */ data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; appl_writel(pcie, data, APPL_PINMUX); } static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) { tegra_pcie_downstream_dev_to_D0(pcie); dw_pcie_host_deinit(&pcie->pci.pp); tegra_pcie_dw_pme_turnoff(pcie); return __deinit_controller(pcie); } static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) { struct pcie_port *pp = &pcie->pci.pp; struct device *dev = pcie->dev; char *name; int ret; if (IS_ENABLED(CONFIG_PCI_MSI)) { pp->msi_irq = of_irq_get_byname(dev->of_node, "msi"); if (!pp->msi_irq) { dev_err(dev, "Failed to get MSI interrupt\n"); return -ENODEV; } } pm_runtime_enable(dev); ret = pm_runtime_get_sync(dev); if (ret < 0) { dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", ret); goto fail_pm_get_sync; } ret = pinctrl_pm_select_default_state(dev); if (ret < 0) { dev_err(dev, "Failed to configure sideband pins: %d\n", ret); goto fail_pinctrl; } tegra_pcie_init_controller(pcie); pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); if (!pcie->link_state) { ret = -ENOMEDIUM; goto fail_host_init; } name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); if (!name) { ret = -ENOMEM; goto fail_host_init; } pcie->debugfs = debugfs_create_dir(name, NULL); if (!pcie->debugfs) dev_err(dev, "Failed to create debugfs\n"); else init_debugfs(pcie); return ret; fail_host_init: tegra_pcie_deinit_controller(pcie); fail_pinctrl: pm_runtime_put_sync(dev); fail_pm_get_sync: pm_runtime_disable(dev); return ret; } static int tegra_pcie_dw_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct resource *atu_dma_res; struct tegra_pcie_dw *pcie; struct resource *dbi_res; struct pcie_port *pp; struct dw_pcie *pci; struct phy **phys; char *name; int ret; u32 i; pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); if (!pcie) return -ENOMEM; pci = &pcie->pci; pci->dev = &pdev->dev; pci->ops = &tegra_dw_pcie_ops; pp = &pci->pp; pcie->dev = &pdev->dev; ret = tegra_pcie_dw_parse_dt(pcie); if (ret < 0) { dev_err(dev, "Failed to parse device tree: %d\n", ret); return ret; } ret = tegra_pcie_get_slot_regulators(pcie); if (ret < 0) { dev_err(dev, "Failed to get slot regulators: %d\n", ret); return ret; } pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl"); if (IS_ERR(pcie->pex_ctl_supply)) { ret = PTR_ERR(pcie->pex_ctl_supply); if (ret != -EPROBE_DEFER) dev_err(dev, "Failed to get regulator: %ld\n", PTR_ERR(pcie->pex_ctl_supply)); return ret; } pcie->core_clk = devm_clk_get(dev, "core"); if (IS_ERR(pcie->core_clk)) { dev_err(dev, "Failed to get core clock: %ld\n", PTR_ERR(pcie->core_clk)); return PTR_ERR(pcie->core_clk); } pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "appl"); if (!pcie->appl_res) { dev_err(dev, "Failed to find \"appl\" region\n"); return -ENODEV; } pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res); if (IS_ERR(pcie->appl_base)) return PTR_ERR(pcie->appl_base); pcie->core_apb_rst = devm_reset_control_get(dev, "apb"); if (IS_ERR(pcie->core_apb_rst)) { dev_err(dev, "Failed to get APB reset: %ld\n", PTR_ERR(pcie->core_apb_rst)); return PTR_ERR(pcie->core_apb_rst); } phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL); if (!phys) return -ENOMEM; for (i = 0; i < pcie->phy_count; i++) { name = kasprintf(GFP_KERNEL, "p2u-%u", i); if (!name) { dev_err(dev, "Failed to create P2U string\n"); return -ENOMEM; } phys[i] = devm_phy_get(dev, name); kfree(name); if (IS_ERR(phys[i])) { ret = PTR_ERR(phys[i]); if (ret != -EPROBE_DEFER) dev_err(dev, "Failed to get PHY: %d\n", ret); return ret; } } pcie->phys = phys; dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); if (!dbi_res) { dev_err(dev, "Failed to find \"dbi\" region\n"); return -ENODEV; } pcie->dbi_res = dbi_res; pci->dbi_base = devm_ioremap_resource(dev, dbi_res); if (IS_ERR(pci->dbi_base)) return PTR_ERR(pci->dbi_base); /* Tegra HW locates DBI2 at a fixed offset from DBI */ pci->dbi_base2 = pci->dbi_base + 0x1000; atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu_dma"); if (!atu_dma_res) { dev_err(dev, "Failed to find \"atu_dma\" region\n"); return -ENODEV; } pcie->atu_dma_res = atu_dma_res; pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); if (IS_ERR(pci->atu_base)) return PTR_ERR(pci->atu_base); pcie->core_rst = devm_reset_control_get(dev, "core"); if (IS_ERR(pcie->core_rst)) { dev_err(dev, "Failed to get core reset: %ld\n", PTR_ERR(pcie->core_rst)); return PTR_ERR(pcie->core_rst); } pp->irq = platform_get_irq_byname(pdev, "intr"); if (!pp->irq) { dev_err(dev, "Failed to get \"intr\" interrupt\n"); return -ENODEV; } ret = devm_request_irq(dev, pp->irq, tegra_pcie_irq_handler, IRQF_SHARED, "tegra-pcie-intr", pcie); if (ret) { dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, ret); return ret; } pcie->bpmp = tegra_bpmp_get(dev); if (IS_ERR(pcie->bpmp)) return PTR_ERR(pcie->bpmp); platform_set_drvdata(pdev, pcie); ret = tegra_pcie_config_rp(pcie); if (ret && ret != -ENOMEDIUM) goto fail; else return 0; fail: tegra_bpmp_put(pcie->bpmp); return ret; } static int tegra_pcie_dw_remove(struct platform_device *pdev) { struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); if (!pcie->link_state) return 0; debugfs_remove_recursive(pcie->debugfs); tegra_pcie_deinit_controller(pcie); pm_runtime_put_sync(pcie->dev); pm_runtime_disable(pcie->dev); tegra_bpmp_put(pcie->bpmp); return 0; } static int tegra_pcie_dw_suspend_late(struct device *dev) { struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); u32 val; if (!pcie->link_state) return 0; /* Enable HW_HOT_RST mode */ val = appl_readl(pcie, APPL_CTRL); val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << APPL_CTRL_HW_HOT_RST_MODE_SHIFT); val |= APPL_CTRL_HW_HOT_RST_EN; appl_writel(pcie, val, APPL_CTRL); return 0; } static int tegra_pcie_dw_suspend_noirq(struct device *dev) { struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); if (!pcie->link_state) return 0; /* Save MSI interrupt vector */ pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN); tegra_pcie_downstream_dev_to_D0(pcie); tegra_pcie_dw_pme_turnoff(pcie); return __deinit_controller(pcie); } static int tegra_pcie_dw_resume_noirq(struct device *dev) { struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); int ret; if (!pcie->link_state) return 0; ret = tegra_pcie_config_controller(pcie, true); if (ret < 0) return ret; ret = tegra_pcie_dw_host_init(&pcie->pci.pp); if (ret < 0) { dev_err(dev, "Failed to init host: %d\n", ret); goto fail_host_init; } /* Restore MSI interrupt vector */ dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN, pcie->msi_ctrl_int); return 0; fail_host_init: return __deinit_controller(pcie); } static int tegra_pcie_dw_resume_early(struct device *dev) { struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); u32 val; if (!pcie->link_state) return 0; /* Disable HW_HOT_RST mode */ val = appl_readl(pcie, APPL_CTRL); val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << APPL_CTRL_HW_HOT_RST_MODE_SHIFT); val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << APPL_CTRL_HW_HOT_RST_MODE_SHIFT; val &= ~APPL_CTRL_HW_HOT_RST_EN; appl_writel(pcie, val, APPL_CTRL); return 0; } static void tegra_pcie_dw_shutdown(struct platform_device *pdev) { struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); if (!pcie->link_state) return; debugfs_remove_recursive(pcie->debugfs); tegra_pcie_downstream_dev_to_D0(pcie); disable_irq(pcie->pci.pp.irq); if (IS_ENABLED(CONFIG_PCI_MSI)) disable_irq(pcie->pci.pp.msi_irq); tegra_pcie_dw_pme_turnoff(pcie); __deinit_controller(pcie); } static const struct of_device_id tegra_pcie_dw_of_match[] = { { .compatible = "nvidia,tegra194-pcie", }, {}, }; static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { .suspend_late = tegra_pcie_dw_suspend_late, .suspend_noirq = tegra_pcie_dw_suspend_noirq, .resume_noirq = tegra_pcie_dw_resume_noirq, .resume_early = tegra_pcie_dw_resume_early, }; static struct platform_driver tegra_pcie_dw_driver = { .probe = tegra_pcie_dw_probe, .remove = tegra_pcie_dw_remove, .shutdown = tegra_pcie_dw_shutdown, .driver = { .name = "tegra194-pcie", .pm = &tegra_pcie_dw_pm_ops, .of_match_table = tegra_pcie_dw_of_match, }, }; module_platform_driver(tegra_pcie_dw_driver); MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match); MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>"); MODULE_DESCRIPTION("NVIDIA PCIe host controller driver"); MODULE_LICENSE("GPL v2");
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1