| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Krishna chaitanya chundru | 962 | 16.63% | 6 | 3.92% |
| Kishon Vijay Abraham I | 876 | 15.14% | 9 | 5.88% |
| Frank Li | 640 | 11.06% | 7 | 4.58% |
| Gustavo Pimentel | 550 | 9.51% | 10 | 6.54% |
| Serge Semin | 463 | 8.00% | 14 | 9.15% |
| Dmitry Eremin-Solenikov | 419 | 7.24% | 4 | 2.61% |
| Rob Herring | 348 | 6.01% | 21 | 13.73% |
| Jingoo Han | 197 | 3.40% | 3 | 1.96% |
| Radu Rendec | 184 | 3.18% | 2 | 1.31% |
| Yoshihiro Shimoda | 153 | 2.64% | 2 | 1.31% |
| Samuel Holland | 125 | 2.16% | 1 | 0.65% |
| Björn Helgaas | 116 | 2.00% | 9 | 5.88% |
| JiSheng Zhang | 87 | 1.50% | 7 | 4.58% |
| Vidya Sagar | 62 | 1.07% | 3 | 1.96% |
| Richard Zhu | 50 | 0.86% | 4 | 2.61% |
| Nam Cao | 49 | 0.85% | 1 | 0.65% |
| Niklas Svensson (Niklas Cassel) | 49 | 0.85% | 2 | 1.31% |
| Manivannan Sadhasivam | 47 | 0.81% | 4 | 2.61% |
| Marc Zyngier | 43 | 0.74% | 3 | 1.96% |
| Ajay Agarwal | 43 | 0.74% | 2 | 1.31% |
| Hou Zhiqiang | 37 | 0.64% | 5 | 3.27% |
| Shawn Lin | 30 | 0.52% | 1 | 0.65% |
| Qiang Yu | 30 | 0.52% | 2 | 1.31% |
| Murali Karicheri | 27 | 0.47% | 3 | 1.96% |
| Niklas Cassel | 22 | 0.38% | 2 | 1.31% |
| Pratyush Anand | 21 | 0.36% | 2 | 1.31% |
| Mayank Rana | 20 | 0.35% | 1 | 0.65% |
| Björn Andersson | 18 | 0.31% | 2 | 1.31% |
| Kunihiko Hayashi | 17 | 0.29% | 1 | 0.65% |
| Will McVicker | 17 | 0.29% | 2 | 1.31% |
| Dilip Kota | 12 | 0.21% | 1 | 0.65% |
| Zhou Wang | 10 | 0.17% | 2 | 1.31% |
| Shradha Todi | 10 | 0.17% | 1 | 0.65% |
| Björn Erik Nilsen | 10 | 0.17% | 1 | 0.65% |
| Minghuan Lian | 8 | 0.14% | 1 | 0.65% |
| Andrey Smirnov | 8 | 0.14% | 1 | 0.65% |
| Lucas Stach | 5 | 0.09% | 2 | 1.31% |
| Siddharth Vadapalli | 5 | 0.09% | 1 | 0.65% |
| Joao Pinto | 4 | 0.07% | 1 | 0.65% |
| Thomas Petazzoni | 3 | 0.05% | 1 | 0.65% |
| Lorenzo Pieralisi | 3 | 0.05% | 1 | 0.65% |
| Johan Hovold | 2 | 0.03% | 1 | 0.65% |
| Koen Vandeputte | 1 | 0.02% | 1 | 0.65% |
| Jiri Slaby (SUSE) | 1 | 0.02% | 1 | 0.65% |
| Alexander A. Klimov | 1 | 0.02% | 1 | 0.65% |
| Martin Kaiser | 1 | 0.02% | 1 | 0.65% |
| Total | 5786 | 153 |
// SPDX-License-Identifier: GPL-2.0 /* * Synopsys DesignWare PCIe host controller driver * * Copyright (C) 2013 Samsung Electronics Co., Ltd. * https://www.samsung.com * * Author: Jingoo Han <jg1.han@samsung.com> */ #include <linux/align.h> #include <linux/iopoll.h> #include <linux/irqchip/chained_irq.h> #include <linux/irqchip/irq-msi-lib.h> #include <linux/irqdomain.h> #include <linux/msi.h> #include <linux/of_address.h> #include <linux/of_pci.h> #include <linux/pci_regs.h> #include <linux/platform_device.h> #include "../../pci.h" #include "pcie-designware.h" static struct pci_ops dw_pcie_ops; static struct pci_ops dw_pcie_ecam_ops; static struct pci_ops dw_child_pcie_ops; #ifdef CONFIG_SMP static void dw_irq_noop(struct irq_data *d) { } #endif static bool dw_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *domain, struct irq_domain *real_parent, struct msi_domain_info *info) { if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) return false; #ifdef CONFIG_SMP info->chip->irq_ack = dw_irq_noop; info->chip->irq_pre_redirect = irq_chip_pre_redirect_parent; #else info->chip->irq_ack = irq_chip_ack_parent; #endif return true; } #define DW_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ MSI_FLAG_USE_DEF_CHIP_OPS | \ MSI_FLAG_PCI_MSI_MASK_PARENT) #define DW_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \ MSI_FLAG_PCI_MSIX | \ MSI_GENERIC_FLAGS_MASK) #define IS_256MB_ALIGNED(x) IS_ALIGNED(x, SZ_256M) static const struct msi_parent_ops dw_pcie_msi_parent_ops = { .required_flags = DW_PCIE_MSI_FLAGS_REQUIRED, .supported_flags = DW_PCIE_MSI_FLAGS_SUPPORTED, .bus_select_token = DOMAIN_BUS_PCI_MSI, .prefix = "DW-", .init_dev_msi_info = dw_pcie_init_dev_msi_info, }; /* MSI int handler */ void dw_handle_msi_irq(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned int i, num_ctrls; num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; for (i = 0; i < num_ctrls; i++) { unsigned int reg_off = i * MSI_REG_CTRL_BLOCK_SIZE; unsigned int irq_off = i * MAX_MSI_IRQS_PER_CTRL; unsigned long status, pos; status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS + reg_off); if (!status) continue; for_each_set_bit(pos, &status, MAX_MSI_IRQS_PER_CTRL) generic_handle_demux_domain_irq(pp->irq_domain, irq_off + pos); } } /* Chained MSI interrupt service routine */ static void dw_chained_msi_isr(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); struct dw_pcie_rp *pp; chained_irq_enter(chip, desc); pp = irq_desc_get_handler_data(desc); dw_handle_msi_irq(pp); chained_irq_exit(chip, desc); } static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg) { struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); struct dw_pcie *pci = to_dw_pcie_from_pp(pp); u64 msi_target = (u64)pp->msi_data; msg->address_lo = lower_32_bits(msi_target); msg->address_hi = upper_32_bits(msi_target); msg->data = d->hwirq; dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n", (int)d->hwirq, msg->address_hi, msg->address_lo); } static void dw_pci_bottom_mask(struct irq_data *d) { struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned int res, bit, ctrl; guard(raw_spinlock)(&pp->lock); ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; pp->irq_mask[ctrl] |= BIT(bit); dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); } static void dw_pci_bottom_unmask(struct irq_data *d) { struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned int res, bit, ctrl; guard(raw_spinlock)(&pp->lock); ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; pp->irq_mask[ctrl] &= ~BIT(bit); dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); } static void dw_pci_bottom_ack(struct irq_data *d) { struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned int res, bit, ctrl; ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit)); } static struct irq_chip dw_pci_msi_bottom_irq_chip = { .name = "DWPCI-MSI", .irq_compose_msi_msg = dw_pci_setup_msi_msg, .irq_mask = dw_pci_bottom_mask, .irq_unmask = dw_pci_bottom_unmask, #ifdef CONFIG_SMP .irq_ack = dw_irq_noop, .irq_pre_redirect = dw_pci_bottom_ack, .irq_set_affinity = irq_chip_redirect_set_affinity, #else .irq_ack = dw_pci_bottom_ack, #endif }; static int dw_pcie_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *args) { struct dw_pcie_rp *pp = domain->host_data; int bit; scoped_guard (raw_spinlock_irq, &pp->lock) { bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors, order_base_2(nr_irqs)); } if (bit < 0) return -ENOSPC; for (unsigned int i = 0; i < nr_irqs; i++) { irq_domain_set_info(domain, virq + i, bit + i, pp->msi_irq_chip, pp, handle_edge_irq, NULL, NULL); } return 0; } static void dw_pcie_irq_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); struct dw_pcie_rp *pp = domain->host_data; guard(raw_spinlock_irq)(&pp->lock); bitmap_release_region(pp->msi_irq_in_use, d->hwirq, order_base_2(nr_irqs)); } static const struct irq_domain_ops dw_pcie_msi_domain_ops = { .alloc = dw_pcie_irq_domain_alloc, .free = dw_pcie_irq_domain_free, }; int dw_pcie_allocate_domains(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct irq_domain_info info = { .fwnode = dev_fwnode(pci->dev), .ops = &dw_pcie_msi_domain_ops, .size = pp->num_vectors, .host_data = pp, }; pp->irq_domain = msi_create_parent_irq_domain(&info, &dw_pcie_msi_parent_ops); if (!pp->irq_domain) { dev_err(pci->dev, "Failed to create IRQ domain\n"); return -ENOMEM; } return 0; } EXPORT_SYMBOL_GPL(dw_pcie_allocate_domains); void dw_pcie_free_msi(struct dw_pcie_rp *pp) { u32 ctrl; for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) { if (pp->msi_irq[ctrl] > 0) irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL); } irq_domain_remove(pp->irq_domain); } EXPORT_SYMBOL_GPL(dw_pcie_free_msi); void dw_pcie_msi_init(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); u64 msi_target = (u64)pp->msi_data; u32 ctrl, num_ctrls; if (!pci_msi_enabled() || !pp->use_imsi_rx) return; num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; /* Initialize IRQ Status array */ for (ctrl = 0; ctrl < num_ctrls; ctrl++) { dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + (ctrl * MSI_REG_CTRL_BLOCK_SIZE), pp->irq_mask[ctrl]); dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE + (ctrl * MSI_REG_CTRL_BLOCK_SIZE), ~0); } /* Program the msi_data */ dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target)); dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target)); } EXPORT_SYMBOL_GPL(dw_pcie_msi_init); static int dw_pcie_parse_split_msi_irq(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct device *dev = pci->dev; struct platform_device *pdev = to_platform_device(dev); u32 ctrl, max_vectors; int irq; /* Parse any "msiX" IRQs described in the devicetree */ for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) { char msi_name[] = "msiX"; msi_name[3] = '0' + ctrl; irq = platform_get_irq_byname_optional(pdev, msi_name); if (irq == -ENXIO) break; if (irq < 0) return dev_err_probe(dev, irq, "Failed to parse MSI IRQ '%s'\n", msi_name); pp->msi_irq[ctrl] = irq; } /* If no "msiX" IRQs, caller should fallback to "msi" IRQ */ if (ctrl == 0) return -ENXIO; max_vectors = ctrl * MAX_MSI_IRQS_PER_CTRL; if (pp->num_vectors > max_vectors) { dev_warn(dev, "Exceeding number of MSI vectors, limiting to %u\n", max_vectors); pp->num_vectors = max_vectors; } if (!pp->num_vectors) pp->num_vectors = max_vectors; return 0; } int dw_pcie_msi_host_init(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct device *dev = pci->dev; struct platform_device *pdev = to_platform_device(dev); u64 *msi_vaddr = NULL; int ret; u32 ctrl, num_ctrls; for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) pp->irq_mask[ctrl] = ~0; if (!pp->msi_irq[0]) { ret = dw_pcie_parse_split_msi_irq(pp); if (ret < 0 && ret != -ENXIO) return ret; } if (!pp->num_vectors) pp->num_vectors = MSI_DEF_NUM_VECTORS; num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; if (!pp->msi_irq[0]) { pp->msi_irq[0] = platform_get_irq_byname_optional(pdev, "msi"); if (pp->msi_irq[0] < 0) { pp->msi_irq[0] = platform_get_irq(pdev, 0); if (pp->msi_irq[0] < 0) return pp->msi_irq[0]; } } dev_dbg(dev, "Using %d MSI vectors\n", pp->num_vectors); pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip; ret = dw_pcie_allocate_domains(pp); if (ret) return ret; for (ctrl = 0; ctrl < num_ctrls; ctrl++) { if (pp->msi_irq[ctrl] > 0) irq_set_chained_handler_and_data(pp->msi_irq[ctrl], dw_chained_msi_isr, pp); } /* * Even though the iMSI-RX Module supports 64-bit addresses some * peripheral PCIe devices may lack 64-bit message support. In * order not to miss MSI TLPs from those devices the MSI target * address has to be within the lowest 4GB. * * Per DWC databook r6.21a, section 3.10.2.3, the incoming MWr TLP * targeting the MSI_CTRL_ADDR is terminated by the iMSI-RX and never * appears on the AXI bus. So MSI_CTRL_ADDR address doesn't need to be * mapped and can be any memory that doesn't get allocated for the BAR * memory. Since most of the platforms provide 32-bit address for * 'config' region, try cfg0_base as the first option for the MSI target * address if it's a 32-bit address. Otherwise, try 32-bit and 64-bit * coherent memory allocation one by one. */ if (!(pp->cfg0_base & GENMASK_ULL(63, 32))) { pp->msi_data = pp->cfg0_base; return 0; } ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); if (!ret) msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, GFP_KERNEL); if (!msi_vaddr) { dev_warn(dev, "Failed to allocate 32-bit MSI address\n"); dma_set_coherent_mask(dev, DMA_BIT_MASK(64)); msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, GFP_KERNEL); if (!msi_vaddr) { dev_err(dev, "Failed to allocate MSI address\n"); dw_pcie_free_msi(pp); return -ENOMEM; } } return 0; } EXPORT_SYMBOL_GPL(dw_pcie_msi_host_init); static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct resource_entry *win; struct resource *res; win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM); if (win) { res = devm_kzalloc(pci->dev, sizeof(*res), GFP_KERNEL); if (!res) return; /* * Allocate MSG TLP region of size 'region_align' at the end of * the host bridge window. */ res->start = win->res->end - pci->region_align + 1; res->end = win->res->end; res->name = "msg"; res->flags = win->res->flags | IORESOURCE_BUSY; if (!devm_request_resource(pci->dev, win->res, res)) pp->msg_res = res; } } static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct dw_pcie_ob_atu_cfg atu = {0}; resource_size_t bus_range_max; struct resource_entry *bus; int ret; bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); /* * Root bus under the host bridge doesn't require any iATU configuration * as DBI region will be used to access root bus config space. * Immediate bus under Root Bus, needs type 0 iATU configuration and * remaining buses need type 1 iATU configuration. */ atu.index = 0; atu.type = PCIE_ATU_TYPE_CFG0; atu.parent_bus_addr = pp->cfg0_base + SZ_1M; /* 1MiB is to cover 1 (bus) * 32 (devices) * 8 (functions) */ atu.size = SZ_1M; atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE; ret = dw_pcie_prog_outbound_atu(pci, &atu); if (ret) return ret; bus_range_max = resource_size(bus->res); if (bus_range_max < 2) return 0; /* Configure remaining buses in type 1 iATU configuration */ atu.index = 1; atu.type = PCIE_ATU_TYPE_CFG1; atu.parent_bus_addr = pp->cfg0_base + SZ_2M; atu.size = (SZ_1M * bus_range_max) - SZ_2M; atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE; return dw_pcie_prog_outbound_atu(pci, &atu); } static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct device *dev = pci->dev; struct resource_entry *bus; bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); if (!bus) return -ENODEV; pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops); if (IS_ERR(pp->cfg)) return PTR_ERR(pp->cfg); return 0; } static bool dw_pcie_ecam_enabled(struct dw_pcie_rp *pp, struct resource *config_res) { struct resource *bus_range; u64 nr_buses; /* Vendor glue drivers may implement their own ECAM mechanism */ if (pp->native_ecam) return false; /* * PCIe spec r6.0, sec 7.2.2 mandates the base address used for ECAM to * be aligned on a 2^(n+20) byte boundary, where n is the number of bits * used for representing 'bus' in BDF. Since the DWC cores always use 8 * bits for representing 'bus', the base address has to be aligned to * 2^28 byte boundary, which is 256 MiB. */ if (!IS_256MB_ALIGNED(config_res->start)) return false; bus_range = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS)->res; if (!bus_range) return false; nr_buses = resource_size(config_res) >> PCIE_ECAM_BUS_SHIFT; return nr_buses >= resource_size(bus_range); } static int dw_pcie_host_get_resources(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct device *dev = pci->dev; struct platform_device *pdev = to_platform_device(dev); struct resource_entry *win; struct resource *res; int ret; res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); if (!res) { dev_err(dev, "Missing \"config\" reg space\n"); return -ENODEV; } pp->cfg0_size = resource_size(res); pp->cfg0_base = res->start; pp->ecam_enabled = dw_pcie_ecam_enabled(pp, res); if (pp->ecam_enabled) { ret = dw_pcie_create_ecam_window(pp, res); if (ret) return ret; pp->bridge->ops = &dw_pcie_ecam_ops; pp->bridge->sysdata = pp->cfg; pp->cfg->priv = pp; } else { pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res); if (IS_ERR(pp->va_cfg0_base)) return PTR_ERR(pp->va_cfg0_base); /* Set default bus ops */ pp->bridge->ops = &dw_pcie_ops; pp->bridge->child_ops = &dw_child_pcie_ops; pp->bridge->sysdata = pp; } ret = dw_pcie_get_resources(pci); if (ret) { if (pp->cfg) pci_ecam_free(pp->cfg); return ret; } /* Get the I/O range from DT */ win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_IO); if (win) { pp->io_size = resource_size(win->res); pp->io_bus_addr = win->res->start - win->offset; pp->io_base = pci_pio_to_address(win->res->start); } /* * visconti_pcie_cpu_addr_fixup() uses pp->io_base, so we have to * call dw_pcie_parent_bus_offset() after setting pp->io_base. */ pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "config", pp->cfg0_base); return 0; } int dw_pcie_host_init(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct device *dev = pci->dev; struct device_node *np = dev->of_node; struct pci_host_bridge *bridge; int ret; raw_spin_lock_init(&pp->lock); bridge = devm_pci_alloc_host_bridge(dev, 0); if (!bridge) return -ENOMEM; pp->bridge = bridge; ret = dw_pcie_host_get_resources(pp); if (ret) return ret; if (pp->ops->init) { ret = pp->ops->init(pp); if (ret) goto err_free_ecam; } if (pci_msi_enabled()) { pp->use_imsi_rx = !(pp->ops->msi_init || of_property_present(np, "msi-parent") || of_property_present(np, "msi-map")); /* * For the use_imsi_rx case the default assignment is handled * in the dw_pcie_msi_host_init(). */ if (!pp->use_imsi_rx && !pp->num_vectors) { pp->num_vectors = MSI_DEF_NUM_VECTORS; } else if (pp->num_vectors > MAX_MSI_IRQS) { dev_err(dev, "Invalid number of vectors\n"); ret = -EINVAL; goto err_deinit_host; } if (pp->ops->msi_init) { ret = pp->ops->msi_init(pp); if (ret < 0) goto err_deinit_host; } else if (pp->use_imsi_rx) { ret = dw_pcie_msi_host_init(pp); if (ret < 0) goto err_deinit_host; } } dw_pcie_version_detect(pci); dw_pcie_iatu_detect(pci); if (pci->num_lanes < 1) pci->num_lanes = dw_pcie_link_get_max_link_width(pci); ret = of_pci_get_equalization_presets(dev, &pp->presets, pci->num_lanes); if (ret) goto err_free_msi; /* * Allocate the resource for MSG TLP before programming the iATU * outbound window in dw_pcie_setup_rc(). Since the allocation depends * on the value of 'region_align', this has to be done after * dw_pcie_iatu_detect(). * * Glue drivers need to set 'use_atu_msg' before dw_pcie_host_init() to * make use of the generic MSG TLP implementation. */ if (pp->use_atu_msg) dw_pcie_host_request_msg_tlp_res(pp); ret = dw_pcie_edma_detect(pci); if (ret) goto err_free_msi; ret = dw_pcie_setup_rc(pp); if (ret) goto err_remove_edma; if (!dw_pcie_link_up(pci)) { ret = dw_pcie_start_link(pci); if (ret) goto err_remove_edma; } /* * Only fail on timeout error. Other errors indicate the device may * become available later, so continue without failing. */ ret = dw_pcie_wait_for_link(pci); if (ret == -ETIMEDOUT) goto err_stop_link; ret = pci_host_probe(bridge); if (ret) goto err_stop_link; if (pp->ops->post_init) pp->ops->post_init(pp); dwc_pcie_debugfs_init(pci, DW_PCIE_RC_TYPE); return 0; err_stop_link: dw_pcie_stop_link(pci); err_remove_edma: dw_pcie_edma_remove(pci); err_free_msi: if (pp->use_imsi_rx) dw_pcie_free_msi(pp); err_deinit_host: if (pp->ops->deinit) pp->ops->deinit(pp); err_free_ecam: if (pp->cfg) pci_ecam_free(pp->cfg); return ret; } EXPORT_SYMBOL_GPL(dw_pcie_host_init); void dw_pcie_host_deinit(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); dwc_pcie_debugfs_deinit(pci); pci_stop_root_bus(pp->bridge->bus); pci_remove_root_bus(pp->bridge->bus); dw_pcie_stop_link(pci); dw_pcie_edma_remove(pci); if (pp->use_imsi_rx) dw_pcie_free_msi(pp); if (pp->ops->deinit) pp->ops->deinit(pp); if (pp->cfg) pci_ecam_free(pp->cfg); } EXPORT_SYMBOL_GPL(dw_pcie_host_deinit); static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) { struct dw_pcie_rp *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct dw_pcie_ob_atu_cfg atu = { 0 }; int type, ret; u32 busdev; /* * Checking whether the link is up here is a last line of defense * against platforms that forward errors on the system bus as * SError upon PCI configuration transactions issued when the link * is down. This check is racy by definition and does not stop * the system from triggering an SError if the link goes down * after this check is performed. */ if (!dw_pcie_link_up(pci)) return NULL; busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) | PCIE_ATU_FUNC(PCI_FUNC(devfn)); if (pci_is_root_bus(bus->parent)) type = PCIE_ATU_TYPE_CFG0; else type = PCIE_ATU_TYPE_CFG1; atu.type = type; atu.parent_bus_addr = pp->cfg0_base - pci->parent_bus_offset; atu.pci_addr = busdev; atu.size = pp->cfg0_size; ret = dw_pcie_prog_outbound_atu(pci, &atu); if (ret) return NULL; return pp->va_cfg0_base + where; } static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { struct dw_pcie_rp *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct dw_pcie_ob_atu_cfg atu = { 0 }; int ret; ret = pci_generic_config_read(bus, devfn, where, size, val); if (ret != PCIBIOS_SUCCESSFUL) return ret; if (pp->cfg0_io_shared) { atu.type = PCIE_ATU_TYPE_IO; atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; atu.pci_addr = pp->io_bus_addr; atu.size = pp->io_size; ret = dw_pcie_prog_outbound_atu(pci, &atu); if (ret) return PCIBIOS_SET_FAILED; } return PCIBIOS_SUCCESSFUL; } static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { struct dw_pcie_rp *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct dw_pcie_ob_atu_cfg atu = { 0 }; int ret; ret = pci_generic_config_write(bus, devfn, where, size, val); if (ret != PCIBIOS_SUCCESSFUL) return ret; if (pp->cfg0_io_shared) { atu.type = PCIE_ATU_TYPE_IO; atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; atu.pci_addr = pp->io_bus_addr; atu.size = pp->io_size; ret = dw_pcie_prog_outbound_atu(pci, &atu); if (ret) return PCIBIOS_SET_FAILED; } return PCIBIOS_SUCCESSFUL; } static struct pci_ops dw_child_pcie_ops = { .map_bus = dw_pcie_other_conf_map_bus, .read = dw_pcie_rd_other_conf, .write = dw_pcie_wr_other_conf, }; void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) { struct dw_pcie_rp *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); if (PCI_SLOT(devfn) > 0) return NULL; return pci->dbi_base + where; } EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus); static void __iomem *dw_pcie_ecam_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) { struct pci_config_window *cfg = bus->sysdata; struct dw_pcie_rp *pp = cfg->priv; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned int busn = bus->number; if (busn > 0) return pci_ecam_map_bus(bus, devfn, where); if (PCI_SLOT(devfn) > 0) return NULL; return pci->dbi_base + where; } static struct pci_ops dw_pcie_ops = { .map_bus = dw_pcie_own_conf_map_bus, .read = pci_generic_config_read, .write = pci_generic_config_write, }; static struct pci_ops dw_pcie_ecam_ops = { .map_bus = dw_pcie_ecam_conf_map_bus, .read = pci_generic_config_read, .write = pci_generic_config_write, }; static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct dw_pcie_ob_atu_cfg atu = { 0 }; struct resource_entry *entry; int ob_iatu_index; int ib_iatu_index; int i, ret; if (!pci->num_ob_windows) { dev_err(pci->dev, "No outbound iATU found\n"); return -EINVAL; } /* * Ensure all out/inbound windows are disabled before proceeding with * the MEM/IO (dma-)ranges setups. */ for (i = 0; i < pci->num_ob_windows; i++) dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, i); for (i = 0; i < pci->num_ib_windows; i++) dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, i); /* * NOTE: For outbound address translation, outbound iATU at index 0 is * reserved for CFG IOs (dw_pcie_other_conf_map_bus()), thus start at * index 1. * * If using ECAM, outbound iATU at index 0 and index 1 is reserved for * CFG IOs. */ if (pp->ecam_enabled) { ob_iatu_index = 2; ret = dw_pcie_config_ecam_iatu(pp); if (ret) { dev_err(pci->dev, "Failed to configure iATU in ECAM mode\n"); return ret; } } else { ob_iatu_index = 1; } resource_list_for_each_entry(entry, &pp->bridge->windows) { resource_size_t res_size; if (resource_type(entry->res) != IORESOURCE_MEM) continue; atu.type = PCIE_ATU_TYPE_MEM; atu.parent_bus_addr = entry->res->start - pci->parent_bus_offset; atu.pci_addr = entry->res->start - entry->offset; /* Adjust iATU size if MSG TLP region was allocated before */ if (pp->msg_res && pp->msg_res->parent == entry->res) res_size = resource_size(entry->res) - resource_size(pp->msg_res); else res_size = resource_size(entry->res); while (res_size > 0) { /* * Return failure if we run out of windows in the * middle. Otherwise, we would end up only partially * mapping a single resource. */ if (ob_iatu_index >= pci->num_ob_windows) { dev_err(pci->dev, "Cannot add outbound window for region: %pr\n", entry->res); return -ENOMEM; } atu.index = ob_iatu_index; atu.size = MIN(pci->region_limit + 1, res_size); ret = dw_pcie_prog_outbound_atu(pci, &atu); if (ret) { dev_err(pci->dev, "Failed to set MEM range %pr\n", entry->res); return ret; } ob_iatu_index++; atu.parent_bus_addr += atu.size; atu.pci_addr += atu.size; res_size -= atu.size; } } if (pp->io_size) { if (ob_iatu_index < pci->num_ob_windows) { atu.index = ob_iatu_index; atu.type = PCIE_ATU_TYPE_IO; atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; atu.pci_addr = pp->io_bus_addr; atu.size = pp->io_size; ret = dw_pcie_prog_outbound_atu(pci, &atu); if (ret) { dev_err(pci->dev, "Failed to set IO range %pr\n", entry->res); return ret; } ob_iatu_index++; } else { /* * If there are not enough outbound windows to give I/O * space its own iATU, the outbound iATU at index 0 will * be shared between I/O space and CFG IOs, by * temporarily reconfiguring the iATU to CFG space, in * order to do a CFG IO, and then immediately restoring * it to I/O space. This is only implemented when using * dw_pcie_other_conf_map_bus(), which is not the case * when using ECAM. */ if (pp->ecam_enabled) { dev_err(pci->dev, "Cannot add outbound window for I/O\n"); return -ENOMEM; } pp->cfg0_io_shared = true; } } if (pp->use_atu_msg) { if (ob_iatu_index >= pci->num_ob_windows) { dev_err(pci->dev, "Cannot add outbound window for MSG TLP\n"); return -ENOMEM; } pp->msg_atu_index = ob_iatu_index++; } ib_iatu_index = 0; resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) { resource_size_t res_start, res_size, window_size; if (resource_type(entry->res) != IORESOURCE_MEM) continue; res_size = resource_size(entry->res); res_start = entry->res->start; while (res_size > 0) { /* * Return failure if we run out of windows in the * middle. Otherwise, we would end up only partially * mapping a single resource. */ if (ib_iatu_index >= pci->num_ib_windows) { dev_err(pci->dev, "Cannot add inbound window for region: %pr\n", entry->res); return -ENOMEM; } window_size = MIN(pci->region_limit + 1, res_size); ret = dw_pcie_prog_inbound_atu(pci, ib_iatu_index, PCIE_ATU_TYPE_MEM, res_start, res_start - entry->offset, window_size); if (ret) { dev_err(pci->dev, "Failed to set DMA range %pr\n", entry->res); return ret; } ib_iatu_index++; res_start += window_size; res_size -= window_size; } } return 0; } static void dw_pcie_program_presets(struct dw_pcie_rp *pp, enum pci_bus_speed speed) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); u8 lane_eq_offset, lane_reg_size, cap_id; u8 *presets; u32 cap; int i; if (speed == PCIE_SPEED_8_0GT) { presets = (u8 *)pp->presets.eq_presets_8gts; lane_eq_offset = PCI_SECPCI_LE_CTRL; cap_id = PCI_EXT_CAP_ID_SECPCI; /* For data rate of 8 GT/S each lane equalization control is 16bits wide*/ lane_reg_size = 0x2; } else if (speed == PCIE_SPEED_16_0GT) { presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_16GTS - 1]; lane_eq_offset = PCI_PL_16GT_LE_CTRL; cap_id = PCI_EXT_CAP_ID_PL_16GT; lane_reg_size = 0x1; } else if (speed == PCIE_SPEED_32_0GT) { presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_32GTS - 1]; lane_eq_offset = PCI_PL_32GT_LE_CTRL; cap_id = PCI_EXT_CAP_ID_PL_32GT; lane_reg_size = 0x1; } else if (speed == PCIE_SPEED_64_0GT) { presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_64GTS - 1]; lane_eq_offset = PCI_PL_64GT_LE_CTRL; cap_id = PCI_EXT_CAP_ID_PL_64GT; lane_reg_size = 0x1; } else { return; } if (presets[0] == PCI_EQ_RESV) return; cap = dw_pcie_find_ext_capability(pci, cap_id); if (!cap) return; /* * Write preset values to the registers byte-by-byte for the given * number of lanes and register size. */ for (i = 0; i < pci->num_lanes * lane_reg_size; i++) dw_pcie_writeb_dbi(pci, cap + lane_eq_offset + i, presets[i]); } static void dw_pcie_config_presets(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); enum pci_bus_speed speed = pcie_link_speed[pci->max_link_speed]; /* * Lane equalization settings need to be applied for all data rates the * controller supports and for all supported lanes. */ if (speed >= PCIE_SPEED_8_0GT) dw_pcie_program_presets(pp, PCIE_SPEED_8_0GT); if (speed >= PCIE_SPEED_16_0GT) dw_pcie_program_presets(pp, PCIE_SPEED_16_0GT); if (speed >= PCIE_SPEED_32_0GT) dw_pcie_program_presets(pp, PCIE_SPEED_32_0GT); if (speed >= PCIE_SPEED_64_0GT) dw_pcie_program_presets(pp, PCIE_SPEED_64_0GT); } int dw_pcie_setup_rc(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); u32 val; int ret; /* * Enable DBI read-only registers for writing/updating configuration. * Write permission gets disabled towards the end of this function. */ dw_pcie_dbi_ro_wr_en(pci); dw_pcie_setup(pci); dw_pcie_msi_init(pp); /* Setup RC BARs */ dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004); dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000); /* Setup interrupt pins */ val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE); val &= 0xffff00ff; val |= 0x00000100; dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val); /* Setup bus numbers */ val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS); val &= 0xff000000; val |= 0x00ff0100; dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val); /* Setup command register */ val = dw_pcie_readl_dbi(pci, PCI_COMMAND); val &= 0xffff0000; val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | PCI_COMMAND_SERR; dw_pcie_writel_dbi(pci, PCI_COMMAND, val); dw_pcie_hide_unsupported_l1ss(pci); dw_pcie_config_presets(pp); /* * If the platform provides its own child bus config accesses, it means * the platform uses its own address translation component rather than * ATU, so we should not program the ATU here. */ if (pp->bridge->child_ops == &dw_child_pcie_ops || pp->ecam_enabled) { ret = dw_pcie_iatu_setup(pp); if (ret) return ret; } dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); /* Program correct class for RC */ dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); val |= PORT_LOGIC_SPEED_CHANGE; dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); dw_pcie_dbi_ro_wr_dis(pci); /* * The iMSI-RX module does not support receiving MSI or MSI-X generated * by the Root Port. If iMSI-RX is used as the MSI controller, remove * the MSI and MSI-X capabilities of the Root Port to allow the drivers * to fall back to INTx instead. */ if (pp->use_imsi_rx) { dw_pcie_remove_capability(pci, PCI_CAP_ID_MSI); dw_pcie_remove_capability(pci, PCI_CAP_ID_MSIX); } return 0; } EXPORT_SYMBOL_GPL(dw_pcie_setup_rc); static int dw_pcie_pme_turn_off(struct dw_pcie *pci) { struct dw_pcie_ob_atu_cfg atu = { 0 }; void __iomem *mem; int ret; if (pci->num_ob_windows <= pci->pp.msg_atu_index) return -ENOSPC; if (!pci->pp.msg_res) return -ENOSPC; atu.code = PCIE_MSG_CODE_PME_TURN_OFF; atu.routing = PCIE_MSG_TYPE_R_BC; atu.type = PCIE_ATU_TYPE_MSG; atu.size = resource_size(pci->pp.msg_res); atu.index = pci->pp.msg_atu_index; atu.parent_bus_addr = pci->pp.msg_res->start - pci->parent_bus_offset; ret = dw_pcie_prog_outbound_atu(pci, &atu); if (ret) return ret; mem = ioremap(pci->pp.msg_res->start, pci->region_align); if (!mem) return -ENOMEM; /* A dummy write is converted to a Msg TLP */ writel(0, mem); iounmap(mem); return 0; } int dw_pcie_suspend_noirq(struct dw_pcie *pci) { u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); int ret = 0; u32 val; if (!dw_pcie_link_up(pci)) goto stop_link; /* * If L1SS is supported, then do not put the link into L2 as some * devices such as NVMe expect low resume latency. */ if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1) return 0; if (pci->pp.ops->pme_turn_off) { pci->pp.ops->pme_turn_off(&pci->pp); } else { ret = dw_pcie_pme_turn_off(pci); if (ret) return ret; } /* * Some SoCs do not support reading the LTSSM register after * PME_Turn_Off broadcast. For those SoCs, skip waiting for L2/L3 Ready * state and wait 10ms as recommended in PCIe spec r6.0, sec 5.3.3.2.1. */ if (pci->pp.skip_l23_ready) { mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000); goto stop_link; } ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE || val <= DW_PCIE_LTSSM_DETECT_WAIT, PCIE_PME_TO_L2_TIMEOUT_US/10, PCIE_PME_TO_L2_TIMEOUT_US, false, pci); if (ret) { /* Only log message when LTSSM isn't in DETECT or POLL */ dev_err(pci->dev, "Timeout waiting for L2 entry! LTSSM: 0x%x\n", val); return ret; } /* * Per PCIe r6.0, sec 5.3.3.2.1, software should wait at least * 100ns after L2/L3 Ready before turning off refclock and * main power. This is harmless when no endpoint is connected. */ udelay(1); stop_link: dw_pcie_stop_link(pci); if (pci->pp.ops->deinit) pci->pp.ops->deinit(&pci->pp); pci->suspended = true; return ret; } EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq); int dw_pcie_resume_noirq(struct dw_pcie *pci) { int ret; if (!pci->suspended) return 0; pci->suspended = false; if (pci->pp.ops->init) { ret = pci->pp.ops->init(&pci->pp); if (ret) { dev_err(pci->dev, "Host init failed: %d\n", ret); return ret; } } dw_pcie_setup_rc(&pci->pp); ret = dw_pcie_start_link(pci); if (ret) return ret; ret = dw_pcie_wait_for_link(pci); if (ret) return ret; if (pci->pp.ops->post_init) pci->pp.ops->post_init(&pci->pp); return ret; } EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1