Author | Tokens | Token Proportion | Commits | Commit Proportion |
---|---|---|---|---|
Frederic Barrat | 2143 | 84.77% | 2 | 22.22% |
Christophe Lombard | 375 | 14.83% | 2 | 22.22% |
Zheng Zengkai | 3 | 0.12% | 1 | 11.11% |
Ben Widawsky | 3 | 0.12% | 1 | 11.11% |
Alastair D'Silva | 2 | 0.08% | 1 | 11.11% |
Julia Lawall | 1 | 0.04% | 1 | 11.11% |
ye xingchen | 1 | 0.04% | 1 | 11.11% |
Total | 2528 | 9 |
// SPDX-License-Identifier: GPL-2.0+ // Copyright 2017 IBM Corp. #include <asm/pnv-ocxl.h> #include <asm/opal.h> #include <misc/ocxl-config.h> #include "pci.h" #define PNV_OCXL_TL_P9_RECV_CAP 0x000000000000000Full #define PNV_OCXL_ACTAG_MAX 64 /* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */ #define PNV_OCXL_PASID_BITS 15 #define PNV_OCXL_PASID_MAX ((1 << PNV_OCXL_PASID_BITS) - 1) #define AFU_PRESENT (1 << 31) #define AFU_INDEX_MASK 0x3F000000 #define AFU_INDEX_SHIFT 24 #define ACTAG_MASK 0xFFF struct actag_range { u16 start; u16 count; }; struct npu_link { struct list_head list; int domain; int bus; int dev; u16 fn_desired_actags[8]; struct actag_range fn_actags[8]; bool assignment_done; }; static struct list_head links_list = LIST_HEAD_INIT(links_list); static DEFINE_MUTEX(links_list_lock); /* * opencapi actags handling: * * When sending commands, the opencapi device references the memory * context it's targeting with an 'actag', which is really an alias * for a (BDF, pasid) combination. When it receives a command, the NPU * must do a lookup of the actag to identify the memory context. The * hardware supports a finite number of actags per link (64 for * POWER9). * * The device can carry multiple functions, and each function can have * multiple AFUs. Each AFU advertises in its config space the number * of desired actags. The host must configure in the config space of * the AFU how many actags the AFU is really allowed to use (which can * be less than what the AFU desires). * * When a PCI function is probed by the driver, it has no visibility * about the other PCI functions and how many actags they'd like, * which makes it impossible to distribute actags fairly among AFUs. * * Unfortunately, the only way to know how many actags a function * desires is by looking at the data for each AFU in the config space * and add them up. Similarly, the only way to know how many actags * all the functions of the physical device desire is by adding the * previously computed function counts. Then we can match that against * what the hardware supports. * * To get a comprehensive view, we use a 'pci fixup': at the end of * PCI enumeration, each function counts how many actags its AFUs * desire and we save it in a 'npu_link' structure, shared between all * the PCI functions of a same device. Therefore, when the first * function is probed by the driver, we can get an idea of the total * count of desired actags for the device, and assign the actags to * the AFUs, by pro-rating if needed. */ static int find_dvsec_from_pos(struct pci_dev *dev, int dvsec_id, int pos) { int vsec = pos; u16 vendor, id; while ((vsec = pci_find_next_ext_capability(dev, vsec, OCXL_EXT_CAP_ID_DVSEC))) { pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET, &vendor); pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id); if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id) return vsec; } return 0; } static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx) { int vsec = 0; u8 idx; while ((vsec = find_dvsec_from_pos(dev, OCXL_DVSEC_AFU_CTRL_ID, vsec))) { pci_read_config_byte(dev, vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX, &idx); if (idx == afu_idx) return vsec; } return 0; } static int get_max_afu_index(struct pci_dev *dev, int *afu_idx) { int pos; u32 val; pos = pci_find_dvsec_capability(dev, PCI_VENDOR_ID_IBM, OCXL_DVSEC_FUNC_ID); if (!pos) return -ESRCH; pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val); if (val & AFU_PRESENT) *afu_idx = (val & AFU_INDEX_MASK) >> AFU_INDEX_SHIFT; else *afu_idx = -1; return 0; } static int get_actag_count(struct pci_dev *dev, int afu_idx, int *actag) { int pos; u16 actag_sup; pos = find_dvsec_afu_ctrl(dev, afu_idx); if (!pos) return -ESRCH; pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, &actag_sup); *actag = actag_sup & ACTAG_MASK; return 0; } static struct npu_link *find_link(struct pci_dev *dev) { struct npu_link *link; list_for_each_entry(link, &links_list, list) { /* The functions of a device all share the same link */ if (link->domain == pci_domain_nr(dev->bus) && link->bus == dev->bus->number && link->dev == PCI_SLOT(dev->devfn)) { return link; } } /* link doesn't exist yet. Allocate one */ link = kzalloc(sizeof(struct npu_link), GFP_KERNEL); if (!link) return NULL; link->domain = pci_domain_nr(dev->bus); link->bus = dev->bus->number; link->dev = PCI_SLOT(dev->devfn); list_add(&link->list, &links_list); return link; } static void pnv_ocxl_fixup_actag(struct pci_dev *dev) { struct pci_controller *hose = pci_bus_to_host(dev->bus); struct pnv_phb *phb = hose->private_data; struct npu_link *link; int rc, afu_idx = -1, i, actag; if (!machine_is(powernv)) return; if (phb->type != PNV_PHB_NPU_OCAPI) return; mutex_lock(&links_list_lock); link = find_link(dev); if (!link) { dev_warn(&dev->dev, "couldn't update actag information\n"); mutex_unlock(&links_list_lock); return; } /* * Check how many actags are desired for the AFUs under that * function and add it to the count for the link */ rc = get_max_afu_index(dev, &afu_idx); if (rc) { /* Most likely an invalid config space */ dev_dbg(&dev->dev, "couldn't find AFU information\n"); afu_idx = -1; } link->fn_desired_actags[PCI_FUNC(dev->devfn)] = 0; for (i = 0; i <= afu_idx; i++) { /* * AFU index 'holes' are allowed. So don't fail if we * can't read the actag info for an index */ rc = get_actag_count(dev, i, &actag); if (rc) continue; link->fn_desired_actags[PCI_FUNC(dev->devfn)] += actag; } dev_dbg(&dev->dev, "total actags for function: %d\n", link->fn_desired_actags[PCI_FUNC(dev->devfn)]); mutex_unlock(&links_list_lock); } DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag); static u16 assign_fn_actags(u16 desired, u16 total) { u16 count; if (total <= PNV_OCXL_ACTAG_MAX) count = desired; else count = PNV_OCXL_ACTAG_MAX * desired / total; return count; } static void assign_actags(struct npu_link *link) { u16 actag_count, range_start = 0, total_desired = 0; int i; for (i = 0; i < 8; i++) total_desired += link->fn_desired_actags[i]; for (i = 0; i < 8; i++) { if (link->fn_desired_actags[i]) { actag_count = assign_fn_actags( link->fn_desired_actags[i], total_desired); link->fn_actags[i].start = range_start; link->fn_actags[i].count = actag_count; range_start += actag_count; WARN_ON(range_start >= PNV_OCXL_ACTAG_MAX); } pr_debug("link %x:%x:%x fct %d actags: start=%d count=%d (desired=%d)\n", link->domain, link->bus, link->dev, i, link->fn_actags[i].start, link->fn_actags[i].count, link->fn_desired_actags[i]); } link->assignment_done = true; } int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, u16 *supported) { struct npu_link *link; mutex_lock(&links_list_lock); link = find_link(dev); if (!link) { dev_err(&dev->dev, "actag information not found\n"); mutex_unlock(&links_list_lock); return -ENODEV; } /* * On p9, we only have 64 actags per link, so they must be * shared by all the functions of the same adapter. We counted * the desired actag counts during PCI enumeration, so that we * can allocate a pro-rated number of actags to each function. */ if (!link->assignment_done) assign_actags(link); *base = link->fn_actags[PCI_FUNC(dev->devfn)].start; *enabled = link->fn_actags[PCI_FUNC(dev->devfn)].count; *supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)]; mutex_unlock(&links_list_lock); return 0; } EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag); int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count) { struct npu_link *link; int i, rc = -EINVAL; /* * The number of PASIDs (process address space ID) which can * be used by a function depends on how many functions exist * on the device. The NPU needs to be configured to know how * many bits are available to PASIDs and how many are to be * used by the function BDF identifier. * * We only support one AFU-carrying function for now. */ mutex_lock(&links_list_lock); link = find_link(dev); if (!link) { dev_err(&dev->dev, "actag information not found\n"); mutex_unlock(&links_list_lock); return -ENODEV; } for (i = 0; i < 8; i++) if (link->fn_desired_actags[i] && (i == PCI_FUNC(dev->devfn))) { *count = PNV_OCXL_PASID_MAX; rc = 0; break; } mutex_unlock(&links_list_lock); dev_dbg(&dev->dev, "%d PASIDs available for function\n", rc ? 0 : *count); return rc; } EXPORT_SYMBOL_GPL(pnv_ocxl_get_pasid_count); static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf) { int shift, idx; WARN_ON(templ > PNV_OCXL_TL_MAX_TEMPLATE); idx = (PNV_OCXL_TL_MAX_TEMPLATE - templ) / 2; shift = 4 * (1 - ((PNV_OCXL_TL_MAX_TEMPLATE - templ) % 2)); buf[idx] |= rate << shift; } int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, char *rate_buf, int rate_buf_size) { if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE) return -EINVAL; /* * The TL capabilities are a characteristic of the NPU, so * we go with hard-coded values. * * The receiving rate of each template is encoded on 4 bits. * * On P9: * - templates 0 -> 3 are supported * - templates 0, 1 and 3 have a 0 receiving rate * - template 2 has receiving rate of 1 (extra cycle) */ memset(rate_buf, 0, rate_buf_size); set_templ_rate(2, 1, rate_buf); *cap = PNV_OCXL_TL_P9_RECV_CAP; return 0; } EXPORT_SYMBOL_GPL(pnv_ocxl_get_tl_cap); int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, uint64_t rate_buf_phys, int rate_buf_size) { struct pci_controller *hose = pci_bus_to_host(dev->bus); struct pnv_phb *phb = hose->private_data; int rc; if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE) return -EINVAL; rc = opal_npu_tl_set(phb->opal_id, dev->devfn, cap, rate_buf_phys, rate_buf_size); if (rc) { dev_err(&dev->dev, "Can't configure host TL: %d\n", rc); return -EINVAL; } return 0; } EXPORT_SYMBOL_GPL(pnv_ocxl_set_tl_conf); int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq) { int rc; rc = of_property_read_u32(dev->dev.of_node, "ibm,opal-xsl-irq", hwirq); if (rc) { dev_err(&dev->dev, "Can't get translation interrupt for device\n"); return rc; } return 0; } EXPORT_SYMBOL_GPL(pnv_ocxl_get_xsl_irq); void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, void __iomem *tfc, void __iomem *pe_handle) { iounmap(dsisr); iounmap(dar); iounmap(tfc); iounmap(pe_handle); } EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_xsl_regs); int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, void __iomem **dar, void __iomem **tfc, void __iomem **pe_handle) { u64 reg; int i, j, rc = 0; void __iomem *regs[4]; /* * opal stores the mmio addresses of the DSISR, DAR, TFC and * PE_HANDLE registers in a device tree property, in that * order */ for (i = 0; i < 4; i++) { rc = of_property_read_u64_index(dev->dev.of_node, "ibm,opal-xsl-mmio", i, ®); if (rc) break; regs[i] = ioremap(reg, 8); if (!regs[i]) { rc = -EINVAL; break; } } if (rc) { dev_err(&dev->dev, "Can't map translation mmio registers\n"); for (j = i - 1; j >= 0; j--) iounmap(regs[j]); } else { *dsisr = regs[0]; *dar = regs[1]; *tfc = regs[2]; *pe_handle = regs[3]; } return rc; } EXPORT_SYMBOL_GPL(pnv_ocxl_map_xsl_regs); struct spa_data { u64 phb_opal_id; u32 bdfn; }; int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, void **platform_data) { struct pci_controller *hose = pci_bus_to_host(dev->bus); struct pnv_phb *phb = hose->private_data; struct spa_data *data; u32 bdfn; int rc; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; bdfn = pci_dev_id(dev); rc = opal_npu_spa_setup(phb->opal_id, bdfn, virt_to_phys(spa_mem), PE_mask); if (rc) { dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc); kfree(data); return rc; } data->phb_opal_id = phb->opal_id; data->bdfn = bdfn; *platform_data = (void *) data; return 0; } EXPORT_SYMBOL_GPL(pnv_ocxl_spa_setup); void pnv_ocxl_spa_release(void *platform_data) { struct spa_data *data = (struct spa_data *) platform_data; int rc; rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0); WARN_ON(rc); kfree(data); } EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release); int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle) { struct spa_data *data = (struct spa_data *) platform_data; return opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle); } EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe_from_cache); int pnv_ocxl_map_lpar(struct pci_dev *dev, uint64_t lparid, uint64_t lpcr, void __iomem **arva) { struct pci_controller *hose = pci_bus_to_host(dev->bus); struct pnv_phb *phb = hose->private_data; u64 mmio_atsd; int rc; /* ATSD physical address. * ATSD LAUNCH register: write access initiates a shoot down to * initiate the TLB Invalidate command. */ rc = of_property_read_u64_index(hose->dn, "ibm,mmio-atsd", 0, &mmio_atsd); if (rc) { dev_info(&dev->dev, "No available ATSD found\n"); return rc; } /* Assign a register set to a Logical Partition and MMIO ATSD * LPARID register to the required value. */ rc = opal_npu_map_lpar(phb->opal_id, pci_dev_id(dev), lparid, lpcr); if (rc) { dev_err(&dev->dev, "Error mapping device to LPAR: %d\n", rc); return rc; } *arva = ioremap(mmio_atsd, 24); if (!(*arva)) { dev_warn(&dev->dev, "ioremap failed - mmio_atsd: %#llx\n", mmio_atsd); rc = -ENOMEM; } return rc; } EXPORT_SYMBOL_GPL(pnv_ocxl_map_lpar); void pnv_ocxl_unmap_lpar(void __iomem *arva) { iounmap(arva); } EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_lpar); void pnv_ocxl_tlb_invalidate(void __iomem *arva, unsigned long pid, unsigned long addr, unsigned long page_size) { unsigned long timeout = jiffies + (HZ * PNV_OCXL_ATSD_TIMEOUT); u64 val = 0ull; int pend; u8 size; if (!(arva)) return; if (addr) { /* load Abbreviated Virtual Address register with * the necessary value */ val |= FIELD_PREP(PNV_OCXL_ATSD_AVA_AVA, addr >> (63-51)); out_be64(arva + PNV_OCXL_ATSD_AVA, val); } /* Write access initiates a shoot down to initiate the * TLB Invalidate command */ val = PNV_OCXL_ATSD_LNCH_R; val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_RIC, 0b10); if (addr) val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_IS, 0b00); else { val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_IS, 0b01); val |= PNV_OCXL_ATSD_LNCH_OCAPI_SINGLETON; } val |= PNV_OCXL_ATSD_LNCH_PRS; /* Actual Page Size to be invalidated * 000 4KB * 101 64KB * 001 2MB * 010 1GB */ size = 0b101; if (page_size == 0x1000) size = 0b000; if (page_size == 0x200000) size = 0b001; if (page_size == 0x40000000) size = 0b010; val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_AP, size); val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_PID, pid); out_be64(arva + PNV_OCXL_ATSD_LNCH, val); /* Poll the ATSD status register to determine when the * TLB Invalidate has been completed. */ val = in_be64(arva + PNV_OCXL_ATSD_STAT); pend = val >> 63; while (pend) { if (time_after_eq(jiffies, timeout)) { pr_err("%s - Timeout while reading XTS MMIO ATSD status register (val=%#llx, pidr=0x%lx)\n", __func__, val, pid); return; } cpu_relax(); val = in_be64(arva + PNV_OCXL_ATSD_STAT); pend = val >> 63; } } EXPORT_SYMBOL_GPL(pnv_ocxl_tlb_invalidate);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1