cregit-Linux how code gets into the kernel

Release 4.14 drivers/vfio/pci/vfio_pci.c

Directory: drivers/vfio/pci
/*
 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
 *     Author: Alex Williamson <alex.williamson@redhat.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * Derived from original vfio:
 * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
 * Author: Tom Lyon, pugs@cisco.com
 */


#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/device.h>
#include <linux/eventfd.h>
#include <linux/file.h>
#include <linux/interrupt.h>
#include <linux/iommu.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/notifier.h>
#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include <linux/vfio.h>
#include <linux/vgaarb.h>

#include "vfio_pci_private.h"


#define DRIVER_VERSION  "0.2"

#define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"

#define DRIVER_DESC     "VFIO PCI - User Level meta-driver"


static char ids[1024] __initdata;
module_param_string(ids, ids, sizeof(ids), 0);
MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified");


static bool nointxmask;
module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(nointxmask,
		  "Disable support for PCI 2.3 style INTx masking.  If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");

#ifdef CONFIG_VFIO_PCI_VGA

static bool disable_vga;
module_param(disable_vga, bool, S_IRUGO);
MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci");
#endif


static bool disable_idle_d3;
module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(disable_idle_d3,
		 "Disable using the PCI D3 low power state for idle, unused devices");

static DEFINE_MUTEX(driver_lock);


static inline bool vfio_vga_disabled(void) { #ifdef CONFIG_VFIO_PCI_VGA return disable_vga; #else return true; #endif }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson22100.00%1100.00%
Total22100.00%1100.00%

/* * Our VGA arbiter participation is limited since we don't know anything * about the device itself. However, if the device is the only VGA device * downstream of a bridge and VFIO VGA support is disabled, then we can * safely return legacy VGA IO and memory as not decoded since the user * has no way to get to it and routing can be disabled externally at the * bridge. */
static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga) { struct vfio_pci_device *vdev = opaque; struct pci_dev *tmp = NULL, *pdev = vdev->pdev; unsigned char max_busnr; unsigned int decodes; if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus)) return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; max_busnr = pci_bus_max_busnr(pdev->bus); decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) { if (tmp == pdev || pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) || pci_is_root_bus(tmp->bus)) continue; if (tmp->bus->number >= pdev->bus->number && tmp->bus->number <= max_busnr) { pci_dev_put(tmp); decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; break; } } return decodes; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson168100.00%1100.00%
Total168100.00%1100.00%


static inline bool vfio_pci_is_vga(struct pci_dev *pdev) { return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson23100.00%1100.00%
Total23100.00%1100.00%


static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev) { struct resource *res; int bar; struct vfio_pci_dummy_resource *dummy_res; INIT_LIST_HEAD(&vdev->dummy_resources_list); for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) { res = vdev->pdev->resource + bar; if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP)) goto no_mmap; if (!(res->flags & IORESOURCE_MEM)) goto no_mmap; /* * The PCI core shouldn't set up a resource with a * type but zero size. But there may be bugs that * cause us to do that. */ if (!resource_size(res)) goto no_mmap; if (resource_size(res) >= PAGE_SIZE) { vdev->bar_mmap_supported[bar] = true; continue; } if (!(res->start & ~PAGE_MASK)) { /* * Add a dummy resource to reserve the remainder * of the exclusive page in case that hot-add * device's bar is assigned into it. */ dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL); if (dummy_res == NULL) goto no_mmap; dummy_res->resource.name = "vfio sub-page reserved"; dummy_res->resource.start = res->end + 1; dummy_res->resource.end = res->start + PAGE_SIZE - 1; dummy_res->resource.flags = res->flags; if (request_resource(res->parent, &dummy_res->resource)) { kfree(dummy_res); goto no_mmap; } dummy_res->index = bar; list_add(&dummy_res->res_next, &vdev->dummy_resources_list); vdev->bar_mmap_supported[bar] = true; continue; } /* * Here we don't handle the case when the BAR is not page * aligned because we can't expect the BAR will be * assigned into the same location in a page in guest * when we passthrough the BAR. And it's hard to access * this BAR in userspace because we have no way to get * the BAR's location in a page. */ no_mmap: vdev->bar_mmap_supported[bar] = false; } }

Contributors

PersonTokensPropCommitsCommitProp
Yongji Xie261100.00%1100.00%
Total261100.00%1100.00%

static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev); static void vfio_pci_disable(struct vfio_pci_device *vdev); /* * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND * _and_ the ability detect when the device is asserting INTx via PCI_STATUS. * If a device implements the former but not the latter we would typically * expect broken_intx_masking be set and require an exclusive interrupt. * However since we do have control of the device's ability to assert INTx, * we can instead pretend that the device does not implement INTx, virtualizing * the pin register to report zero and maintaining DisINTx set on the host. */
static bool vfio_pci_nointx(struct pci_dev *pdev) { switch (pdev->vendor) { case PCI_VENDOR_ID_INTEL: switch (pdev->device) { /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */ case 0x1572: case 0x1574: case 0x1580 ... 0x1581: case 0x1583 ... 0x158b: case 0x37d0 ... 0x37d2: return true; default: return false; } } return false; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson62100.00%2100.00%
Total62100.00%2100.00%


static int vfio_pci_enable(struct vfio_pci_device *vdev) { struct pci_dev *pdev = vdev->pdev; int ret; u16 cmd; u8 msix_pos; pci_set_power_state(pdev, PCI_D0); /* Don't allow our initial saved state to include busmaster */ pci_clear_master(pdev); ret = pci_enable_device(pdev); if (ret) return ret; /* If reset fails because of the device lock, fail this path entirely */ ret = pci_try_reset_function(pdev); if (ret == -EAGAIN) { pci_disable_device(pdev); return ret; } vdev->reset_works = !ret; pci_save_state(pdev); vdev->pci_saved_state = pci_store_saved_state(pdev); if (!vdev->pci_saved_state) pr_debug("%s: Couldn't store %s saved state\n", __func__, dev_name(&pdev->dev)); if (likely(!nointxmask)) { if (vfio_pci_nointx(pdev)) { dev_info(&pdev->dev, "Masking broken INTx support\n"); vdev->nointx = true; pci_intx(pdev, 0); } else vdev->pci_2_3 = pci_intx_mask_supported(pdev); } pci_read_config_word(pdev, PCI_COMMAND, &cmd); if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) { cmd &= ~PCI_COMMAND_INTX_DISABLE; pci_write_config_word(pdev, PCI_COMMAND, cmd); } ret = vfio_config_init(vdev); if (ret) { kfree(vdev->pci_saved_state); vdev->pci_saved_state = NULL; pci_disable_device(pdev); return ret; } msix_pos = pdev->msix_cap; if (msix_pos) { u16 flags; u32 table; pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags); pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table); vdev->msix_bar = table & PCI_MSIX_TABLE_BIR; vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET; vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16; } else vdev->msix_bar = 0xFF; if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev)) vdev->has_vga = true; if (vfio_pci_is_vga(pdev) && pdev->vendor == PCI_VENDOR_ID_INTEL && IS_ENABLED(CONFIG_VFIO_PCI_IGD)) { ret = vfio_pci_igd_init(vdev); if (ret) { dev_warn(&vdev->pdev->dev, "Failed to setup Intel IGD regions\n"); vfio_pci_disable(vdev); return ret; } } vfio_pci_probe_mmaps(vdev); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson39897.79%1280.00%
Yongji Xie51.23%16.67%
Björn Helgaas40.98%213.33%
Total407100.00%15100.00%


static void vfio_pci_disable(struct vfio_pci_device *vdev) { struct pci_dev *pdev = vdev->pdev; struct vfio_pci_dummy_resource *dummy_res, *tmp; int i, bar; /* Stop the device from further DMA */ pci_clear_master(pdev); vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, vdev->irq_type, 0, 0, NULL); vdev->virq_disabled = false; for (i = 0; i < vdev->num_regions; i++) vdev->region[i].ops->release(vdev, &vdev->region[i]); vdev->num_regions = 0; kfree(vdev->region); vdev->region = NULL; /* don't krealloc a freed pointer */ vfio_config_free(vdev); for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) { if (!vdev->barmap[bar]) continue; pci_iounmap(pdev, vdev->barmap[bar]); pci_release_selected_regions(pdev, 1 << bar); vdev->barmap[bar] = NULL; } list_for_each_entry_safe(dummy_res, tmp, &vdev->dummy_resources_list, res_next) { list_del(&dummy_res->res_next); release_resource(&dummy_res->resource); kfree(dummy_res); } vdev->needs_reset = true; /* * If we have saved state, restore it. If we can reset the device, * even better. Resetting with current state seems better than * nothing, but saving and restoring current state without reset * is just busy work. */ if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) { pr_info("%s: Couldn't reload %s saved state\n", __func__, dev_name(&pdev->dev)); if (!vdev->reset_works) goto out; pci_save_state(pdev); } /* * Disable INTx and MSI, presumably to avoid spurious interrupts * during reset. Stolen from pci_reset_function() */ pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); /* * Try to reset the device. The success of this is dependent on * being able to lock the device, which is not always possible. */ if (vdev->reset_works && !pci_try_reset_function(pdev)) vdev->needs_reset = false; pci_restore_state(pdev); out: pci_disable_device(pdev); vfio_pci_try_bus_reset(vdev); if (!disable_idle_d3) pci_set_power_state(pdev, PCI_D3hot); }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson28387.35%990.00%
Yongji Xie4112.65%110.00%
Total324100.00%10100.00%


static void vfio_pci_release(void *device_data) { struct vfio_pci_device *vdev = device_data; mutex_lock(&driver_lock); if (!(--vdev->refcnt)) { vfio_spapr_pci_eeh_release(vdev->pdev); vfio_pci_disable(vdev); } mutex_unlock(&driver_lock); module_put(THIS_MODULE); }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson4984.48%266.67%
Gavin Shan915.52%133.33%
Total58100.00%3100.00%


static int vfio_pci_open(void *device_data) { struct vfio_pci_device *vdev = device_data; int ret = 0; if (!try_module_get(THIS_MODULE)) return -ENODEV; mutex_lock(&driver_lock); if (!vdev->refcnt) { ret = vfio_pci_enable(vdev); if (ret) goto error; vfio_spapr_pci_eeh_open(vdev->pdev); } vdev->refcnt++; error: mutex_unlock(&driver_lock); if (ret) module_put(THIS_MODULE); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson7376.84%266.67%
Gavin Shan2223.16%133.33%
Total95100.00%3100.00%


static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type) { if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { u8 pin; pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); if (IS_ENABLED(CONFIG_VFIO_PCI_INTX) && !vdev->nointx && pin) return 1; } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) { u8 pos; u16 flags; pos = vdev->pdev->msi_cap; if (pos) { pci_read_config_word(vdev->pdev, pos + PCI_MSI_FLAGS, &flags); return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1); } } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) { u8 pos; u16 flags; pos = vdev->pdev->msix_cap; if (pos) { pci_read_config_word(vdev->pdev, pos + PCI_MSIX_FLAGS, &flags); return (flags & PCI_MSIX_FLAGS_QSIZE) + 1; } } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) { if (pci_is_pcie(vdev->pdev)) return 1; } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) { return 1; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson16683.84%342.86%
Vijay Mohan Pandarathil199.60%114.29%
Frank Blaschka52.53%114.29%
Gavin Shan42.02%114.29%
Björn Helgaas42.02%114.29%
Total198100.00%7100.00%


static int vfio_pci_count_devs(struct pci_dev *pdev, void *data) { (*(int *)data)++; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson28100.00%2100.00%
Total28100.00%2100.00%

struct vfio_pci_fill_info { int max; int cur; struct vfio_pci_dependent_device *devices; };
static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) { struct vfio_pci_fill_info *fill = data; struct iommu_group *iommu_group; if (fill->cur == fill->max) return -EAGAIN; /* Something changed, try again */ iommu_group = iommu_group_get(&pdev->dev); if (!iommu_group) return -EPERM; /* Cannot reset non-isolated devices */ fill->devices[fill->cur].group_id = iommu_group_id(iommu_group); fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus); fill->devices[fill->cur].bus = pdev->bus->number; fill->devices[fill->cur].devfn = pdev->devfn; fill->cur++; iommu_group_put(iommu_group); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson141100.00%2100.00%
Total141100.00%2100.00%

struct vfio_pci_group_entry { struct vfio_group *group; int id; }; struct vfio_pci_group_info { int count; struct vfio_pci_group_entry *groups; };
static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data) { struct vfio_pci_group_info *info = data; struct iommu_group *group; int id, i; group = iommu_group_get(&pdev->dev); if (!group) return -EPERM; id = iommu_group_id(group); for (i = 0; i < info->count; i++) if (info->groups[i].id == id) break; iommu_group_put(group); return (i == info->count) ? -EINVAL : 0; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson106100.00%2100.00%
Total106100.00%2100.00%


static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot) { for (; pdev; pdev = pdev->bus->self) if (pdev->bus == slot->bus) return (pdev->slot == slot); return false; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson51100.00%1100.00%
Total51100.00%1100.00%

struct vfio_pci_walk_info { int (*fn)(struct pci_dev *, void *data); void *data; struct pci_dev *pdev; bool slot; int ret; };
static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data) { struct vfio_pci_walk_info *walk = data; if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot)) walk->ret = walk->fn(pdev, walk->data); return walk->ret; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson60100.00%1100.00%
Total60100.00%1100.00%


static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev, int (*fn)(struct pci_dev *, void *data), void *data, bool slot) { struct vfio_pci_walk_info walk = { .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0, }; pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk); return walk.ret; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson81100.00%1100.00%
Total81100.00%1100.00%


static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev, struct vfio_info_cap *caps) { struct vfio_region_info_cap_sparse_mmap *sparse; size_t end, size; int nr_areas = 2, i = 0, ret; end = pci_resource_len(vdev->pdev, vdev->msix_bar); /* If MSI-X table is aligned to the start or end, only one area */ if (((vdev->msix_offset & PAGE_MASK) == 0) || (PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) >= end)) nr_areas = 1; size = sizeof(*sparse) + (nr_areas * sizeof(*sparse->areas)); sparse = kzalloc(size, GFP_KERNEL); if (!sparse) return -ENOMEM; sparse->nr_areas = nr_areas; if (vdev->msix_offset & PAGE_MASK) { sparse->areas[i].offset = 0; sparse->areas[i].size = vdev->msix_offset & PAGE_MASK; i++; } if (PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) < end) { sparse->areas[i].offset = PAGE_ALIGN(vdev->msix_offset + vdev->msix_size); sparse->areas[i].size = end - sparse->areas[i].offset; i++; } ret = vfio_info_add_capability(caps, VFIO_REGION_INFO_CAP_SPARSE_MMAP, sparse); kfree(sparse); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson23093.50%375.00%
Kirti Wankhede166.50%125.00%
Total246100.00%4100.00%


int vfio_pci_register_dev_region(struct vfio_pci_device *vdev, unsigned int type, unsigned int subtype, const struct vfio_pci_regops *ops, size_t size, u32 flags, void *data) { struct vfio_pci_region *region; region = krealloc(vdev->region, (vdev->num_regions + 1) * sizeof(*region), GFP_KERNEL); if (!region) return -ENOMEM; vdev->region = region; vdev->region[vdev->num_regions].type = type; vdev->region[vdev->num_regions].subtype = subtype; vdev->region[vdev->num_regions].ops = ops; vdev->region[vdev->num_regions].size = size; vdev->region[vdev->num_regions].flags = flags; vdev->region[vdev->num_regions].data = data; vdev->num_regions++; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Alex Williamson165100.00%1100.00%
Total165100.00%1100.00%


static long vfio_pci_ioctl(void *device_data, unsigned int cmd, unsigned long arg) { struct vfio_pci_device *vdev = device_data; unsigned long minsz; if (cmd == VFIO_DEVICE_GET_INFO) { struct vfio_device_info info; minsz = offsetofend(struct vfio_device_info, num_irqs); if (copy_from_user(&info, (void __user *)arg, minsz)) return -EFAULT; if (info.argsz < minsz) return -EINVAL; info.flags = VFIO_DEVICE_FLAGS_PCI; if (vdev->reset_works) info.flags |= VFIO_DEVICE_FLAGS_RESET; info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; info.num_irqs = VFIO_PCI_NUM_IRQS; return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { struct pci_dev *pdev = vdev->pdev; struct vfio_region_info info; struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; int i, ret; minsz = offsetofend(struct vfio_region_info, offset); if (copy_from_user(&info, (void __user *)arg, minsz)) return -EFAULT; if (info.argsz < minsz) return -EINVAL; switch (info.index) { case VFIO_PCI_CONFIG_REGION_INDEX: info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); info.size = pdev->cfg_size; info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE; break; case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX