Release 4.13 drivers/nvme/host/pci.c

Directory: drivers/nvme/host
/*
 * NVM Express device driver
 * Copyright (c) 2011-2014, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/aer.h>
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/blk-mq.h>
#include <linux/blk-mq-pci.h>
#include <linux/dmi.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/poison.h>
#include <linux/t10-pi.h>
#include <linux/timer.h>
#include <linux/types.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <asm/unaligned.h>
#include <linux/sed-opal.h>

#include "nvme.h"


#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))

#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))

/*
 * We handle AEN commands ourselves and don't even let the
 * block layer know about them.
 */

#define NVME_AQ_BLKMQ_DEPTH	(NVME_AQ_DEPTH - NVME_NR_AERS)


static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);


static bool use_cmb_sqes = true;
module_param(use_cmb_sqes, bool, 0644);
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");


static unsigned int max_host_mem_size_mb = 128;
module_param(max_host_mem_size_mb, uint, 0444);
MODULE_PARM_DESC(max_host_mem_size_mb,
	"Maximum Host Memory Buffer (HMB) size per controller (in MiB)");

static int io_queue_depth_set(const char *val, const struct kernel_param *kp);

static const struct kernel_param_ops io_queue_depth_ops = {
	.set = io_queue_depth_set,
	.get = param_get_int,
};


static int io_queue_depth = 1024;
module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2");

struct nvme_dev;
struct nvme_queue;

static void nvme_process_cq(struct nvme_queue *nvmeq);
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);

/*
 * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 */

struct nvme_dev {
	
struct nvme_queue **queues;
	
struct blk_mq_tag_set tagset;
	
struct blk_mq_tag_set admin_tagset;
	
u32 __iomem *dbs;
	
struct device *dev;
	
struct dma_pool *prp_page_pool;
	
struct dma_pool *prp_small_pool;
	
unsigned online_queues;
	
unsigned max_qid;
	
int q_depth;
	
u32 db_stride;
	
void __iomem *bar;
	
unsigned long bar_mapped_size;
	
struct work_struct remove_work;
	
struct mutex shutdown_lock;
	
bool subsystem;
	
void __iomem *cmb;
	
dma_addr_t cmb_dma_addr;
	
u64 cmb_size;
	
u32 cmbsz;
	
u32 cmbloc;
	
struct nvme_ctrl ctrl;
	
struct completion ioq_wait;

	/* shadow doorbell buffer support: */
	
u32 *dbbuf_dbs;
	
dma_addr_t dbbuf_dbs_dma_addr;
	
u32 *dbbuf_eis;
	
dma_addr_t dbbuf_eis_dma_addr;

	/* host memory buffer support: */
	
u64 host_mem_size;
	
u32 nr_host_mem_descs;
	
dma_addr_t host_mem_descs_dma;
	
struct nvme_host_mem_buf_desc *host_mem_descs;
	
void **host_mem_desc_bufs;
};



static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
{
	int n = 0, ret;

	ret = kstrtoint(val, 10, &n);
	if (ret != 0 || n < 2)
		return -EINVAL;

	return param_set_int(val, kp);
}

Contributors
Person Tokens Prop Commits CommitProp
weiping zhang 58 100.00% 1 100.00%
Total 58 100.00% 1 100.00%



static inline unsigned int sq_idx(unsigned int qid, u32 stride)
{
	return qid * 2 * stride;
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 22 100.00% 1 100.00%
Total 22 100.00% 1 100.00%



static inline unsigned int cq_idx(unsigned int qid, u32 stride)
{
	return (qid * 2 + 1) * stride;
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 26 100.00% 1 100.00%
Total 26 100.00% 1 100.00%



static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
{
	return container_of(ctrl, struct nvme_dev, ctrl);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 25 100.00% 1 100.00%
Total 25 100.00% 1 100.00%

/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */

struct nvme_queue {
	
struct device *q_dmadev;
	
struct nvme_dev *dev;
	
spinlock_t q_lock;
	
struct nvme_command *sq_cmds;
	
struct nvme_command __iomem *sq_cmds_io;
	
volatile struct nvme_completion *cqes;
	
struct blk_mq_tags **tags;
	
dma_addr_t sq_dma_addr;
	
dma_addr_t cq_dma_addr;
	
u32 __iomem *q_db;
	
u16 q_depth;
	
s16 cq_vector;
	
u16 sq_tail;
	
u16 cq_head;
	
u16 qid;
	
u8 cq_phase;
	
u8 cqe_seen;
	
u32 *dbbuf_sq_db;
	
u32 *dbbuf_cq_db;
	
u32 *dbbuf_sq_ei;
	
u32 *dbbuf_cq_ei;
};

/*
 * The nvme_iod describes the data in an I/O, including the list of PRP
 * entries.  You can't see it in this data structure because C doesn't let
 * me express that.  Use nvme_init_iod to ensure there's enough space
 * allocated to store the PRP list.
 */

struct nvme_iod {
	
struct nvme_request req;
	
struct nvme_queue *nvmeq;
	
int aborted;
	
int npages;		/* In the PRP list. 0 means small pool in use */
	
int nents;		/* Used in scatterlist */
	
int length;		/* Of data, in bytes */
	
dma_addr_t first_dma;
	
struct scatterlist meta_sg; /* metadata requires single contiguous buffer */
	
struct scatterlist *sg;
	
struct scatterlist inline_sg[0];
};

/*
 * Check we didin't inadvertently grow the command struct
 */


static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
	BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 68 44.74% 1 20.00%
Helen Mae Koike Fornazier 61 40.13% 1 20.00%
Vishal Verma 11 7.24% 1 20.00%
Keith Busch 10 6.58% 1 20.00%
Johannes Thumshirn 2 1.32% 1 20.00%
Total 152 100.00% 5 100.00%



static inline unsigned int nvme_dbbuf_size(u32 stride)
{
	return ((num_possible_cpus() + 1) * 8 * stride);
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 25 100.00% 1 100.00%
Total 25 100.00% 1 100.00%



static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
{
	unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);

	if (dev->dbbuf_dbs)
		return 0;

	dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
					    &dev->dbbuf_dbs_dma_addr,
					    GFP_KERNEL);
	if (!dev->dbbuf_dbs)
		return -ENOMEM;
	dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
					    &dev->dbbuf_eis_dma_addr,
					    GFP_KERNEL);
	if (!dev->dbbuf_eis) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
		dev->dbbuf_dbs = NULL;
		return -ENOMEM;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 121 100.00% 1 100.00%
Total 121 100.00% 1 100.00%



static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
{
	unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);

	if (dev->dbbuf_dbs) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
		dev->dbbuf_dbs = NULL;
	}
	if (dev->dbbuf_eis) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
		dev->dbbuf_eis = NULL;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 84 100.00% 1 100.00%
Total 84 100.00% 1 100.00%



static void nvme_dbbuf_init(struct nvme_dev *dev,
			    struct nvme_queue *nvmeq, int qid)
{
	if (!dev->dbbuf_dbs || !qid)
		return;

	nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 106 100.00% 1 100.00%
Total 106 100.00% 1 100.00%



static void nvme_dbbuf_set(struct nvme_dev *dev)
{
	struct nvme_command c;

	if (!dev->dbbuf_dbs)
		return;

	memset(&c, 0, sizeof(c));
	c.dbbuf.opcode = nvme_admin_dbbuf;
	c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
	c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);

	if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
		dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
		/* Free memory and continue on */
		nvme_dbbuf_dma_free(dev);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 87 81.31% 1 20.00%
Matthew Wilcox 12 11.21% 1 20.00%
Christoph Hellwig 3 2.80% 1 20.00%
Keith Busch 3 2.80% 1 20.00%
Vishal Verma 2 1.87% 1 20.00%
Total 107 100.00% 5 100.00%



static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
{
	return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 34 91.89% 1 50.00%
Matthew Wilcox 3 8.11% 1 50.00%
Total 37 100.00% 2 100.00%

/* Update dbbuf and return true if an MMIO is required */


static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db,
					      volatile u32 *dbbuf_ei)
{
	if (dbbuf_db) {
		u16 old_value;

		/*
                 * Ensure that the queue is written before updating
                 * the doorbell in memory
                 */
		wmb();

		old_value = *dbbuf_db;
		*dbbuf_db = value;

		if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value))
			return false;
	}

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Helen Mae Koike Fornazier 56 93.33% 1 33.33%
Keith Busch 2 3.33% 1 33.33%
Matthew Wilcox 2 3.33% 1 33.33%
Total 60 100.00% 3 100.00%

/*
 * Max size of iod being embedded in the request payload
 */

#define NVME_INT_PAGES		2

#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->ctrl.page_size)

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */


static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
	unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
				      dev->ctrl.page_size);
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

Contributors
Person Tokens Prop Commits CommitProp
Jens Axboe 42 91.30% 1 50.00%
Christoph Hellwig 4 8.70% 1 50.00%
Total 46 100.00% 2 100.00%



static unsigned int nvme_iod_alloc_size(struct nvme_dev *dev,
		unsigned int size, unsigned int nseg)
{
	return sizeof(__le64 *) * nvme_npages(size, dev) +
			sizeof(struct scatterlist) * nseg;
}

Contributors
Person Tokens Prop Commits CommitProp
Jens Axboe 27 64.29% 1 50.00%
Christoph Hellwig 15 35.71% 1 50.00%
Total 42 100.00% 2 100.00%



static unsigned int nvme_cmd_size(struct nvme_dev *dev)
{
	return sizeof(struct nvme_iod) +
		nvme_iod_alloc_size(dev, NVME_INT_BYTES(dev), NVME_INT_PAGES);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 28 90.32% 1 50.00%
Jens Axboe 3 9.68% 1 50.00%
Total 31 100.00% 2 100.00%



static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
{
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[0];

	WARN_ON(hctx_idx != 0);
	WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
	WARN_ON(nvmeq->tags);

	hctx->driver_data = nvmeq;
	nvmeq->tags = &dev->admin_tagset.tags[0];
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Matias Björling 41 45.05% 1 25.00%
Keith Busch 38 41.76% 2 50.00%
Matthew Wilcox 12 13.19% 1 25.00%
Total 91 100.00% 4 100.00%



static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->tags = NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 30 100.00% 1 100.00%
Total 30 100.00% 1 100.00%



static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
{
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];

	if (!nvmeq->tags)
		nvmeq->tags = &dev->tagset.tags[hctx_idx];

	WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
	hctx->driver_data = nvmeq;
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Matias Björling 50 58.14% 1 25.00%
Matthew Wilcox 19 22.09% 2 50.00%
Keith Busch 17 19.77% 1 25.00%
Total 86 100.00% 4 100.00%



static int nvme_init_request(struct blk_mq_tag_set *set, struct request *req,
		unsigned int hctx_idx, unsigned int numa_node)
{
	struct nvme_dev *dev = set->driver_data;
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0;
	struct nvme_queue *nvmeq = dev->queues[queue_idx];

	BUG_ON(!nvmeq);
	iod->nvmeq = nvmeq;
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Matias Björling 54 61.36% 1 20.00%
Christoph Hellwig 28 31.82% 3 60.00%
Matthew Wilcox 6 6.82% 1 20.00%
Total 88 100.00% 5 100.00%



static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
{
	struct nvme_dev *dev = set->driver_data;

	return blk_mq_pci_map_queues(set, to_pci_dev(dev->dev));
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 33 100.00% 1 100.00%
Total 33 100.00% 1 100.00%

/**
 * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
 * @nvmeq: The queue to use
 * @cmd: The command to send
 *
 * Safe to use from interrupt context
 */


static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
						struct nvme_command *cmd)
{
	u16 tail = nvmeq->sq_tail;

	if (nvmeq->sq_cmds_io)
		memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
	else
		memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));

	if (++tail == nvmeq->q_depth)
		tail = 0;
	if (nvme_dbbuf_update_and_check_event(tail, nvmeq->dbbuf_sq_db,
					      nvmeq->dbbuf_sq_ei))
		writel(tail, nvmeq->q_db);
	nvmeq->sq_tail = tail;
}

Contributors
Person Tokens Prop Commits CommitProp
Matias Björling 35 31.53% 1 10.00%
Matthew Wilcox 30 27.03% 3 30.00%
Keith Busch 17 15.32% 4 40.00%
Helen Mae Koike Fornazier 15 13.51% 1 10.00%
Jon Derrick 14 12.61% 1 10.00%
Total 111 100.00% 10 100.00%



static __le64 **iod_list(struct request *req)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	return (__le64 **)(iod->sg + blk_rq_nr_phys_segments(req));
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 18 45.00% 2 28.57%
Matias Björling 12 30.00% 1 14.29%
Matthew Wilcox 5 12.50% 2 28.57%
Jens Axboe 4 10.00% 1 14.29%
Keith Busch 1 2.50% 1 14.29%
Total 40 100.00% 7 100.00%



static blk_status_t nvme_init_iod(struct request *rq, struct nvme_dev *dev)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);
	int nseg = blk_rq_nr_phys_segments(rq);
	unsigned int size = blk_rq_payload_bytes(rq);

	if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) {
		iod->sg = kmalloc(nvme_iod_alloc_size(dev, size, nseg), GFP_ATOMIC);
		if (!iod->sg)
			return BLK_STS_RESOURCE;
	} else {
		iod->sg = iod->inline_sg;
	}

	iod->aborted = 0;
	iod->npages = -1;
	iod->nents = 0;
	iod->length = size;

	return BLK_STS_OK;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 76 60.80% 4 40.00%
Keith Busch 18 14.40% 3 30.00%
Matthew Wilcox 13 10.40% 1 10.00%
Jens Axboe 12 9.60% 1 10.00%
Matias Björling 6 4.80% 1 10.00%
Total 125 100.00% 10 100.00%



static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	const int last_prp = dev->ctrl.page_size / 8 - 1;
	int i;
	__le64 **list = iod_list(req);
	dma_addr_t prp_dma = iod->first_dma;

	if (iod->npages == 0)
		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
	for (i = 0; i < iod->npages; i++) {
		__le64 *prp_list = list[i];
		dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
		dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
		prp_dma = next_prp_dma;
	}

	if (iod->sg != iod->inline_sg)
		kfree(iod->sg);
}

Contributors
Person Tokens Prop Commits CommitProp
Shane Michael Matthews 62 41.06% 1 7.14%
Matthew Wilcox 55 36.42% 8 57.14%
Christoph Hellwig 16 10.60% 2 14.29%
Matias Björling 12 7.95% 1 7.14%
Jens Axboe 4 2.65% 1 7.14%
Keith Busch 2 1.32% 1 7.14%
Total 151 100.00% 14 100.00%

#ifdef CONFIG_BLK_DEV_INTEGRITY


static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == v)
		pi->ref_tag = cpu_to_be32(p);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 19 51.35% 1 20.00%
Matthew Wilcox 11 29.73% 3 60.00%
Matias Björling 7 18.92% 1 20.00%
Total 37 100.00% 5 100.00%



static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == p)
		pi->ref_tag = cpu_to_be32(v);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 29 78.38% 2 50.00%
Matthew Wilcox 6 16.22% 1 25.00%
Matias Björling 2 5.41% 1 25.00%
Total 37 100.00% 4 100.00%

/**
 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 *
 * The virtual start sector is the one that was originally submitted by the
 * block layer. Due to partitioning, MD/DM cloning, etc. the actual physical
 * start sector may be different. Remap protection information to match the
 * physical LBA on writes, and back to the original seed on reads.
 *
 * Type 0 and 3 do not have a ref tag, so no remapping required.
 */


static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
	struct nvme_ns *ns = req->rq_disk->private_data;
	struct bio_integrity_payload *bip;
	struct t10_pi_tuple *pi;
	void *p, *pmap;
	u32 i, nlb, ts, phys, virt;

	if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
		return;

	bip = bio_integrity(req->bio);
	if (!bip)
		return;

	pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;

	p = pmap;
	virt = bip_get_seed(bip);
	phys = nvme_block_nr(ns, blk_rq_pos(req));
	nlb = (blk_rq_bytes(req) >> ns->lba_shift);
	ts = ns->disk->queue->integrity.tuple_size;

	for (i = 0; i < nlb; i++, virt++, phys++) {
		pi = (struct t10_pi_tuple *)p;
		dif_swap(phys, virt, pi);
		p += ts;
	}
	kunmap_atomic(pmap);
}
Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 169 80.48% 5 45.45%
Matias Björling 27 12.86% 1 9.09%
Jon Derrick 6 2.86% 1 9.09%
Matthew Wilcox 6 2.86% 2 18.18%
Dan J Williams 1 0.48% 1 9.09%
Martin K. Petersen 1 0.48% 1 9.09%
Total 210 100.00% 11 100.00%

#else /* CONFIG_BLK_DEV_INTEGRITY */


static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
}
Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 20 71.43% 1 33.33%
Matthew Wilcox 7 25.00% 1 33.33%
Sunad Bhandary 1 3.57% 1 33.33%
Total 28 100.00% 3 100.00%



static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 11 68.75% 1 33.33%
Matthew Wilcox 4 25.00% 1 33.33%
Shane Michael Matthews 1 6.25% 1 33.33%
Total 16 100.00% 3 100.00%



static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 10 62.50% 1 33.33%
Jens Axboe 3 18.75% 1 33.33%
Matthew Wilcox 3 18.75% 1 33.33%
Total 16 100.00% 3 100.00%

#endif



static blk_status_t nvme_setup_prps(struct nvme_dev *dev, struct request *req)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct dma_pool *pool;
	int length = blk_rq_payload_bytes(req);
	struct scatterlist *sg = iod->sg;
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
	u32 page_size = dev->ctrl.page_size;
	int offset = dma_addr & (page_size - 1);
	__le64 *prp_list;
	__le64 **list = iod_list(req);
	dma_addr_t prp_dma;
	int nprps, i;

	length -= (page_size - offset);
	if (length <= 0)
		return BLK_STS_OK;

	dma_len -= (page_size - offset);
	if (dma_len) {
		dma_addr += (page_size - offset);
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

	if (length <= page_size) {
		iod->first_dma = dma_addr;
		return BLK_STS_OK;
	}

	nprps = DIV_ROUND_UP(length, page_size);
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
		iod->npages = 0;
	} else {
		pool = dev->prp_page_pool;
		iod->npages = 1;
	}

	prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
	if (!prp_list) {
		iod->first_dma = dma_addr;
		iod->npages = -1;
		return BLK_STS_RESOURCE;
	}
	list[0] = prp_list;
	iod->first_dma = prp_dma;
	i = 0;
	for (;;) {
		if (i == page_size >> 3) {
			__le64 *old_prp_list = prp_list;
			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
			if (!prp_list)
				return BLK_STS_RESOURCE;
			list[iod->npages++] = prp_list;
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
		if (unlikely(dma_len < 0))
			goto bad_sgl;
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

	return BLK_STS_OK;

 bad_sgl:
	if (WARN_ONCE(1, "Invalid SGL for payload:%d nents:%d\n",
				blk_rq_payload_bytes(req), iod->nents)) {
		for_each_sg(iod->sg, sg, iod->nents, i) {
			dma_addr_t phys = sg_phys(sg);
			pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
			       "dma_address:%pad dma_length:%d\n", i, &phys,
					sg->offset, sg->length,
					&sg_dma_address(sg),
					sg_dma_len(sg));
		}
	}
	return BLK_STS_IOERR;

}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 239 46.41% 7 25.93%
Matthew Wilcox 161 31.26% 10 37.04%
Shane Michael Matthews 59 11.46% 1 3.70%
Matias Björling 25 4.85% 1 3.70%
Christoph Hellwig 13 2.52% 4 14.81%
Jens Axboe 7 1.36% 1 3.70%
Murali Iyer 7 1.36% 1 3.70%
Nisheeth Bhat 2 0.39% 1 3.70%
Jon Derrick 2 0.39% 1 3.70%
Total 515 100.00% 27 100.00%



static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
		struct nvme_command *cmnd)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct request_queue *q = req->q;
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;
	blk_status_t ret = BLK_STS_IOERR;

	sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
	iod->nents = blk_rq_map_sg(q, req, iod->sg);
	if (!iod->nents)
		goto out;

	ret = BLK_STS_RESOURCE;
	if (!dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, dma_dir,
				DMA_ATTR_NO_WARN))
		goto out;

	ret = nvme_setup_prps(dev, req);
	if (ret != BLK_STS_OK)
		goto out_unmap;

	ret = BLK_STS_IOERR;
	if (blk_integrity_rq(req)) {
		if (blk_rq_count_integrity_sg(q, req->bio) != 1)
			goto out_unmap;

		sg_init_table(&iod->meta_sg, 1);
		if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
			goto out_unmap;

		if (rq_data_dir(req))
			nvme_dif_remap(req, nvme_dif_prep);

		if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
			goto out_unmap;
	}

	cmnd->rw.dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd->rw.dptr.prp2 = cpu_to_le64(iod->first_dma);
	if (blk_integrity_rq(req))
		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
	return BLK_STS_OK;

out_unmap:
	dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
out:
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 215 66.77% 7 25.00%
Matthew Wilcox 63 19.57% 9 32.14%
Keith Busch 33 10.25% 6 21.43%
Santosh Yaraganavi 3 0.93% 1 3.57%
Mauricio Faria de Oliveira 3 0.93% 1 3.57%
Dan Carpenter 2 0.62% 1 3.57%
Linus Torvalds 1 0.31% 1 3.57%
Jon Derrick 1 0.31% 1 3.57%
Matias Björling 1 0.31% 1 3.57%
Total 322 100.00% 28 100.00%



static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;

	if (iod->nents) {
		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
		if (blk_integrity_rq(req)) {
			if (!rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_complete);
			dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
		}
	}

	nvme_cleanup_cmd(req);
	nvme_free_iod(dev, req);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 90 77.59% 5 45.45%
Keith Busch 17 14.66% 2 18.18%
Matthew Wilcox 9 7.76% 4 36.36%
Total 116 100.00% 11 100.00%

/*
 * NOTE: ns is NULL when called on the admin queue.
 */


static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
			 const struct blk_mq_queue_data *bd)
{
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
	struct nvme_dev *dev = nvmeq->dev;
	struct request *req = bd->rq;
	struct nvme_command cmnd;
	blk_status_t ret;

	ret = nvme_setup_cmd(ns, req, &cmnd);
	if (ret)
		return ret;

	ret = nvme_init_iod(req, dev);
	if (ret)
		goto out_free_cmd;

	if (blk_rq_nr_phys_segments(req)) {
		ret = nvme_map_data(dev, req, &cmnd);
		if (ret)
			goto out_cleanup_iod;
	}

	blk_mq_start_request(req);

	spin_lock_irq(&nvmeq->q_lock);
	if (unlikely(nvmeq->cq_vector < 0)) {
		ret = BLK_STS_IOERR;
		spin_unlock_irq(&nvmeq->q_lock);
		goto out_cleanup_iod;
	}
	__nvme_submit_cmd(nvmeq, &cmnd);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
	return BLK_STS_OK;
out_cleanup_iod:
	nvme_free_iod(dev, req);
out_free_cmd:
	nvme_cleanup_cmd(req);
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 75 35.89% 8 28.57%
Keith Busch 61 29.19% 11 39.29%
Matias Björling 37 17.70% 1 3.57%
Matthew Wilcox 20 9.57% 3 10.71%
Jon Derrick 9 4.31% 1 3.57%
Ming Lin 4 1.91% 2 7.14%
Shane Michael Matthews 2 0.96% 1 3.57%
Jens Axboe 1 0.48% 1 3.57%
Total 209 100.00% 28 100.00%



static void nvme_pci_complete_rq(struct request *req)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);

	nvme_unmap_data(iod->nvmeq->dev, req);
	nvme_complete_rq(req);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 22 59.46% 2 40.00%
Christoph Hellwig 15 40.54% 3 60.00%
Total 37 100.00% 5 100.00%

/* We read the CQE phase first to check if the rest of the entry is valid */


static inline bool nvme_cqe_valid(struct nvme_queue *nvmeq, u16 head,
		u16 phase)
{
	return (le16_to_cpu(nvmeq->cqes[head].status) & 1) == phase;
}

Contributors
Person Tokens Prop Commits CommitProp
Marta Rybczynska 37 100.00% 1 100.00%
Total 37 100.00% 1 100.00%



static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
{
	u16 head = nvmeq->cq_head;

	if (likely(nvmeq->cq_vector >= 0)) {
		if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
						      nvmeq->dbbuf_cq_ei))
			writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Sagi Grimberg 39 62.90% 1 14.29%
Matthew Wilcox 8 12.90% 3 42.86%
Marta Rybczynska 7 11.29% 1 14.29%
Matias Björling 7 11.29% 1 14.29%
Jens Axboe 1 1.61% 1 14.29%
Total 62 100.00% 7 100.00%



static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
		struct nvme_completion *cqe)
{
	struct request *req;

	if (unlikely(cqe->command_id >= nvmeq->q_depth)) {
		dev_warn(nvmeq->dev->ctrl.device,
			"invalid id %d completed on queue %d\n",
			cqe->command_id, le16_to_cpu(cqe->sq_id));
		return;
	}

	/*
         * AEN requests are special as they don't time out and can
         * survive any kind of queue freeze and often don't respond to
         * aborts.  We don't even bother to allocate a struct request
         * for them but rather special case them here.
         */
	if (unlikely(nvmeq->qid == 0 &&
			cqe->command_id >= NVME_AQ_BLKMQ_DEPTH)) {
		nvme_complete_async_event(&nvmeq->dev->ctrl,
				cqe->status, &cqe->result);
		return;
	}

	nvmeq->cqe_seen = 1;
	req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
	nvme_end_request(req, cqe->status, cqe->result);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 78 57.78% 7 36.84%
Sagi Grimberg 22 16.30% 2 10.53%
Matthew Wilcox 11 8.15% 3 15.79%
Matias Björling 11 8.15% 1 5.26%
Keith Busch 9 6.67% 3 15.79%
Shane Michael Matthews 2 1.48% 1 5.26%
Jens Axboe 1 0.74% 1 5.26%
Marta Rybczynska 1 0.74% 1 5.26%
Total 135 100.00% 19 100.00%



static inline bool nvme_read_cqe(struct nvme_queue *nvmeq,
		struct nvme_completion *cqe)
{
	if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) {
		*cqe = nvmeq->cqes[nvmeq->cq_head];

		if (++nvmeq->cq_head == nvmeq->q_depth) {
			nvmeq->cq_head = 0;
			nvmeq->cq_phase = !nvmeq->cq_phase;
		}
		return true;
	}
	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
Sagi Grimberg 49 61.25% 2 20.00%
Matthew Wilcox 13 16.25% 3 30.00%
Matias Björling 9 11.25% 1 10.00%
Keith Busch 5 6.25% 3 30.00%
Helen Mae Koike Fornazier 4 5.00% 1 10.00%
Total 80 100.00% 10 100.00%



static void nvme_process_cq(struct nvme_queue *nvmeq)
{
	struct nvme_completion cqe;
	int consumed = 0;

	while (nvme_read_cqe(nvmeq, &cqe)) {
		nvme_handle_cqe(nvmeq, &cqe);
		consumed++;
	}

	if (consumed)
		nvme_ring_cq_doorbell(nvmeq);
}

Contributors
Person Tokens Prop Commits CommitProp
Sagi Grimberg 33 63.46% 3 30.00%
Matthew Wilcox 9 17.31% 3 30.00%
Keith Busch 5 9.62% 2 20.00%
Jens Axboe 4 7.69% 1 10.00%
Vishal Verma 1 1.92% 1 10.00%
Total 52 100.00% 10 100.00%



static irqreturn_t nvme_irq(int irq, void *data)
{
	irqreturn_t result;
	struct nvme_queue *nvmeq = data;
	spin_lock(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
	nvmeq->cqe_seen = 0;
	spin_unlock(&nvmeq->q_lock);
	return result;
}

Contributors
Person Tokens Prop Commits CommitProp
Matias Björling 35 55.56% 1 14.29%
Keith Busch 22 34.92% 3 42.86%
Matthew Wilcox 6 9.52% 3 42.86%
Total 63 100.00% 7 100.00%



static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
	if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase))
		return IRQ_WAKE_THREAD;
	return IRQ_NONE;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 13 31.71% 5 45.45%
Matias Björling 12 29.27% 1 9.09%
Marta Rybczynska 9 21.95% 1 9.09%
Matthew Wilcox 5 12.20% 3 27.27%
Christoph Hellwig 2 4.88% 1 9.09%
Total 41 100.00% 11 100.00%



static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
{
	struct nvme_completion cqe;
	int found = 0, consumed = 0;

	if (!nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase))
		return 0;

	spin_lock_irq(&nvmeq->q_lock);
	while (nvme_read_cqe(nvmeq, &cqe)) {
		nvme_handle_cqe(nvmeq, &cqe);
		consumed++;

		if (tag == cqe.command_id) {
			found = 1;
			break;
		}
       }

	if (consumed)
		nvme_ring_cq_doorbell(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);

	return found;
}

Contributors
Person Tokens Prop Commits CommitProp
Sagi Grimberg 56 49.56% 1 12.50%
Jens Axboe 37 32.74% 1 12.50%
Keith Busch 10 8.85% 3 37.50%
Marta Rybczynska 4 3.54% 1 12.50%
Dan J Williams 3 2.65% 1 12.50%
Matias Björling 3 2.65% 1 12.50%
Total 113 100.00% 8 100.00%



static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	return __nvme_poll(nvmeq, tag);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 32 100.00% 1 100.00%
Total 32 100.00% 1 100.00%



static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl, int aer_idx)
{
	struct nvme_dev *dev = to_nvme_dev(ctrl);
	struct nvme_queue *nvmeq = dev->queues[0];
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
	c.common.command_id = NVME_AQ_BLKMQ_DEPTH + aer_idx;

	spin_lock_irq(&nvmeq->q_lock);
	__nvme_submit_cmd(nvmeq, &c);
	spin_unlock_irq(&nvmeq->q_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 78 82.11% 6 54.55%
Keith Busch 9 9.47% 3 27.27%
Matthew Wilcox 8 8.42% 2 18.18%
Total 95 100.00% 11 100.00%



static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 57 81.43% 2 20.00%
Christoph Hellwig 6 8.57% 3 30.00%
Keith Busch 5 7.14% 3 30.00%
Matias Björling 2 2.86% 2 20.00%
Total 70 100.00% 10 100.00%



static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

	/*
         * Note: we (ab)use the fact the the prp fields survive if no data
         * is attached to the request.
         */
	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);

	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 105 79.55% 3 27.27%
Keith Busch 16 12.12% 4 36.36%
Christoph Hellwig 7 5.30% 3 27.27%
Dan J Williams 4 3.03% 1 9.09%
Total 132 100.00% 11 100.00%



static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG;

	/*
         * Note: we (ab)use the fact the the prp fields survive if no data
         * is attached to the request.
         */
	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 63 49.22% 3 37.50%
Keith Busch 56 43.75% 1 12.50%
Christoph Hellwig 9 7.03% 4 50.00%
Total 128 100.00% 8 100.00%



static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 15 62.50% 1 50.00%
Matthew Wilcox 9 37.50% 1 50.00%
Total 24 100.00% 2 100.00%



static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 18 75.00% 2 66.67%
Matthew Wilcox 6 25.00% 1 33.33%
Total 24 100.00% 3 100.00%



static void abort_endio(struct request *req, blk_status_t error)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = iod->nvmeq;

	dev_warn(nvmeq->dev->ctrl.device,
		 "Abort status: 0x%x", nvme_req(req)->status);
	atomic_inc(&nvmeq->dev->ctrl.abort_limit);
	blk_mq_free_request(req);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 47 67.14% 5 71.43%
Keith Busch 18 25.71% 1 14.29%
Sagi Grimberg 5 7.14% 1 14.29%
Total 70 100.00% 7 100.00%



static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
{

	/* If true, indicates loss of adapter communication, possibly by a
         * NVMe Subsystem reset.
         */
	bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);

	/* If there is a reset ongoing, we shouldn't reset again. */
	if (dev->ctrl.state == NVME_CTRL_RESETTING)
		return false;

	/* We shouldn't reset unless the controller is on fatal error state
         * _or_ if we lost the communication with it.
         */
	if (!(csts & NVME_CSTS_CFS) && !nssro)
		return false;

	/* If PCI error recovery process is happening, we cannot reset or
         * the recovery mechanism will surely fail.
         */
	if (pci_channel_offline(to_pci_dev(dev->dev)))
		return false;

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 71 92.21% 5 62.50%
Christoph Hellwig 5 6.49% 2 25.00%
Matias Björling 1 1.30% 1 12.50%
Total 77 100.00% 8 100.00%



static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
{
	/* Read a config register to help see what died. */
	u16 pci_status;
	int result;

	result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
				      &pci_status);
	if (result == PCIBIOS_SUCCESSFUL)
		dev_warn(dev->ctrl.device,
			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
			 csts, pci_status);
	else
		dev_warn(dev->ctrl.device,
			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
			 csts, result);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 75 100.00% 1 100.00%
Total 75 100.00% 1 100.00%



static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = iod->nvmeq;
	struct nvme_dev *dev = nvmeq->dev;
	struct request *abort_req;
	struct nvme_command cmd;
	u32 csts = readl(dev->bar + NVME_REG_CSTS);

	/*
         * Reset immediately if the controller is failed
         */
	if (nvme_should_reset(dev, csts)) {
		nvme_warn_reset(dev, csts);
		nvme_dev_disable(dev, false);
		nvme_reset_ctrl(&dev->ctrl);
		return BLK_EH_HANDLED;
	}

	/*
         * Did we miss an interrupt?
         */
	if (__nvme_poll(nvmeq, req->tag)) {
		dev_warn(dev->ctrl.device,
			 "I/O %d QID %d timeout, completion polled\n",
			 req->tag, nvmeq->qid);
		return BLK_EH_HANDLED;
	}

	/*
         * Shutdown immediately if controller times out while starting. The
         * reset work will see the pci device disabled when it gets the forced
         * cancellation error. All outstanding requests are completed on
         * shutdown, so we return BLK_EH_HANDLED.
         */
	if (dev->ctrl.state == NVME_CTRL_RESETTING) {
		dev_warn(dev->ctrl.device,
			 "I/O %d QID %d timeout, disable controller\n",
			 req->tag, nvmeq->qid);
		nvme_dev_disable(dev, false);
		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
		return BLK_EH_HANDLED;
	}

	/*
         * Shutdown the controller immediately and schedule a reset if the
         * command was already aborted once before and still hasn't been
         * returned to the driver, or if this is the admin queue.
         */
	if (!nvmeq->qid || iod->aborted) {
		dev_warn(dev->ctrl.device,
			 "I/O %d QID %d timeout, reset controller\n",
			 req->tag, nvmeq->qid);
		nvme_dev_disable(dev, false);
		nvme_reset_ctrl(&dev->ctrl);

		/*
                 * Mark the request as handled, since the inline shutdown
                 * forces all outstanding requests to complete.
                 */
		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
		return BLK_EH_HANDLED;
	}

	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
		atomic_inc(&dev->ctrl.abort_limit);
		return BLK_EH_RESET_TIMER;
	}
	iod->aborted = 1;

	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
	cmd.abort.cid = req->tag;
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);

	dev_warn(nvmeq->dev->ctrl.device,
		"I/O %d QID %d timeout, aborting\n",
		 req->tag, nvmeq->qid);

	abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
			BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
	if (IS_ERR(abort_req)) {
		atomic_inc(&dev->ctrl.abort_limit);
		return BLK_EH_RESET_TIMER;
	}

	abort_req->timeout = ADMIN_TIMEOUT;
	abort_req->end_io_data = NULL;
	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);

	/*
         * The aborted req will be completed on receiving the abort req.
         * We enable the timer again. If hit twice, it'll cause a device reset,
         * as the device then is in a faulty state.
         */
	return BLK_EH_RESET_TIMER;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 208 49.52% 15 48.39%
Christoph Hellwig 159 37.86% 12 38.71%
Matias Björling 32 7.62% 1 3.23%
Sagi Grimberg 11 2.62% 1 3.23%
Matthew Wilcox 10 2.38% 2 6.45%
Total 420 100.00% 31 100.00%



static void nvme_free_queue(struct nvme_queue *nvmeq)
{
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
	if (nvmeq->sq_cmds)
		dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
	kfree(nvmeq);
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 28 40.00% 5 45.45%
Matias Björling 23 32.86% 1 9.09%
Keith Busch 17 24.29% 4 36.36%
Jon Derrick 2 2.86% 1 9.09%
Total 70 100.00% 11 100.00%



static void nvme_free_queues(struct nvme_dev *dev, int lowest)
{
	int i;

	for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
		struct nvme_queue *nvmeq = dev->queues[i];
		dev->ctrl.queue_count--;
		dev->queues[i] = NULL;
		nvme_free_queue(nvmeq);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 47 66.20% 5 50.00%
Matias Björling 15 21.13% 1 10.00%
Sagi Grimberg 4 5.63% 1 10.00%
Matthew Wilcox 4 5.63% 2 20.00%
kaoudis 1 1.41% 1 10.00%
Total 71 100.00% 10 100.00%

/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */


static int nvme_suspend_queue(struct nvme_queue *nvmeq)
{
	int vector;

	spin_lock_irq(&nvmeq->q_lock);
	if (nvmeq->cq_vector == -1) {
		spin_unlock_irq(&nvmeq->q_lock);
		return 1;
	}
	vector = nvmeq->cq_vector;
	nvmeq->dev->online_queues--;
	nvmeq->cq_vector = -1;
	spin_unlock_irq(&nvmeq->q_lock);

	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
		blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q);

	pci_free_irq(to_pci_dev(nvmeq->dev->dev), vector, nvmeq);

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 57 48.72% 5 41.67%
Matias Björling 24 20.51% 1 8.33%
Matthew Wilcox 19 16.24% 3 25.00%
Christoph Hellwig 16 13.68% 2 16.67%
Sagi Grimberg 1 0.85% 1 8.33%
Total 117 100.00% 12 100.00%



static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
{
	struct nvme_queue *nvmeq = dev->queues[0];

	if (!nvmeq)
		return;
	if (nvme_suspend_queue(nvmeq))
		return;

	if (shutdown)
		nvme_shutdown_ctrl(&dev->ctrl);
	else
		nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 60 68.18% 7 58.33%
Matthew Wilcox 14 15.91% 3 25.00%
Matias Björling 11 12.50% 1 8.33%
Sagi Grimberg 3 3.41% 1 8.33%
Total 88 100.00% 12 100.00%



static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
				int entry_size)
{
	int q_depth = dev->q_depth;
	unsigned q_size_aligned = roundup(q_depth * entry_size,
					  dev->ctrl.page_size);

	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
		mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
		q_depth = div_u64(mem_per_q, entry_size);

		/*
                 * Ensure the reduced q_depth is above some threshold where it
                 * would be better to map queues in system memory with the
                 * original depth
                 */
		if (q_depth < 64)
			return -ENOMEM;
	}

	return q_depth;
}

Contributors
Person Tokens Prop Commits CommitProp
Jon Derrick 54 54.00% 2 18.18%
Matias Björling 15 15.00% 1 9.09%
Matthew Wilcox 12 12.00% 3 27.27%
Christoph Hellwig 10 10.00% 2 18.18%
Keith Busch 9 9.00% 3 27.27%
Total 100 100.00% 11 100.00%



static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
				int qid, int depth)
{
	if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
		unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
						      dev->ctrl.page_size);
		nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
		nvmeq->sq_cmds_io = dev->cmb + offset;
	} else {
		nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
					&nvmeq->sq_dma_addr, GFP_KERNEL);
		if (!nvmeq->sq_cmds)
			return -ENOMEM;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Jon Derrick 67 54.03% 1 8.33%
Keith Busch 23 18.55% 3 25.00%
Matthew Wilcox 14 11.29% 3 25.00%
Matias Björling 12 9.68% 1 8.33%
Christoph Hellwig 4 3.23% 2 16.67%
Ramachandra Rao Gajula 3 2.42% 1 8.33%
Alexander Gordeev 1 0.81% 1 8.33%
Total 124 100.00% 12 100.00%



static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
							int depth, int node)
{
	struct nvme_queue *nvmeq = kzalloc_node(sizeof(*nvmeq), GFP_KERNEL,
							node);
	if (!nvmeq)
		return NULL;

	nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
	if (!nvmeq->cqes)
		goto free_nvmeq;

	if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
		goto free_cqdma;

	nvmeq->q_dmadev = dev->dev;
	nvmeq->dev = dev;
	spin_lock_init(&nvmeq->q_lock);
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
	nvmeq->q_depth = depth;
	nvmeq->qid = qid;
	nvmeq->cq_vector = -1;
	dev->queues[qid] = nvmeq;
	dev->ctrl.queue_count++;

	return nvmeq;

 free_cqdma:
	dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
							nvmeq->cq_dma_addr);
 free_nvmeq:
	kfree(nvmeq);
	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 93 41.70% 6 26.09%
Jon Derrick 75 33.63% 3 13.04%
Keith Busch 31 13.90% 5 21.74%
Shaohua Li 6 2.69% 1 4.35%
Christoph Hellwig 5 2.24% 1 4.35%
Matias Björling 3 1.35% 1 4.35%
Ramachandra Rao Gajula 3 1.35% 1 4.35%
Haiyan Hu 2 0.90% 1 4.35%
Sagi Grimberg 2 0.90% 1 4.35%
Joe Perches 1 0.45% 1 4.35%
Alexander Gordeev 1 0.45% 1 4.35%
Greg Kroah-Hartman 1 0.45% 1 4.35%
Total 223 100.00% 23 100.00%



static int queue_request_irq(struct nvme_queue *nvmeq)
{
	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
	int nr = nvmeq->dev->ctrl.instance;

	if (use_threaded_interrupts) {
		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
				nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
	} else {
		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
				NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 57 61.29% 2 20.00%
Matthew Wilcox 22 23.66% 4 40.00%
Jon Derrick 8 8.60% 1 10.00%
Keith Busch 6 6.45% 3 30.00%
Total 93 100.00% 10 100.00%



static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
{
	struct nvme_dev *dev = nvmeq->dev;

	spin_lock_irq(&nvmeq->q_lock);
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
	nvme_dbbuf_init(dev, nvmeq, qid);
	dev->online_queues++;
	spin_unlock_irq(&nvmeq->q_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 71 65.14% 4 33.33%
Matias Björling 12 11.01% 1 8.33%
Matthew Wilcox 11 10.09% 4 33.33%
Helen Mae Koike Fornazier 9 8.26% 1 8.33%
Greg Kroah-Hartman 4 3.67% 1 8.33%
Haiyan Hu 2 1.83% 1 8.33%
Total 109 100.00% 12 100.00%



static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;

	nvmeq->cq_vector = qid - 1;
	result = adapter_alloc_cq(dev, qid, nvmeq);
	if (result < 0)
		return result;

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
		goto release_cq;

	result = queue_request_irq(nvmeq);
	if (result < 0)
		goto release_sq;

	nvme_init_queue(nvmeq, qid);
	return result;

 release_sq:
	adapter_delete_sq(dev, qid);
 release_cq:
	adapter_delete_cq(dev, qid);
	return result;
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 47 38.84% 6 35.29%
Keith Busch 43 35.54% 6 35.29%
Jon Derrick 19 15.70% 1 5.88%
Matias Björling 8 6.61% 1 5.88%
Ramachandra Rao Gajula 2 1.65% 1 5.88%
Christoph Hellwig 1 0.83% 1 5.88%
Greg Kroah-Hartman 1 0.83% 1 5.88%
Total 121 100.00% 17 100.00%


static const struct blk_mq_ops nvme_mq_admin_ops = {
	.queue_rq	= nvme_queue_rq,
	.complete	= nvme_pci_complete_rq,
	.init_hctx	= nvme_admin_init_hctx,
	.exit_hctx      = nvme_admin_exit_hctx,
	.init_request	= nvme_init_request,
	.timeout	= nvme_timeout,
};


static const struct blk_mq_ops nvme_mq_ops = {
	.queue_rq	= nvme_queue_rq,
	.complete	= nvme_pci_complete_rq,
	.init_hctx	= nvme_init_hctx,
	.init_request	= nvme_init_request,
	.map_queues	= nvme_pci_map_queues,
	.timeout	= nvme_timeout,
	.poll		= nvme_poll,
};



static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
		/*
                 * If the controller was reset during removal, it's possible
                 * user requests may be waiting on a stopped queue. Start the
                 * queue to flush these to completion.
                 */
		blk_mq_unquiesce_queue(dev->ctrl.admin_q);
		blk_cleanup_queue(dev->ctrl.admin_q);
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 45 77.59% 5 55.56%
Christoph Hellwig 6 10.34% 1 11.11%
Matias Björling 4 6.90% 1 11.11%
Matthew Wilcox 2 3.45% 1 11.11%
Sagi Grimberg 1 1.72% 1 11.11%
Total 58 100.00% 9 100.00%



static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
	if (!dev->ctrl.admin_q) {
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;

		/*
                 * Subtract one to leave an empty queue entry for 'Full Queue'
                 * condition. See NVM-Express 1.2 specification, section 4.1.2.
                 */
		dev->admin_tagset.queue_depth = NVME_AQ_BLKMQ_DEPTH - 1;
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
		dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
		dev->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;

		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
		if (IS_ERR(dev->ctrl.admin_q)) {
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
		if (!blk_get_queue(dev->ctrl.admin_q)) {
			nvme_dev_remove_admin(dev);
			dev->ctrl.admin_q = NULL;
			return -ENODEV;
		}
	} else
		blk_mq_unquiesce_queue(dev->ctrl.admin_q);

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Matias Björling 88 45.13% 1 3.85%
Keith Busch 66 33.85% 12 46.15%
Matthew Wilcox 14 7.18% 5 19.23%
Christoph Hellwig 13 6.67% 2 7.69%
Jens Axboe 10 5.13% 3 11.54%
Jon Derrick 2 1.03% 1 3.85%
Sagi Grimberg 1 0.51% 1 3.85%
Ming Lei 1 0.51% 1 3.85%
Total 195 100.00% 26 100.00%



static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
	return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
}

Contributors
Person Tokens Prop Commits CommitProp
Xu Yu 24 75.00% 1 33.33%
Keith Busch 6 18.75% 1 33.33%
Matias Björling 2 6.25% 1 33.33%
Total 32 100.00% 3 100.00%



static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
{
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (size <= dev->bar_mapped_size)
		return 0;
	if (size > pci_resource_len(pdev, 0))
		return -ENOMEM;
	if (dev->bar)
		iounmap(dev->bar);
	dev->bar = ioremap(pci_resource_start(pdev, 0), size);
	if (!dev->bar) {
		dev->bar_mapped_size = 0;
		return -ENOMEM;
	}
	dev->bar_mapped_size = size;
	dev->dbs = dev->bar + NVME_REG_DBS;

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Xu Yu 120 100.00% 1 100.00%
Total 120 100.00% 1 100.00%



static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
{
	int result;
	u32 aqa;
	struct nvme_queue *nvmeq;

	result = nvme_remap_bar(dev, db_bar_size(dev, 0));
	if (result < 0)
		return result;

	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
				NVME_CAP_NSSRC(dev->ctrl.cap) : 0;

	if (dev->subsystem &&
	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);

	result = nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
	if (result < 0)
		return result;

	nvmeq = dev->queues[0];
	if (!nvmeq) {
		nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH,
					dev_to_node(dev->dev));
		if (!nvmeq)
			return -ENOMEM;
	}

	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

	writel(aqa, dev->bar + NVME_REG_AQA);
	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);

	result = nvme_enable_ctrl(&dev->ctrl, dev->ctrl.cap);
	if (result)
		return result;

	nvmeq->cq_vector = 0;
	result = queue_request_irq(nvmeq);
	if (result) {
		nvmeq->cq_vector = -1;
		return result;
	}

	return result;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 107 37.94% 9 32.14%
Matthew Wilcox 85 30.14% 8 28.57%
Xu Yu 23 8.16% 1 3.57%
Christoph Hellwig 15 5.32% 2 7.14%
Matias Björling 13 4.61% 1 3.57%
Sagi Grimberg 13 4.61% 2 7.14%
Jon Derrick 8 2.84% 1 3.57%
Jens Axboe 7 2.48% 1 3.57%
Shaohua Li 7 2.48% 1 3.57%
Stephan Günther 2 0.71% 1 3.57%
Gabriel Krisman Bertazi 2 0.71% 1 3.57%
Total 282 100.00% 28 100.00%



static int nvme_create_io_queues(struct nvme_dev *dev)
{
	unsigned i, max;
	int ret = 0;

	for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
		/* vector == qid - 1, match nvme_create_queue */
		if (!nvme_alloc_queue(dev, i, dev->q_depth,
		     pci_irq_get_node(to_pci_dev(dev->dev), i - 1))) {
			ret = -ENOMEM;
			break;
		}
	}

	max = min(dev->max_qid, dev->ctrl.queue_count - 1);
	for (i = dev->online_queues; i <= max; i++) {
		ret = nvme_create_queue(dev->queues[i], i);
		if (ret)
			break;
	}

	/*
         * Ignore failing Create SQ/CQ commands, we can continue with less
         * than the desired aount of queues, and even a controller without
         * I/O queues an still be used to issue admin commands.  This might
         * be useful to upgrade a buggy firmware for example.
         */
	return ret >= 0 ? 0 : ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 65 45.77% 2 12.50%
Keith Busch 30 21.13% 6 37.50%
Guilherme G. Piccoli 16 11.27% 1 6.25%
Shaohua Li 15 10.56% 1 6.25%
Matthew Wilcox 11 7.75% 4 25.00%
Sagi Grimberg 4 2.82% 1 6.25%
Matias Björling 1 0.70% 1 6.25%
Total 142 100.00% 16 100.00%



static ssize_t nvme_cmb_show(struct device *dev,
			     struct device_attribute *attr,
			     char *buf)
{
	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));

	return scnprintf(buf, PAGE_SIZE, "cmbloc : x%08x\ncmbsz  : x%08x\n",
		       ndev->cmbloc, ndev->cmbsz);
}
Contributors
Person Tokens Prop Commits CommitProp
Stephen Bates 51 100.00% 2 100.00%
Total 51 100.00% 2 100.00%

static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL);



static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
{
	u64 szu, size, offset;
	resource_size_t bar_size;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
	void __iomem *cmb;
	dma_addr_t dma_addr;

	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
	if (!(NVME_CMB_SZ(dev->cmbsz)))
		return NULL;
	dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);

	if (!use_cmb_sqes)
		return NULL;

	szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
	size = szu * NVME_CMB_SZ(dev->cmbsz);
	offset = szu * NVME_CMB_OFST(dev->cmbloc);
	bar_size = pci_resource_len(pdev, NVME_CMB_BIR(dev->cmbloc));

	if (offset > bar_size)
		return NULL;

	/*
         * Controllers may support a CMB size larger than their BAR,
         * for example, due to being behind a bridge. Reduce the CMB to
         * the reported size of the BAR
         */
	if (size > bar_size - offset)
		size = bar_size - offset;

	dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(dev->cmbloc)) + offset;
	cmb = ioremap_wc(dma_addr, size);
	if (!cmb)
		return NULL;

	dev->cmb_dma_addr = dma_addr;
	dev->cmb_size = size;
	return cmb;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 112 50.91% 2 10.53%
Keith Busch 38 17.27% 6 31.58%
Matias Björling 35 15.91% 1 5.26%
Stephen Bates 16 7.27% 1 5.26%
Matthew Wilcox 7 3.18% 3 15.79%
Ramachandra Rao Gajula 4 1.82% 1 5.26%
Sathyavathi M 3 1.36% 1 5.26%
Jon Derrick 2 0.91% 2 10.53%
Linus Torvalds 2 0.91% 1 5.26%
Jens Axboe 1 0.45% 1 5.26%
Total 220 100.00% 19 100.00%



static inline void nvme_release_cmb(struct nvme_dev *dev)
{
	if (dev->cmb) {
		iounmap(dev->cmb);
		dev->cmb = NULL;
		sysfs_remove_file_from_group(&dev->ctrl.device->kobj,
					     &dev_attr_cmb.attr, NULL);
		dev->cmbsz = 0;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Jon Derrick 26 44.83% 1 16.67%
Christoph Hellwig 19 32.76% 1 16.67%
Matias Björling 6 10.34% 1 16.67%
Matthew Wilcox 4 6.90% 1 16.67%
Keith Busch 3 5.17% 2 33.33%
Total 58 100.00% 6 100.00%



static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
{
	u64 dma_addr = dev->host_mem_descs_dma;
	struct nvme_command c;
	int ret;

	memset(&c, 0, sizeof(c));
	c.features.opcode	= nvme_admin_set_features;
	c.features.fid		= cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
	c.features.dword11	= cpu_to_le32(bits);
	c.features.dword12	= cpu_to_le32(dev->host_mem_size >>
					      ilog2(dev->ctrl.page_size));
	c.features.dword13	= cpu_to_le32(lower_32_bits(dma_addr));
	c.features.dword14	= cpu_to_le32(upper_32_bits(dma_addr));
	c.features.dword15	= cpu_to_le32(dev->nr_host_mem_descs);

	ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
	if (ret) {
		dev_warn(dev->ctrl.device,
			 "failed to set host mem (err %d, flags %#x).\n",
			 ret, bits);
	}
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 147 83.52% 3 33.33%
Matthew Wilcox 13 7.39% 2 22.22%
Keith Busch 12 6.82% 3 33.33%
Jens Axboe 4 2.27% 1 11.11%
Total 176 100.00% 9 100.00%



static void nvme_free_host_mem(struct nvme_dev *dev)
{
	int i;

	for (i = 0; i < dev->nr_host_mem_descs; i++) {
		struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
		size_t size = le32_to_cpu(desc->size) * dev->ctrl.page_size;

		dma_free_coherent(dev->dev, size, dev->host_mem_desc_bufs[i],
				le64_to_cpu(desc->addr));
	}

	kfree(dev->host_mem_desc_bufs);
	dev->host_mem_desc_bufs = NULL;
	dma_free_coherent(dev->dev,
			dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
			dev->host_mem_descs, dev->host_mem_descs_dma);
	dev->host_mem_descs = NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 129 100.00% 2 100.00%
Total 129 100.00% 2 100.00%



static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
{
	struct nvme_host_mem_buf_desc *descs;
	u32 chunk_size, max_entries, len;
	dma_addr_t descs_dma;
	int i = 0;
	void **bufs;
	u64 size = 0, tmp;

	/* start big and work our way down */
	chunk_size = min(preferred, (u64)PAGE_SIZE << MAX_ORDER);
retry:
	tmp = (preferred + chunk_size - 1);
	do_div(tmp, chunk_size);
	max_entries = tmp;
	descs = dma_zalloc_coherent(dev->dev, max_entries * sizeof(*descs),
			&descs_dma, GFP_KERNEL);
	if (!descs)
		goto out;

	bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
	if (!bufs)
		goto out_free_descs;

	for (size = 0; size < preferred; size += len) {
		dma_addr_t dma_addr;

		len = min_t(u64, chunk_size, preferred - size);
		bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
				DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
		if (!bufs[i])
			break;

		descs[i].addr = cpu_to_le64(dma_addr);
		descs[i].size = cpu_to_le32(len / dev->ctrl.page_size);
		i++;
	}

	if (!size || (min && size < min)) {
		dev_warn(dev->ctrl.device,
			"failed to allocate host memory buffer.\n");
		goto out_free_bufs;
	}

	dev_info(dev->ctrl.device,
		"allocated %lld MiB host memory buffer.\n",
		size >> ilog2(SZ_1M));
	dev->nr_host_mem_descs = i;
	dev->host_mem_size = size;
	dev->host_mem_descs = descs;
	dev->host_mem_descs_dma = descs_dma;
	dev->host_mem_desc_bufs = bufs;
	return 0;

out_free_bufs:
	while (--i >= 0) {
		size_t size = le32_to_cpu(descs[i].size) * dev->ctrl.page_size;

		dma_free_coherent(dev->dev, size, bufs[i],
				le64_to_cpu(descs[i].addr));
	}

	kfree(bufs);
out_free_descs:
	dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
			descs_dma);
out:
	/* try a smaller chunk size if we failed early */
	if (chunk_size >= PAGE_SIZE * 2 && (i == 0 || size < min)) {
		chunk_size /= 2;
		goto retry;
	}
	dev->host_mem_descs = NULL;
	return -ENOMEM;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 426 97.48% 4 57.14%
Matias Björling 6 1.37% 1 14.29%
Dan Carpenter 4 0.92% 1 14.29%
Jon Derrick 1 0.23% 1 14.29%
Total 437 100.00% 7 100.00%



static void nvme_setup_host_mem(struct nvme_dev *dev)
{
	u64 max = (u64)max_host_mem_size_mb * SZ_1M;
	u64 preferred = (u64)dev->ctrl.hmpre * 4096;
	u64 min = (u64)dev->ctrl.hmmin * 4096;
	u32 enable_bits = NVME_HOST_MEM_ENABLE;

	preferred = min(preferred, max);
	if (min > max) {
		dev_warn(dev->ctrl.device,
			"min host memory (%lld MiB) above limit (%d MiB).\n",
			min >> ilog2(SZ_1M), max_host_mem_size_mb);
		nvme_free_host_mem(dev);
		return;
	}

	/*
         * If we already have a buffer allocated check if we can reuse it.
         */
	if (dev->host_mem_descs) {
		if (dev->host_mem_size >= min)
			enable_bits |= NVME_HOST_MEM_RETURN;
		else
			nvme_free_host_mem(dev);
	}

	if (!dev->host_mem_descs) {
		if (nvme_alloc_host_mem(dev, min, preferred))
			return;
	}

	if (nvme_set_host_mem(dev, enable_bits))
		nvme_free_host_mem(dev);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 134 84.28% 2 25.00%
Jon Derrick 17 10.69% 1 12.50%
Keith Busch 5 3.14% 3 37.50%
Matthew Wilcox 2 1.26% 1 12.50%
Matias Björling 1 0.63% 1 12.50%
Total 159 100.00% 8 100.00%



static int nvme_setup_io_queues(struct nvme_dev *dev)
{
	struct nvme_queue *adminq = dev->queues[0];
	struct pci_dev *pdev = to_pci_dev(dev->dev);
	int result, nr_io_queues;
	unsigned long size;

	nr_io_queues = num_present_cpus();
	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
	if (result < 0)
		return result;

	if (nr_io_queues == 0)
		return 0;

	if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
		result = nvme_cmb_qdepth(dev, nr_io_queues,
				sizeof(struct nvme_command));
		if (result > 0)
			dev->q_depth = result;
		else
			nvme_release_cmb(dev);
	}

	do {
		size = db_bar_size(dev, nr_io_queues);
		result = nvme_remap_bar(dev, size);
		if (!result)
			break;
		if (!--nr_io_queues)
			return -ENOMEM;
	} while (1);
	adminq->q_db = dev->dbs;

	/* Deregister the admin queue's interrupt */
	pci_free_irq(pdev, 0, adminq);

	/*
         * If we enable msix early due to not intx, disable it again before
         * setting up the full range we need.
         */
	pci_free_irq_vectors(pdev);
	nr_io_queues = pci_alloc_irq_vectors(pdev, 1, nr_io_queues,
			PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY);
	if (nr_io_queues <= 0)
		return -EIO;
	dev->max_qid = nr_io_queues;

	/*
         * Should investigate if there's a performance win from allocating
         * more queues than interrupt vectors; it might allow the submission
         * path to scale better, even if the receive path is limited by the
         * number of interrupts.
         */

	result = queue_request_irq(adminq);
	if (result) {
		adminq->cq_vector = -1;
		return result;
	}
	return nvme_create_io_queues(dev);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 144 56.47% 8 33.33%
Keith Busch 54 21.18% 7 29.17%
Matthew Wilcox 29 11.37% 5 20.83%
Jens Axboe 14 5.49% 2 8.33%
Xu Yu 10 3.92% 1 4.17%
Jon Derrick 4 1.57% 1 4.17%
Total 255 100.00% 24 100.00%



static void nvme_del_queue_end(struct request *req, blk_status_t error)
{
	struct nvme_queue *nvmeq = req->end_io_data;

	blk_mq_free_request(req);
	complete(&nvmeq->dev->ioq_wait);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 35 92.11% 3 60.00%
Matias Björling 2 5.26% 1 20.00%
Christoph Hellwig 1 2.63% 1 20.00%
Total 38 100.00% 5 100.00%



static void nvme_del_cq_end(struct request *req, blk_status_t error)
{
	struct nvme_queue *nvmeq = req->end_io_data;

	if (!error) {
		unsigned long flags;

		/*
                 * We might be called with the AQ q_lock held
                 * and the I/O queue q_lock should always
                 * nest inside the AQ one.
                 */
		spin_lock_irqsave_nested(&nvmeq->q_lock, flags,
					SINGLE_DEPTH_NESTING);
		nvme_process_cq(nvmeq);
		spin_unlock_irqrestore(&nvmeq->q_lock, flags);
	}

	nvme_del_queue_end(req, error);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 60 86.96% 5 50.00%
Ming Lin 4 5.80% 1 10.00%
Matias Björling 2 2.90% 1 10.00%
Christoph Hellwig 2 2.90% 2 20.00%
Matthew Wilcox 1 1.45% 1 10.00%
Total 69 100.00% 10 100.00%



static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
{
	struct request_queue *q = nvmeq->dev->ctrl.admin_q;
	struct request *req;
	struct nvme_command cmd;

	memset(&cmd, 0, sizeof(cmd));
	cmd.delete_queue.opcode = opcode;
	cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);

	req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = ADMIN_TIMEOUT;
	req->end_io_data = nvmeq;

	blk_execute_rq_nowait(q, NULL, req, false,
			opcode == nvme_admin_delete_cq ?
				nvme_del_cq_end : nvme_del_queue_end);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 122 93.13% 4 50.00%
Matthew Wilcox 6 4.58% 2 25.00%
Christoph Hellwig 3 2.29% 2 25.00%
Total 131 100.00% 8 100.00%



static void nvme_disable_io_queues(struct nvme_dev *dev, int queues)
{
	int pass;
	unsigned long timeout;
	u8 opcode = nvme_admin_delete_sq;

	for (pass = 0; pass < 2; pass++) {
		int sent = 0, i = queues;

		reinit_completion(&dev->ioq_wait);
 retry:
		timeout = ADMIN_TIMEOUT;
		for (; i > 0; i--, sent++)
			if (nvme_delete_queue(dev->queues[i], opcode))
				break;

		while (sent--) {
			timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout);
			if (timeout == 0)
				return;
			if (i)
				goto retry;
		}
		opcode = nvme_admin_delete_cq;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 87 67.44% 6 50.00%
Jens Axboe 16 12.40% 1 8.33%
Matthew Wilcox 15 11.63% 3 25.00%
Gabriel Krisman Bertazi 7 5.43% 1 8.33%
Matias Björling 4 3.10% 1 8.33%
Total 129 100.00% 12 100.00%

/*
 * Return: error value if an error occurred setting up the queues or calling
 * Identify Device.  0 if these succeeded, even if adding some of the
 * namespaces failed.  At the moment, these failures are silent.  TBD which
 * failures should be reported.
 */


static int nvme_dev_add(struct nvme_dev *dev)
{
	if (!dev->ctrl.tagset) {
		dev->tagset.ops = &nvme_mq_ops;
		dev->tagset.nr_hw_queues = dev->online_queues - 1;
		dev->tagset.timeout = NVME_IO_TIMEOUT;
		dev->tagset.numa_node = dev_to_node(dev->dev);
		dev->tagset.queue_depth =
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
		dev->tagset.cmd_size = nvme_cmd_size(dev);
		dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
		dev->tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->tagset))
			return 0;
		dev->ctrl.tagset = &dev->tagset;

		nvme_dbbuf_set(dev);
	} else {
		blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);

		/* Free previously allocated queues that are no longer usable */
		nvme_free_queues(dev, dev->online_queues);
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 136 80.47% 6 50.00%
Matthew Wilcox 13 7.69% 3 25.00%
Dan McLeran 10 5.92% 1 8.33%
Jon Derrick 5 2.96% 1 8.33%
Helen Mae Koike Fornazier 5 2.96% 1 8.33%
Total 169 100.00% 12 100.00%



static int nvme_pci_enable(struct nvme_dev *dev)
{
	int result = -ENOMEM;
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (pci_enable_device_mem(pdev))
		return result;

	pci_set_master(pdev);

	if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
		goto disable;

	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
		result = -ENODEV;
		goto disable;
	}

	/*
         * Some devices and/or platforms don't advertise or work with INTx
         * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
         * adjust this later.
         */
	result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
	if (result < 0)
		return result;

	dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);

	dev->q_depth = min_t(int, NVME_CAP_MQES(dev->ctrl.cap) + 1,
				io_queue_depth);
	dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
	dev->dbs = dev->bar + 4096;

	/*
         * Temporary fix for the Apple controller found in the MacBook8,1 and
         * some MacBook7,1 to avoid controller resets and data loss.
         */
	if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
		dev->q_depth = 2;
		dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
			"set queue depth=%u to work around controller resets\n",
			dev->q_depth);
	} else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
		   (pdev->device == 0xa821 || pdev->device == 0xa822) &&
		   NVME_CAP_MQES(dev->ctrl.cap) == 0) {
		dev->q_depth = 64;
		dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
                        "set queue depth=%u\n", dev->q_depth);
	}

	/*
         * CMBs can currently only exist on >=1.2 PCIe devices. We only
         * populate sysfs if a CMB is implemented. Since nvme_dev_attrs_group
         * has no name we can pass NULL as final argument to
         * sysfs_add_file_to_group.
         */

	if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2, 0)) {
		dev->cmb = nvme_map_cmb(dev);
		if (dev->cmb) {
			if (sysfs_add_file_to_group(&dev->ctrl.device->kobj,
						    &dev_attr_cmb.attr, NULL))
				dev_warn(dev->ctrl.device,
					 "failed to add sysfs attribute for CMB\n");
		}
	}

	pci_enable_pcie_error_reporting(pdev);
	pci_save_state(pdev);
	return 0;

 disable:
	pci_disable_device(pdev);
	return result;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 191 51.07% 10 41.67%
Martin K. Petersen 54 14.44% 1 4.17%
Stephen Bates 38 10.16% 1 4.17%
Christoph Hellwig 31 8.29% 4 16.67%
Linus Torvalds 23 6.15% 1 4.17%
Sagi Grimberg 16 4.28% 1 4.17%
Matthew Wilcox 10 2.67% 2 8.33%
Matias Björling 6 1.60% 1 4.17%
Max Gurtovoy 2 0.53% 1 4.17%
Gabriel Krisman Bertazi 2 0.53% 1 4.17%
weiping zhang 1 0.27% 1 4.17%
Total 374 100.00% 24 100.00%



static void nvme_dev_unmap(struct nvme_dev *dev)
{
	if (dev->bar)
		iounmap(dev->bar);
	pci_release_mem_regions(to_pci_dev(dev->dev));
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 30 88.24% 6 66.67%
Johannes Thumshirn 2 5.88% 2 22.22%
Matthew Wilcox 2 5.88% 1 11.11%
Total 34 100.00% 9 100.00%



static void nvme_pci_disable(struct nvme_dev *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	nvme_release_cmb(dev);
	pci_free_irq_vectors(pdev);

	if (pci_is_enabled(pdev)) {
		pci_disable_pcie_error_reporting(pdev);
		pci_disable_device(pdev);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 30 57.69% 6 46.15%
Matthew Wilcox 12 23.08% 3 23.08%
Jon Derrick 5 9.62% 1 7.69%
Christoph Hellwig 4 7.69% 2 15.38%
Quoc-Son Anh 1 1.92% 1 7.69%
Total 52 100.00% 13 100.00%



static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
{
	int i, queues;
	bool dead = true;
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	mutex_lock(&dev->shutdown_lock);
	if (pci_is_enabled(pdev)) {
		u32 csts = readl(dev->bar + NVME_REG_CSTS);

		if (dev->ctrl.state == NVME_CTRL_LIVE ||
		    dev->ctrl.state == NVME_CTRL_RESETTING)
			nvme_start_freeze(&dev->ctrl);
		dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
			pdev->error_state  != pci_channel_io_normal);
	}

	/*
         * Give the controller a chance to complete all entered requests if
         * doing a safe shutdown.
         */
	if (!dead) {
		if (shutdown)
			nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);

		/*
                 * If the controller is still alive tell it to stop using the
                 * host memory buffer.  In theory the shutdown / reset should
                 * make sure that it doesn't access the host memoery anymore,
                 * but I'd rather be safe than sorry..
                 */
		if (dev->host_mem_descs)
			nvme_set_host_mem(dev, 0);

	}
	nvme_stop_queues(&dev->ctrl);

	queues = dev->online_queues - 1;
	for (i = dev->ctrl.queue_count - 1; i > 0; i--)
		nvme_suspend_queue(dev->queues[i]);

	if (dead) {
		/* A device might become IO incapable very soon during
                 * probe, before the admin queue is configured. Thus,
                 * queue_count can be 0 here.
                 */
		if (dev->ctrl.queue_count)
			nvme_suspend_queue(dev->queues[0]);
	} else {
		nvme_disable_io_queues(dev, queues);
		nvme_disable_admin_queue(dev, shutdown);
	}
	nvme_pci_disable(dev);

	blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl);
	blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl);

	/*
         * The driver will not be starting up queues again if shutting down so
         * must flush all entered requests to their failed completion to avoid
         * deadlocking blk-mq hot-cpu notifier.
         */
	if (shutdown)
		nvme_start_queues(&dev->ctrl);
	mutex_unlock(&dev->shutdown_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 201 68.14% 18 60.00%
Gabriel Krisman Bertazi 22 7.46% 2 6.67%
Sagi Grimberg 21 7.12% 3 10.00%
Christoph Hellwig 19 6.44% 1 3.33%
Matthew Wilcox 14 4.75% 3 10.00%
Lin Ming 8 2.71% 1 3.33%
Matias Björling 6 2.03% 1 3.33%
Indraneel Mukherjee 4 1.36% 1 3.33%
Total 295 100.00% 30 100.00%



static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
	dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

	/* Optimisation for I/Os between 4k and 128k */
	dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 69 82.14% 4 44.44%
Matthew Wilcox 9 10.71% 4 44.44%
Christoph Hellwig 6 7.14% 1 11.11%
Total 84 100.00% 9 100.00%



static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
	dma_pool_destroy(dev->prp_small_pool);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 19 76.00% 2 50.00%
Matthew Wilcox 6 24.00% 2 50.00%
Total 25 100.00% 4 100.00%



static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
{
	struct nvme_dev *dev = to_nvme_dev(ctrl);

	nvme_dbbuf_dma_free(dev);
	put_device(dev->dev);
	if (dev->tagset.tags)
		blk_mq_free_tag_set(&dev->tagset);
	if (dev->ctrl.admin_q)
		blk_put_queue(dev->ctrl.admin_q);
	kfree(dev->queues);
	free_opal_dev(dev->ctrl.opal_dev);
	kfree(dev);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 35 40.23% 7 41.18%
Christoph Hellwig 20 22.99% 4 23.53%
Matias Björling 14 16.09% 1 5.88%
Matthew Wilcox 11 12.64% 2 11.76%
Helen Mae Koike Fornazier 5 5.75% 1 5.88%
Jon Derrick 1 1.15% 1 5.88%
Scott Bauer 1 1.15% 1 5.88%
Total 87 100.00% 17 100.00%



static void nvme_remove_dead_ctrl(struct nvme_dev *dev, int status)
{
	dev_warn(dev->ctrl.device, "Removing after probe failure status: %d\n", status);

	kref_get(&dev->ctrl.kref);
	nvme_dev_disable(dev, false);
	if (!schedule_work(&dev->remove_work))
		nvme_put_ctrl(&dev->ctrl);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 60 95.24% 2 66.67%
Linus Torvalds 3 4.76% 1 33.33%
Total 63 100.00% 3 100.00%



static void nvme_reset_work(struct work_struct *work)
{
	struct nvme_dev *dev =
		container_of(work, struct nvme_dev, ctrl.reset_work);
	bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
	int result = -ENODEV;

	if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING))
		goto out;

	/*
         * If we're called to reset a live controller first shut it down before
         * moving on.
         */
	if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
		nvme_dev_disable(dev, false);

	result = nvme_pci_enable(dev);
	if (result)
		goto out;

	result = nvme_pci_configure_admin_queue(dev);
	if (result)
		goto out;

	nvme_init_queue(dev->queues[0], 0);
	result = nvme_alloc_admin_tags(dev);
	if (result)
		goto out;

	result = nvme_init_identify(&dev->ctrl);
	if (result)
		goto out;

	if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) {
		if (!dev->ctrl.opal_dev)
			dev->ctrl.opal_dev =
				init_opal_dev(&dev->ctrl, &nvme_sec_submit);
		else if (was_suspend)
			opal_unlock_from_suspend(dev->ctrl.opal_dev);
	} else {
		free_opal_dev(dev->ctrl.opal_dev);
		dev->ctrl.opal_dev = NULL;
	}

	if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
		result = nvme_dbbuf_dma_alloc(dev);
		if (result)
			dev_warn(dev->dev,
				 "unable to allocate dma for dbbuf\n");
	}

	if (dev->ctrl.hmpre)
		nvme_setup_host_mem(dev);

	result = nvme_setup_io_queues(dev);
	if (result)
		goto out;

	/*
         * Keep the controller around but remove all namespaces if we don't have
         * any working I/O queue.
         */
	if (dev->online_queues < 2) {
		dev_warn(dev->ctrl.device, "IO queues not created\n");
		nvme_kill_queues(&dev->ctrl);
		nvme_remove_namespaces(&dev->ctrl);
	} else {
		nvme_start_queues(&dev->ctrl);
		nvme_wait_freeze(&dev->ctrl);
		nvme_dev_add(dev);
		nvme_unfreeze(&dev->ctrl);
	}

	if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
		dev_warn(dev->ctrl.device, "failed to mark controller live\n");
		goto out;
	}

	nvme_start_ctrl(&dev->ctrl);
	return;

 out:
	nvme_remove_dead_ctrl(dev, result);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 165 41.04% 13 37.14%
Keith Busch 89 22.14% 9 25.71%
Scott Bauer 72 17.91% 3 8.57%
Helen Mae Koike Fornazier 32 7.96% 1 2.86%
Dan McLeran 20 4.98% 1 2.86%
Sagi Grimberg 8 1.99% 4 11.43%
Matthew Wilcox 7 1.74% 1 2.86%
Matias Björling 4 1.00% 1 2.86%
Quoc-Son Anh 4 1.00% 1 2.86%
Rakesh Pandit 1 0.25% 1 2.86%
Total 402 100.00% 35 100.00%



static void nvme_remove_dead_ctrl_work(struct work_struct *work)
{
	struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	nvme_kill_queues(&dev->ctrl);
	if (pci_get_drvdata(pdev))
		device_release_driver(&pdev->dev);
	nvme_put_ctrl(&dev->ctrl);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 51 73.91% 4 57.14%
Christoph Hellwig 18 26.09% 3 42.86%
Total 69 100.00% 7 100.00%



static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
{
	*val = readl(to_nvme_dev(ctrl)->bar + off);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 22 61.11% 1 50.00%
Keith Busch 14 38.89% 1 50.00%
Total 36 100.00% 2 100.00%



static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
{
	writel(val, to_nvme_dev(ctrl)->bar + off);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 26 76.47% 1 33.33%
Keith Busch 8 23.53% 2 66.67%
Total 34 100.00% 3 100.00%



static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
{
	*val = readq(to_nvme_dev(ctrl)->bar + off);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 29 80.56% 1 50.00%
Keith Busch 7 19.44% 1 50.00%
Total 36 100.00% 2 100.00%


static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
	.name			= "pcie",
	.module			= THIS_MODULE,
	.flags			= NVME_F_METADATA_SUPPORTED,
	.reg_read32		= nvme_pci_reg_read32,
	.reg_write32		= nvme_pci_reg_write32,
	.reg_read64		= nvme_pci_reg_read64,
	.free_ctrl		= nvme_pci_free_ctrl,
	.submit_async_event	= nvme_pci_submit_async_event,
};



static int nvme_dev_map(struct nvme_dev *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (pci_request_mem_regions(pdev, "nvme"))
		return -ENODEV;

	if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
		goto release;

	return 0;
  release:
	pci_release_mem_regions(pdev);
	return -ENODEV;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 55 85.94% 1 33.33%
Xu Yu 7 10.94% 1 33.33%
Johannes Thumshirn 2 3.12% 1 33.33%
Total 64 100.00% 3 100.00%



static unsigned long check_dell_samsung_bug(struct pci_dev *pdev)
{
	if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
		/*
                 * Several Samsung devices seem to drop off the PCIe bus
                 * randomly when APST is on and uses the deepest sleep state.
                 * This has been observed on a Samsung "SM951 NVMe SAMSUNG
                 * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
                 * 950 PRO 256GB", but it seems to be restricted to two Dell
                 * laptops.
                 */
		if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
		    (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
		     dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
			return NVME_QUIRK_NO_DEEPEST_PS;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 60 100.00% 1 100.00%
Total 60 100.00% 1 100.00%



static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	int node, result = -ENOMEM;
	struct nvme_dev *dev;
	unsigned long quirks = id->driver_data;

	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
		set_dev_node(&pdev->dev, first_memory_node);

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
	if (!dev)
		return -ENOMEM;
	dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
							GFP_KERNEL, node);
	if (!dev->queues)
		goto free;

	dev->dev = get_device(&pdev->dev);
	pci_set_drvdata(pdev, dev);

	result = nvme_dev_map(dev);
	if (result)
		goto put_pci;

	INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
	INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
	mutex_init(&dev->shutdown_lock);
	init_completion(&dev->ioq_wait);

	result = nvme_setup_prp_pools(dev);
	if (result)
		goto unmap;

	quirks |= check_dell_samsung_bug(pdev);

	result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
			quirks);
	if (result)
		goto release_pools;

	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING);
	dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));

	queue_work(nvme_wq, &dev->ctrl.reset_work);
	return 0;

 release_pools:
	nvme_release_prp_pools(dev);
 unmap:
	nvme_dev_unmap(dev);
 put_pci:
	put_device(dev->dev);
 free:
	kfree(dev->queues);
	kfree(dev);
	return result;
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 107 32.92% 4 14.81%
Keith Busch 70 21.54% 10 37.04%
Christoph Hellwig 55 16.92% 6 22.22%
Matias Björling 37 11.38% 1 3.70%
Sagi Grimberg 20 6.15% 2 7.41%
Andrew Lutomirski 16 4.92% 1 3.70%
Rakesh Pandit 10 3.08% 1 3.70%
Christophe Jaillet 9 2.77% 1 3.70%
Masayoshi Mizuma 1 0.31% 1 3.70%
Total 325 100.00% 27 100.00%



static void nvme_reset_prepare(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
	nvme_dev_disable(dev, false);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 26 92.86% 2 66.67%
Christoph Hellwig 2 7.14% 1 33.33%
Total 28 100.00% 3 100.00%



static void nvme_reset_done(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
	nvme_reset_ctrl(&dev->ctrl);
}

Contributors
Person Tokens Prop Commits CommitProp
Christoph Hellwig 18 62.07% 2 50.00%
Linus Torvalds 6 20.69% 1 25.00%
Keith Busch 5 17.24% 1 25.00%
Total 29 100.00% 4 100.00%



static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
	nvme_dev_disable(dev, true);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 28 100.00% 2 100.00%
Total 28 100.00% 2 100.00%

/*
 * The driver's remove may be called on a device in a partially initialized
 * state. This function must not have any dependencies on the device state in
 * order to proceed.
 */


static void nvme_remove(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);

	cancel_work_sync(&dev->ctrl.reset_work);
	pci_set_drvdata(pdev, NULL);

	if (!pci_device_is_present(pdev)) {
		nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
		nvme_dev_disable(dev, false);
	}

	flush_work(&dev->ctrl.reset_work);
	nvme_stop_ctrl(&dev->ctrl);
	nvme_remove_namespaces(&dev->ctrl);
	nvme_dev_disable(dev, true);
	nvme_free_host_mem(dev);
	nvme_dev_remove_admin(dev);
	nvme_free_queues(dev, 0);
	nvme_uninit_ctrl(&dev->ctrl);
	nvme_release_prp_pools(dev);
	nvme_dev_unmap(dev);
	nvme_put_ctrl(&dev->ctrl);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 82 54.30% 13 65.00%
Matthew Wilcox 29 19.21% 1 5.00%
Sagi Grimberg 17 11.26% 1 5.00%
Christoph Hellwig 15 9.93% 4 20.00%
Rakesh Pandit 8 5.30% 1 5.00%
Total 151 100.00% 20 100.00%



static int nvme_pci_sriov_configure(struct pci_dev *pdev, int numvfs)
{
	int ret = 0;

	if (numvfs == 0) {
		if (pci_vfs_assigned(pdev)) {
			dev_warn(&pdev->dev,
				"Cannot disable SR-IOV VFs while assigned\n");
			return -EPERM;
		}
		pci_disable_sriov(pdev);
		return 0;
	}

	ret = pci_enable_sriov(pdev, numvfs);
	return ret ? ret : numvfs;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 74 100.00% 1 100.00%
Total 74 100.00% 1 100.00%

#ifdef CONFIG_PM_SLEEP


static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

	nvme_dev_disable(ndev, true);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 40 97.56% 2 66.67%
Matthew Wilcox 1 2.44% 1 33.33%
Total 41 100.00% 3 100.00%



static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

	nvme_reset_ctrl(&ndev->ctrl);
	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 37 88.10% 2 50.00%
Christoph Hellwig 4 9.52% 1 25.00%
Matthew Wilcox 1 2.38% 1 25.00%
Total 42 100.00% 4 100.00%

#endif

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);



static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
						pci_channel_state_t state)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

	/*
         * A frozen channel requires a reset. When detected, this method will
         * shutdown the controller to quiesce. The controller will be restarted
         * after the slot reset through driver's slot_reset callback.
         */
	switch (state) {
	case pci_channel_io_normal:
		return PCI_ERS_RESULT_CAN_RECOVER;
	case pci_channel_io_frozen:
		dev_warn(dev->ctrl.device,
			"frozen state error detected, reset controller\n");
		nvme_dev_disable(dev, false);
		return PCI_ERS_RESULT_NEED_RESET;
	case pci_channel_io_perm_failure:
		dev_warn(dev->ctrl.device,
			"failure state error detected, request disconnect\n");
		return PCI_ERS_RESULT_DISCONNECT;
	}
	return PCI_ERS_RESULT_NEED_RESET;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 81 100.00% 4 100.00%
Total 81 100.00% 4 100.00%



static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

	dev_info(dev->ctrl.device, "restart after slot reset\n");
	pci_restore_state(pdev);
	nvme_reset_ctrl(&dev->ctrl);
	return PCI_ERS_RESULT_RECOVERED;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 40 83.33% 2 50.00%
Sagi Grimberg 4 8.33% 1 25.00%
Christoph Hellwig 4 8.33% 1 25.00%
Total 48 100.00% 4 100.00%



static void nvme_error_resume(struct pci_dev *pdev)
{
	pci_cleanup_aer_uncorrect_error_status(pdev);
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 16 100.00% 2 100.00%
Total 16 100.00% 2 100.00%


static const struct pci_error_handlers nvme_err_handler = {
	.error_detected	= nvme_error_detected,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
	.reset_prepare	= nvme_reset_prepare,
	.reset_done	= nvme_reset_done,
};


static const struct pci_device_id nvme_id_table[] = {
	{ PCI_VDEVICE(INTEL, 0x0953),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
				NVME_QUIRK_DEALLOCATE_ZEROES, },
	{ PCI_VDEVICE(INTEL, 0x0a53),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
				NVME_QUIRK_DEALLOCATE_ZEROES, },
	{ PCI_VDEVICE(INTEL, 0x0a54),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
				NVME_QUIRK_DEALLOCATE_ZEROES, },
	{ PCI_VDEVICE(INTEL, 0x0a55),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
				NVME_QUIRK_DEALLOCATE_ZEROES, },
	{ PCI_VDEVICE(INTEL, 0xf1a5),	/* Intel 600P/P3100 */
		.driver_data = NVME_QUIRK_NO_DEEPEST_PS },
	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
	{ PCI_DEVICE(0x1c58, 0x0003),	/* HGST adapter */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
	{ PCI_DEVICE(0x1c5f, 0x0540),	/* Memblaze Pblaze4 adapter */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
	{ PCI_DEVICE(0x144d, 0xa821),   /* Samsung PM1725 */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
	{ PCI_DEVICE(0x144d, 0xa822),   /* Samsung PM1725a */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);


static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
	.remove		= nvme_remove,
	.shutdown	= nvme_shutdown,
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
        },
	.sriov_configure = nvme_pci_sriov_configure,
	.err_handler	= &nvme_err_handler,
};



static int __init nvme_init(void)
{
	return pci_register_driver(&nvme_driver);
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 15 93.75% 1 50.00%
Dan McLeran 1 6.25% 1 50.00%
Total 16 100.00% 2 100.00%



static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
	_nvme_check_size();
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 18 100.00% 2 100.00%
Total 18 100.00% 2 100.00%

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
MODULE_VERSION("1.0");

module_init(nvme_init);

module_exit(nvme_exit);
Overall Contributors
Person Tokens Prop Commits CommitProp
Keith Busch 4030 32.11% 112 33.73%
Christoph Hellwig 3059 24.38% 69 20.78%
Matthew Wilcox 1769 14.10% 53 15.96%
Matias Björling 832 6.63% 2 0.60%
Helen Mae Koike Fornazier 723 5.76% 1 0.30%
Jon Derrick 356 2.84% 7 2.11%
Sagi Grimberg 335 2.67% 14 4.22%
Jens Axboe 218 1.74% 7 2.11%
Xu Yu 187 1.49% 1 0.30%
Shane Michael Matthews 126 1.00% 1 0.30%
Stephen Bates 120 0.96% 2 0.60%
weiping zhang 119 0.95% 1 0.30%
Andrew Lutomirski 94 0.75% 2 0.60%
Martin K. Petersen 85 0.68% 2 0.60%
Scott Bauer 76 0.61% 3 0.90%
Marta Rybczynska 59 0.47% 1 0.30%
Linus Torvalds 35 0.28% 5 1.51%
Gabriel Krisman Bertazi 33 0.26% 3 0.90%
Dan McLeran 33 0.26% 2 0.60%
Guilherme G. Piccoli 31 0.25% 2 0.60%
Shaohua Li 28 0.22% 1 0.30%
Rakesh Pandit 19 0.15% 1 0.30%
David Wayne Fugate 16 0.13% 1 0.30%
Wenbo Wang 15 0.12% 1 0.30%
Vishal Verma 14 0.11% 2 0.60%
Ming Lin 13 0.10% 4 1.20%
Ramachandra Rao Gajula 12 0.10% 1 0.30%
Stephan Günther 11 0.09% 2 0.60%
Christophe Jaillet 9 0.07% 1 0.30%
Daniel Roschka 9 0.07% 1 0.30%
Dan J Williams 8 0.06% 2 0.60%
Lin Ming 8 0.06% 1 0.30%
Murali Iyer 7 0.06% 1 0.30%
Greg Kroah-Hartman 7 0.06% 1 0.30%
Dan Carpenter 6 0.05% 2 0.60%
Johannes Thumshirn 6 0.05% 3 0.90%
Quoc-Son Anh 5 0.04% 1 0.30%
Jingoo Han 5 0.04% 1 0.30%
Haiyan Hu 4 0.03% 1 0.30%
Indraneel Mukherjee 4 0.03% 1 0.30%
Sathyavathi M 3 0.02% 1 0.30%
Santosh Yaraganavi 3 0.02% 1 0.30%
Mauricio Faria de Oliveira 3 0.02% 1 0.30%
Nisheeth Bhat 2 0.02% 1 0.30%
Eric Biggers 2 0.02% 1 0.30%
Max Gurtovoy 2 0.02% 1 0.30%
Alexander Gordeev 2 0.02% 1 0.30%
Masayoshi Mizuma 1 0.01% 1 0.30%
Joe Perches 1 0.01% 1 0.30%
Sunad Bhandary 1 0.01% 1 0.30%
Stephen Hemminger 1 0.01% 1 0.30%
Ming Lei 1 0.01% 1 0.30%
kaoudis 1 0.01% 1 0.30%
Total 12549 100.00% 332 100.00%
Directory: drivers/nvme/host

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.
Person	Tokens	Prop	Commits	CommitProp
weiping zhang	58	100.00%	1	100.00%
Total	58	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.13 drivers/nvme/host/pci.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors
