| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| James Smart | 11473 | 97.31% | 41 | 64.06% |
| Amit Engel | 142 | 1.20% | 1 | 1.56% |
| Leonid Ravich | 67 | 0.57% | 1 | 1.56% |
| Christoph Hellwig | 47 | 0.40% | 9 | 14.06% |
| Sagi Grimberg | 18 | 0.15% | 2 | 3.12% |
| Gustavo A. R. Silva | 11 | 0.09% | 1 | 1.56% |
| Minwoo Im | 8 | 0.07% | 1 | 1.56% |
| Bart Van Assche | 7 | 0.06% | 1 | 1.56% |
| Hannes Reinecke | 6 | 0.05% | 1 | 1.56% |
| Christophe Jaillet | 6 | 0.05% | 1 | 1.56% |
| Chaitanya Kulkarni | 3 | 0.03% | 3 | 4.69% |
| Milan P. Gandhi | 1 | 0.01% | 1 | 1.56% |
| Max Gurtovoy | 1 | 0.01% | 1 | 1.56% |
| Total | 11790 | 64 |
// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2016 Avago Technologies. All rights reserved. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/module.h> #include <linux/slab.h> #include <linux/blk-mq.h> #include <linux/parser.h> #include <linux/random.h> #include <uapi/scsi/fc/fc_fs.h> #include <uapi/scsi/fc/fc_els.h> #include "nvmet.h" #include <linux/nvme-fc-driver.h> #include <linux/nvme-fc.h> #include "../host/fc.h" /* *************************** Data Structures/Defines ****************** */ #define NVMET_LS_CTX_COUNT 256 struct nvmet_fc_tgtport; struct nvmet_fc_tgt_assoc; struct nvmet_fc_ls_iod { /* for an LS RQST RCV */ struct nvmefc_ls_rsp *lsrsp; struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */ struct list_head ls_rcv_list; /* tgtport->ls_rcv_list */ struct nvmet_fc_tgtport *tgtport; struct nvmet_fc_tgt_assoc *assoc; void *hosthandle; union nvmefc_ls_requests *rqstbuf; union nvmefc_ls_responses *rspbuf; u16 rqstdatalen; dma_addr_t rspdma; struct scatterlist sg[2]; struct work_struct work; } __aligned(sizeof(unsigned long long)); struct nvmet_fc_ls_req_op { /* for an LS RQST XMT */ struct nvmefc_ls_req ls_req; struct nvmet_fc_tgtport *tgtport; void *hosthandle; int ls_error; struct list_head lsreq_list; /* tgtport->ls_req_list */ bool req_queued; }; /* desired maximum for a single sequence - if sg list allows it */ #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024) enum nvmet_fcp_datadir { NVMET_FCP_NODATA, NVMET_FCP_WRITE, NVMET_FCP_READ, NVMET_FCP_ABORTED, }; struct nvmet_fc_fcp_iod { struct nvmefc_tgt_fcp_req *fcpreq; struct nvme_fc_cmd_iu cmdiubuf; struct nvme_fc_ersp_iu rspiubuf; dma_addr_t rspdma; struct scatterlist *next_sg; struct scatterlist *data_sg; int data_sg_cnt; u32 offset; enum nvmet_fcp_datadir io_dir; bool active; bool abort; bool aborted; bool writedataactive; spinlock_t flock; struct nvmet_req req; struct work_struct defer_work; struct nvmet_fc_tgtport *tgtport; struct nvmet_fc_tgt_queue *queue; struct list_head fcp_list; /* tgtport->fcp_list */ }; struct nvmet_fc_tgtport { struct nvmet_fc_target_port fc_target_port; struct list_head tgt_list; /* nvmet_fc_target_list */ struct device *dev; /* dev for dma mapping */ struct nvmet_fc_target_template *ops; struct nvmet_fc_ls_iod *iod; spinlock_t lock; struct list_head ls_rcv_list; struct list_head ls_req_list; struct list_head ls_busylist; struct list_head assoc_list; struct list_head host_list; struct ida assoc_cnt; struct nvmet_fc_port_entry *pe; struct kref ref; u32 max_sg_cnt; }; struct nvmet_fc_port_entry { struct nvmet_fc_tgtport *tgtport; struct nvmet_port *port; u64 node_name; u64 port_name; struct list_head pe_list; }; struct nvmet_fc_defer_fcp_req { struct list_head req_list; struct nvmefc_tgt_fcp_req *fcp_req; }; struct nvmet_fc_tgt_queue { bool ninetypercent; u16 qid; u16 sqsize; u16 ersp_ratio; __le16 sqhd; atomic_t connected; atomic_t sqtail; atomic_t zrspcnt; atomic_t rsn; spinlock_t qlock; struct nvmet_cq nvme_cq; struct nvmet_sq nvme_sq; struct nvmet_fc_tgt_assoc *assoc; struct list_head fod_list; struct list_head pending_cmd_list; struct list_head avail_defer_list; struct workqueue_struct *work_q; struct kref ref; struct rcu_head rcu; struct nvmet_fc_fcp_iod fod[]; /* array of fcp_iods */ } __aligned(sizeof(unsigned long long)); struct nvmet_fc_hostport { struct nvmet_fc_tgtport *tgtport; void *hosthandle; struct list_head host_list; struct kref ref; u8 invalid; }; struct nvmet_fc_tgt_assoc { u64 association_id; u32 a_id; atomic_t terminating; struct nvmet_fc_tgtport *tgtport; struct nvmet_fc_hostport *hostport; struct nvmet_fc_ls_iod *rcv_disconn; struct list_head a_list; struct nvmet_fc_tgt_queue __rcu *queues[NVMET_NR_QUEUES + 1]; struct kref ref; struct work_struct del_work; struct rcu_head rcu; }; static inline int nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr) { return (iodptr - iodptr->tgtport->iod); } static inline int nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr) { return (fodptr - fodptr->queue->fod); } /* * Association and Connection IDs: * * Association ID will have random number in upper 6 bytes and zero * in lower 2 bytes * * Connection IDs will be Association ID with QID or'd in lower 2 bytes * * note: Association ID = Connection ID for queue 0 */ #define BYTES_FOR_QID sizeof(u16) #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8) #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1)) static inline u64 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid) { return (assoc->association_id | qid); } static inline u64 nvmet_fc_getassociationid(u64 connectionid) { return connectionid & ~NVMET_FC_QUEUEID_MASK; } static inline u16 nvmet_fc_getqueueid(u64 connectionid) { return (u16)(connectionid & NVMET_FC_QUEUEID_MASK); } static inline struct nvmet_fc_tgtport * targetport_to_tgtport(struct nvmet_fc_target_port *targetport) { return container_of(targetport, struct nvmet_fc_tgtport, fc_target_port); } static inline struct nvmet_fc_fcp_iod * nvmet_req_to_fod(struct nvmet_req *nvme_req) { return container_of(nvme_req, struct nvmet_fc_fcp_iod, req); } /* *************************** Globals **************************** */ static DEFINE_SPINLOCK(nvmet_fc_tgtlock); static LIST_HEAD(nvmet_fc_target_list); static DEFINE_IDA(nvmet_fc_tgtport_cnt); static LIST_HEAD(nvmet_fc_portentry_list); static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work); static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work); static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc); static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc); static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue); static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue); static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport); static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport); static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_fcp_iod *fod); static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc); static void nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod); /* *********************** FC-NVME DMA Handling **************************** */ /* * The fcloop device passes in a NULL device pointer. Real LLD's will * pass in a valid device pointer. If NULL is passed to the dma mapping * routines, depending on the platform, it may or may not succeed, and * may crash. * * As such: * Wrapper all the dma routines and check the dev pointer. * * If simple mappings (return just a dma address, we'll noop them, * returning a dma address of 0. * * On more complex mappings (dma_map_sg), a pseudo routine fills * in the scatter list, setting all dma addresses to 0. */ static inline dma_addr_t fc_dma_map_single(struct device *dev, void *ptr, size_t size, enum dma_data_direction dir) { return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L; } static inline int fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) { return dev ? dma_mapping_error(dev, dma_addr) : 0; } static inline void fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { if (dev) dma_unmap_single(dev, addr, size, dir); } static inline void fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { if (dev) dma_sync_single_for_cpu(dev, addr, size, dir); } static inline void fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { if (dev) dma_sync_single_for_device(dev, addr, size, dir); } /* pseudo dma_map_sg call */ static int fc_map_sg(struct scatterlist *sg, int nents) { struct scatterlist *s; int i; WARN_ON(nents == 0 || sg[0].length == 0); for_each_sg(sg, s, nents, i) { s->dma_address = 0L; #ifdef CONFIG_NEED_SG_DMA_LENGTH s->dma_length = s->length; #endif } return nents; } static inline int fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir) { return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents); } static inline void fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir) { if (dev) dma_unmap_sg(dev, sg, nents, dir); } /* ********************** FC-NVME LS XMT Handling ************************* */ static void __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop) { struct nvmet_fc_tgtport *tgtport = lsop->tgtport; struct nvmefc_ls_req *lsreq = &lsop->ls_req; unsigned long flags; spin_lock_irqsave(&tgtport->lock, flags); if (!lsop->req_queued) { spin_unlock_irqrestore(&tgtport->lock, flags); return; } list_del(&lsop->lsreq_list); lsop->req_queued = false; spin_unlock_irqrestore(&tgtport->lock, flags); fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma, (lsreq->rqstlen + lsreq->rsplen), DMA_BIDIRECTIONAL); nvmet_fc_tgtport_put(tgtport); } static int __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_req_op *lsop, void (*done)(struct nvmefc_ls_req *req, int status)) { struct nvmefc_ls_req *lsreq = &lsop->ls_req; unsigned long flags; int ret = 0; if (!tgtport->ops->ls_req) return -EOPNOTSUPP; if (!nvmet_fc_tgtport_get(tgtport)) return -ESHUTDOWN; lsreq->done = done; lsop->req_queued = false; INIT_LIST_HEAD(&lsop->lsreq_list); lsreq->rqstdma = fc_dma_map_single(tgtport->dev, lsreq->rqstaddr, lsreq->rqstlen + lsreq->rsplen, DMA_BIDIRECTIONAL); if (fc_dma_mapping_error(tgtport->dev, lsreq->rqstdma)) { ret = -EFAULT; goto out_puttgtport; } lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen; spin_lock_irqsave(&tgtport->lock, flags); list_add_tail(&lsop->lsreq_list, &tgtport->ls_req_list); lsop->req_queued = true; spin_unlock_irqrestore(&tgtport->lock, flags); ret = tgtport->ops->ls_req(&tgtport->fc_target_port, lsop->hosthandle, lsreq); if (ret) goto out_unlink; return 0; out_unlink: lsop->ls_error = ret; spin_lock_irqsave(&tgtport->lock, flags); lsop->req_queued = false; list_del(&lsop->lsreq_list); spin_unlock_irqrestore(&tgtport->lock, flags); fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma, (lsreq->rqstlen + lsreq->rsplen), DMA_BIDIRECTIONAL); out_puttgtport: nvmet_fc_tgtport_put(tgtport); return ret; } static int nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_req_op *lsop, void (*done)(struct nvmefc_ls_req *req, int status)) { /* don't wait for completion */ return __nvmet_fc_send_ls_req(tgtport, lsop, done); } static void nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) { struct nvmet_fc_ls_req_op *lsop = container_of(lsreq, struct nvmet_fc_ls_req_op, ls_req); __nvmet_fc_finish_ls_req(lsop); /* fc-nvme target doesn't care about success or failure of cmd */ kfree(lsop); } /* * This routine sends a FC-NVME LS to disconnect (aka terminate) * the FC-NVME Association. Terminating the association also * terminates the FC-NVME connections (per queue, both admin and io * queues) that are part of the association. E.g. things are torn * down, and the related FC-NVME Association ID and Connection IDs * become invalid. * * The behavior of the fc-nvme target is such that it's * understanding of the association and connections will implicitly * be torn down. The action is implicit as it may be due to a loss of * connectivity with the fc-nvme host, so the target may never get a * response even if it tried. As such, the action of this routine * is to asynchronously send the LS, ignore any results of the LS, and * continue on with terminating the association. If the fc-nvme host * is present and receives the LS, it too can tear down. */ static void nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc) { struct nvmet_fc_tgtport *tgtport = assoc->tgtport; struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst; struct fcnvme_ls_disconnect_assoc_acc *discon_acc; struct nvmet_fc_ls_req_op *lsop; struct nvmefc_ls_req *lsreq; int ret; /* * If ls_req is NULL or no hosthandle, it's an older lldd and no * message is normal. Otherwise, send unless the hostport has * already been invalidated by the lldd. */ if (!tgtport->ops->ls_req || !assoc->hostport || assoc->hostport->invalid) return; lsop = kzalloc((sizeof(*lsop) + sizeof(*discon_rqst) + sizeof(*discon_acc) + tgtport->ops->lsrqst_priv_sz), GFP_KERNEL); if (!lsop) { dev_info(tgtport->dev, "{%d:%d} send Disconnect Association failed: ENOMEM\n", tgtport->fc_target_port.port_num, assoc->a_id); return; } discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1]; discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1]; lsreq = &lsop->ls_req; if (tgtport->ops->lsrqst_priv_sz) lsreq->private = (void *)&discon_acc[1]; else lsreq->private = NULL; lsop->tgtport = tgtport; lsop->hosthandle = assoc->hostport->hosthandle; nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc, assoc->association_id); ret = nvmet_fc_send_ls_req_async(tgtport, lsop, nvmet_fc_disconnect_assoc_done); if (ret) { dev_info(tgtport->dev, "{%d:%d} XMT Disconnect Association failed: %d\n", tgtport->fc_target_port.port_num, assoc->a_id, ret); kfree(lsop); } } /* *********************** FC-NVME Port Management ************************ */ static int nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) { struct nvmet_fc_ls_iod *iod; int i; iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod), GFP_KERNEL); if (!iod) return -ENOMEM; tgtport->iod = iod; for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work); iod->tgtport = tgtport; list_add_tail(&iod->ls_rcv_list, &tgtport->ls_rcv_list); iod->rqstbuf = kzalloc(sizeof(union nvmefc_ls_requests) + sizeof(union nvmefc_ls_responses), GFP_KERNEL); if (!iod->rqstbuf) goto out_fail; iod->rspbuf = (union nvmefc_ls_responses *)&iod->rqstbuf[1]; iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf, sizeof(*iod->rspbuf), DMA_TO_DEVICE); if (fc_dma_mapping_error(tgtport->dev, iod->rspdma)) goto out_fail; } return 0; out_fail: kfree(iod->rqstbuf); list_del(&iod->ls_rcv_list); for (iod--, i--; i >= 0; iod--, i--) { fc_dma_unmap_single(tgtport->dev, iod->rspdma, sizeof(*iod->rspbuf), DMA_TO_DEVICE); kfree(iod->rqstbuf); list_del(&iod->ls_rcv_list); } kfree(iod); return -EFAULT; } static void nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport) { struct nvmet_fc_ls_iod *iod = tgtport->iod; int i; for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { fc_dma_unmap_single(tgtport->dev, iod->rspdma, sizeof(*iod->rspbuf), DMA_TO_DEVICE); kfree(iod->rqstbuf); list_del(&iod->ls_rcv_list); } kfree(tgtport->iod); } static struct nvmet_fc_ls_iod * nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport) { struct nvmet_fc_ls_iod *iod; unsigned long flags; spin_lock_irqsave(&tgtport->lock, flags); iod = list_first_entry_or_null(&tgtport->ls_rcv_list, struct nvmet_fc_ls_iod, ls_rcv_list); if (iod) list_move_tail(&iod->ls_rcv_list, &tgtport->ls_busylist); spin_unlock_irqrestore(&tgtport->lock, flags); return iod; } static void nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { unsigned long flags; spin_lock_irqsave(&tgtport->lock, flags); list_move(&iod->ls_rcv_list, &tgtport->ls_rcv_list); spin_unlock_irqrestore(&tgtport->lock, flags); } static void nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_tgt_queue *queue) { struct nvmet_fc_fcp_iod *fod = queue->fod; int i; for (i = 0; i < queue->sqsize; fod++, i++) { INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work); fod->tgtport = tgtport; fod->queue = queue; fod->active = false; fod->abort = false; fod->aborted = false; fod->fcpreq = NULL; list_add_tail(&fod->fcp_list, &queue->fod_list); spin_lock_init(&fod->flock); fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf, sizeof(fod->rspiubuf), DMA_TO_DEVICE); if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) { list_del(&fod->fcp_list); for (fod--, i--; i >= 0; fod--, i--) { fc_dma_unmap_single(tgtport->dev, fod->rspdma, sizeof(fod->rspiubuf), DMA_TO_DEVICE); fod->rspdma = 0L; list_del(&fod->fcp_list); } return; } } } static void nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_tgt_queue *queue) { struct nvmet_fc_fcp_iod *fod = queue->fod; int i; for (i = 0; i < queue->sqsize; fod++, i++) { if (fod->rspdma) fc_dma_unmap_single(tgtport->dev, fod->rspdma, sizeof(fod->rspiubuf), DMA_TO_DEVICE); } } static struct nvmet_fc_fcp_iod * nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue) { struct nvmet_fc_fcp_iod *fod; lockdep_assert_held(&queue->qlock); fod = list_first_entry_or_null(&queue->fod_list, struct nvmet_fc_fcp_iod, fcp_list); if (fod) { list_del(&fod->fcp_list); fod->active = true; /* * no queue reference is taken, as it was taken by the * queue lookup just prior to the allocation. The iod * will "inherit" that reference. */ } return fod; } static void nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_tgt_queue *queue, struct nvmefc_tgt_fcp_req *fcpreq) { struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; /* * put all admin cmds on hw queue id 0. All io commands go to * the respective hw queue based on a modulo basis */ fcpreq->hwqid = queue->qid ? ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0; nvmet_fc_handle_fcp_rqst(tgtport, fod); } static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work) { struct nvmet_fc_fcp_iod *fod = container_of(work, struct nvmet_fc_fcp_iod, defer_work); /* Submit deferred IO for processing */ nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq); } static void nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue, struct nvmet_fc_fcp_iod *fod) { struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; struct nvmet_fc_tgtport *tgtport = fod->tgtport; struct nvmet_fc_defer_fcp_req *deferfcp; unsigned long flags; fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma, sizeof(fod->rspiubuf), DMA_TO_DEVICE); fcpreq->nvmet_fc_private = NULL; fod->active = false; fod->abort = false; fod->aborted = false; fod->writedataactive = false; fod->fcpreq = NULL; tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq); /* release the queue lookup reference on the completed IO */ nvmet_fc_tgt_q_put(queue); spin_lock_irqsave(&queue->qlock, flags); deferfcp = list_first_entry_or_null(&queue->pending_cmd_list, struct nvmet_fc_defer_fcp_req, req_list); if (!deferfcp) { list_add_tail(&fod->fcp_list, &fod->queue->fod_list); spin_unlock_irqrestore(&queue->qlock, flags); return; } /* Re-use the fod for the next pending cmd that was deferred */ list_del(&deferfcp->req_list); fcpreq = deferfcp->fcp_req; /* deferfcp can be reused for another IO at a later date */ list_add_tail(&deferfcp->req_list, &queue->avail_defer_list); spin_unlock_irqrestore(&queue->qlock, flags); /* Save NVME CMD IO in fod */ memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen); /* Setup new fcpreq to be processed */ fcpreq->rspaddr = NULL; fcpreq->rsplen = 0; fcpreq->nvmet_fc_private = fod; fod->fcpreq = fcpreq; fod->active = true; /* inform LLDD IO is now being processed */ tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq); /* * Leave the queue lookup get reference taken when * fod was originally allocated. */ queue_work(queue->work_q, &fod->defer_work); } static struct nvmet_fc_tgt_queue * nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc, u16 qid, u16 sqsize) { struct nvmet_fc_tgt_queue *queue; int ret; if (qid > NVMET_NR_QUEUES) return NULL; queue = kzalloc(struct_size(queue, fod, sqsize), GFP_KERNEL); if (!queue) return NULL; if (!nvmet_fc_tgt_a_get(assoc)) goto out_free_queue; queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0, assoc->tgtport->fc_target_port.port_num, assoc->a_id, qid); if (!queue->work_q) goto out_a_put; queue->qid = qid; queue->sqsize = sqsize; queue->assoc = assoc; INIT_LIST_HEAD(&queue->fod_list); INIT_LIST_HEAD(&queue->avail_defer_list); INIT_LIST_HEAD(&queue->pending_cmd_list); atomic_set(&queue->connected, 0); atomic_set(&queue->sqtail, 0); atomic_set(&queue->rsn, 1); atomic_set(&queue->zrspcnt, 0); spin_lock_init(&queue->qlock); kref_init(&queue->ref); nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue); ret = nvmet_sq_init(&queue->nvme_sq); if (ret) goto out_fail_iodlist; WARN_ON(assoc->queues[qid]); rcu_assign_pointer(assoc->queues[qid], queue); return queue; out_fail_iodlist: nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue); destroy_workqueue(queue->work_q); out_a_put: nvmet_fc_tgt_a_put(assoc); out_free_queue: kfree(queue); return NULL; } static void nvmet_fc_tgt_queue_free(struct kref *ref) { struct nvmet_fc_tgt_queue *queue = container_of(ref, struct nvmet_fc_tgt_queue, ref); rcu_assign_pointer(queue->assoc->queues[queue->qid], NULL); nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue); nvmet_fc_tgt_a_put(queue->assoc); destroy_workqueue(queue->work_q); kfree_rcu(queue, rcu); } static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue) { kref_put(&queue->ref, nvmet_fc_tgt_queue_free); } static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue) { return kref_get_unless_zero(&queue->ref); } static void nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue) { struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport; struct nvmet_fc_fcp_iod *fod = queue->fod; struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr; unsigned long flags; int i; bool disconnect; disconnect = atomic_xchg(&queue->connected, 0); /* if not connected, nothing to do */ if (!disconnect) return; spin_lock_irqsave(&queue->qlock, flags); /* abort outstanding io's */ for (i = 0; i < queue->sqsize; fod++, i++) { if (fod->active) { spin_lock(&fod->flock); fod->abort = true; /* * only call lldd abort routine if waiting for * writedata. other outstanding ops should finish * on their own. */ if (fod->writedataactive) { fod->aborted = true; spin_unlock(&fod->flock); tgtport->ops->fcp_abort( &tgtport->fc_target_port, fod->fcpreq); } else spin_unlock(&fod->flock); } } /* Cleanup defer'ed IOs in queue */ list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list, req_list) { list_del(&deferfcp->req_list); kfree(deferfcp); } for (;;) { deferfcp = list_first_entry_or_null(&queue->pending_cmd_list, struct nvmet_fc_defer_fcp_req, req_list); if (!deferfcp) break; list_del(&deferfcp->req_list); spin_unlock_irqrestore(&queue->qlock, flags); tgtport->ops->defer_rcv(&tgtport->fc_target_port, deferfcp->fcp_req); tgtport->ops->fcp_abort(&tgtport->fc_target_port, deferfcp->fcp_req); tgtport->ops->fcp_req_release(&tgtport->fc_target_port, deferfcp->fcp_req); /* release the queue lookup reference */ nvmet_fc_tgt_q_put(queue); kfree(deferfcp); spin_lock_irqsave(&queue->qlock, flags); } spin_unlock_irqrestore(&queue->qlock, flags); flush_workqueue(queue->work_q); nvmet_sq_destroy(&queue->nvme_sq); nvmet_fc_tgt_q_put(queue); } static struct nvmet_fc_tgt_queue * nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport, u64 connection_id) { struct nvmet_fc_tgt_assoc *assoc; struct nvmet_fc_tgt_queue *queue; u64 association_id = nvmet_fc_getassociationid(connection_id); u16 qid = nvmet_fc_getqueueid(connection_id); if (qid > NVMET_NR_QUEUES) return NULL; rcu_read_lock(); list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { if (association_id == assoc->association_id) { queue = rcu_dereference(assoc->queues[qid]); if (queue && (!atomic_read(&queue->connected) || !nvmet_fc_tgt_q_get(queue))) queue = NULL; rcu_read_unlock(); return queue; } } rcu_read_unlock(); return NULL; } static void nvmet_fc_hostport_free(struct kref *ref) { struct nvmet_fc_hostport *hostport = container_of(ref, struct nvmet_fc_hostport, ref); struct nvmet_fc_tgtport *tgtport = hostport->tgtport; unsigned long flags; spin_lock_irqsave(&tgtport->lock, flags); list_del(&hostport->host_list); spin_unlock_irqrestore(&tgtport->lock, flags); if (tgtport->ops->host_release && hostport->invalid) tgtport->ops->host_release(hostport->hosthandle); kfree(hostport); nvmet_fc_tgtport_put(tgtport); } static void nvmet_fc_hostport_put(struct nvmet_fc_hostport *hostport) { kref_put(&hostport->ref, nvmet_fc_hostport_free); } static int nvmet_fc_hostport_get(struct nvmet_fc_hostport *hostport) { return kref_get_unless_zero(&hostport->ref); } static void nvmet_fc_free_hostport(struct nvmet_fc_hostport *hostport) { /* if LLDD not implemented, leave as NULL */ if (!hostport || !hostport->hosthandle) return; nvmet_fc_hostport_put(hostport); } static struct nvmet_fc_hostport * nvmet_fc_match_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle) { struct nvmet_fc_hostport *host; lockdep_assert_held(&tgtport->lock); list_for_each_entry(host, &tgtport->host_list, host_list) { if (host->hosthandle == hosthandle && !host->invalid) { if (nvmet_fc_hostport_get(host)) return (host); } } return NULL; } static struct nvmet_fc_hostport * nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle) { struct nvmet_fc_hostport *newhost, *match = NULL; unsigned long flags; /* if LLDD not implemented, leave as NULL */ if (!hosthandle) return NULL; /* * take reference for what will be the newly allocated hostport if * we end up using a new allocation */ if (!nvmet_fc_tgtport_get(tgtport)) return ERR_PTR(-EINVAL); spin_lock_irqsave(&tgtport->lock, flags); match = nvmet_fc_match_hostport(tgtport, hosthandle); spin_unlock_irqrestore(&tgtport->lock, flags); if (match) { /* no new allocation - release reference */ nvmet_fc_tgtport_put(tgtport); return match; } newhost = kzalloc(sizeof(*newhost), GFP_KERNEL); if (!newhost) { /* no new allocation - release reference */ nvmet_fc_tgtport_put(tgtport); return ERR_PTR(-ENOMEM); } spin_lock_irqsave(&tgtport->lock, flags); match = nvmet_fc_match_hostport(tgtport, hosthandle); if (match) { /* new allocation not needed */ kfree(newhost); newhost = match; /* no new allocation - release reference */ nvmet_fc_tgtport_put(tgtport); } else { newhost->tgtport = tgtport; newhost->hosthandle = hosthandle; INIT_LIST_HEAD(&newhost->host_list); kref_init(&newhost->ref); list_add_tail(&newhost->host_list, &tgtport->host_list); } spin_unlock_irqrestore(&tgtport->lock, flags); return newhost; } static void nvmet_fc_delete_assoc(struct work_struct *work) { struct nvmet_fc_tgt_assoc *assoc = container_of(work, struct nvmet_fc_tgt_assoc, del_work); nvmet_fc_delete_target_assoc(assoc); nvmet_fc_tgt_a_put(assoc); } static struct nvmet_fc_tgt_assoc * nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle) { struct nvmet_fc_tgt_assoc *assoc, *tmpassoc; unsigned long flags; u64 ran; int idx; bool needrandom = true; assoc = kzalloc(sizeof(*assoc), GFP_KERNEL); if (!assoc) return NULL; idx = ida_alloc(&tgtport->assoc_cnt, GFP_KERNEL); if (idx < 0) goto out_free_assoc; if (!nvmet_fc_tgtport_get(tgtport)) goto out_ida; assoc->hostport = nvmet_fc_alloc_hostport(tgtport, hosthandle); if (IS_ERR(assoc->hostport)) goto out_put; assoc->tgtport = tgtport; assoc->a_id = idx; INIT_LIST_HEAD(&assoc->a_list); kref_init(&assoc->ref); INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc); atomic_set(&assoc->terminating, 0); while (needrandom) { get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID); ran = ran << BYTES_FOR_QID_SHIFT; spin_lock_irqsave(&tgtport->lock, flags); needrandom = false; list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list) { if (ran == tmpassoc->association_id) { needrandom = true; break; } } if (!needrandom) { assoc->association_id = ran; list_add_tail_rcu(&assoc->a_list, &tgtport->assoc_list); } spin_unlock_irqrestore(&tgtport->lock, flags); } return assoc; out_put: nvmet_fc_tgtport_put(tgtport); out_ida: ida_free(&tgtport->assoc_cnt, idx); out_free_assoc: kfree(assoc); return NULL; } static void nvmet_fc_target_assoc_free(struct kref *ref) { struct nvmet_fc_tgt_assoc *assoc = container_of(ref, struct nvmet_fc_tgt_assoc, ref); struct nvmet_fc_tgtport *tgtport = assoc->tgtport; struct nvmet_fc_ls_iod *oldls; unsigned long flags; /* Send Disconnect now that all i/o has completed */ nvmet_fc_xmt_disconnect_assoc(assoc); nvmet_fc_free_hostport(assoc->hostport); spin_lock_irqsave(&tgtport->lock, flags); list_del_rcu(&assoc->a_list); oldls = assoc->rcv_disconn; spin_unlock_irqrestore(&tgtport->lock, flags); /* if pending Rcv Disconnect Association LS, send rsp now */ if (oldls) nvmet_fc_xmt_ls_rsp(tgtport, oldls); ida_free(&tgtport->assoc_cnt, assoc->a_id); dev_info(tgtport->dev, "{%d:%d} Association freed\n", tgtport->fc_target_port.port_num, assoc->a_id); kfree_rcu(assoc, rcu); nvmet_fc_tgtport_put(tgtport); } static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc) { kref_put(&assoc->ref, nvmet_fc_target_assoc_free); } static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc) { return kref_get_unless_zero(&assoc->ref); } static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc) { struct nvmet_fc_tgtport *tgtport = assoc->tgtport; struct nvmet_fc_tgt_queue *queue; int i, terminating; terminating = atomic_xchg(&assoc->terminating, 1); /* if already terminating, do nothing */ if (terminating) return; for (i = NVMET_NR_QUEUES; i >= 0; i--) { rcu_read_lock(); queue = rcu_dereference(assoc->queues[i]); if (!queue) { rcu_read_unlock(); continue; } if (!nvmet_fc_tgt_q_get(queue)) { rcu_read_unlock(); continue; } rcu_read_unlock(); nvmet_fc_delete_target_queue(queue); nvmet_fc_tgt_q_put(queue); } dev_info(tgtport->dev, "{%d:%d} Association deleted\n", tgtport->fc_target_port.port_num, assoc->a_id); nvmet_fc_tgt_a_put(assoc); } static struct nvmet_fc_tgt_assoc * nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport, u64 association_id) { struct nvmet_fc_tgt_assoc *assoc; struct nvmet_fc_tgt_assoc *ret = NULL; rcu_read_lock(); list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { if (association_id == assoc->association_id) { ret = assoc; if (!nvmet_fc_tgt_a_get(assoc)) ret = NULL; break; } } rcu_read_unlock(); return ret; } static void nvmet_fc_portentry_bind(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_port_entry *pe, struct nvmet_port *port) { lockdep_assert_held(&nvmet_fc_tgtlock); pe->tgtport = tgtport; tgtport->pe = pe; pe->port = port; port->priv = pe; pe->node_name = tgtport->fc_target_port.node_name; pe->port_name = tgtport->fc_target_port.port_name; INIT_LIST_HEAD(&pe->pe_list); list_add_tail(&pe->pe_list, &nvmet_fc_portentry_list); } static void nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry *pe) { unsigned long flags; spin_lock_irqsave(&nvmet_fc_tgtlock, flags); if (pe->tgtport) pe->tgtport->pe = NULL; list_del(&pe->pe_list); spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); } /* * called when a targetport deregisters. Breaks the relationship * with the nvmet port, but leaves the port_entry in place so that * re-registration can resume operation. */ static void nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport *tgtport) { struct nvmet_fc_port_entry *pe; unsigned long flags; spin_lock_irqsave(&nvmet_fc_tgtlock, flags); pe = tgtport->pe; if (pe) pe->tgtport = NULL; tgtport->pe = NULL; spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); } /* * called when a new targetport is registered. Looks in the * existing nvmet port_entries to see if the nvmet layer is * configured for the targetport's wwn's. (the targetport existed, * nvmet configured, the lldd unregistered the tgtport, and is now * reregistering the same targetport). If so, set the nvmet port * port entry on the targetport. */ static void nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport *tgtport) { struct nvmet_fc_port_entry *pe; unsigned long flags; spin_lock_irqsave(&nvmet_fc_tgtlock, flags); list_for_each_entry(pe, &nvmet_fc_portentry_list, pe_list) { if (tgtport->fc_target_port.node_name == pe->node_name && tgtport->fc_target_port.port_name == pe->port_name) { WARN_ON(pe->tgtport); tgtport->pe = pe; pe->tgtport = tgtport; break; } } spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); } /** * nvmet_fc_register_targetport - transport entry point called by an * LLDD to register the existence of a local * NVME subystem FC port. * @pinfo: pointer to information about the port to be registered * @template: LLDD entrypoints and operational parameters for the port * @dev: physical hardware device node port corresponds to. Will be * used for DMA mappings * @portptr: pointer to a local port pointer. Upon success, the routine * will allocate a nvme_fc_local_port structure and place its * address in the local port pointer. Upon failure, local port * pointer will be set to NULL. * * Returns: * a completion status. Must be 0 upon success; a negative errno * (ex: -ENXIO) upon failure. */ int nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, struct nvmet_fc_target_template *template, struct device *dev, struct nvmet_fc_target_port **portptr) { struct nvmet_fc_tgtport *newrec; unsigned long flags; int ret, idx; if (!template->xmt_ls_rsp || !template->fcp_op || !template->fcp_abort || !template->fcp_req_release || !template->targetport_delete || !template->max_hw_queues || !template->max_sgl_segments || !template->max_dif_sgl_segments || !template->dma_boundary) { ret = -EINVAL; goto out_regtgt_failed; } newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz), GFP_KERNEL); if (!newrec) { ret = -ENOMEM; goto out_regtgt_failed; } idx = ida_alloc(&nvmet_fc_tgtport_cnt, GFP_KERNEL); if (idx < 0) { ret = -ENOSPC; goto out_fail_kfree; } if (!get_device(dev) && dev) { ret = -ENODEV; goto out_ida_put; } newrec->fc_target_port.node_name = pinfo->node_name; newrec->fc_target_port.port_name = pinfo->port_name; if (template->target_priv_sz) newrec->fc_target_port.private = &newrec[1]; else newrec->fc_target_port.private = NULL; newrec->fc_target_port.port_id = pinfo->port_id; newrec->fc_target_port.port_num = idx; INIT_LIST_HEAD(&newrec->tgt_list); newrec->dev = dev; newrec->ops = template; spin_lock_init(&newrec->lock); INIT_LIST_HEAD(&newrec->ls_rcv_list); INIT_LIST_HEAD(&newrec->ls_req_list); INIT_LIST_HEAD(&newrec->ls_busylist); INIT_LIST_HEAD(&newrec->assoc_list); INIT_LIST_HEAD(&newrec->host_list); kref_init(&newrec->ref); ida_init(&newrec->assoc_cnt); newrec->max_sg_cnt = template->max_sgl_segments; ret = nvmet_fc_alloc_ls_iodlist(newrec); if (ret) { ret = -ENOMEM; goto out_free_newrec; } nvmet_fc_portentry_rebind_tgt(newrec); spin_lock_irqsave(&nvmet_fc_tgtlock, flags); list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list); spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); *portptr = &newrec->fc_target_port; return 0; out_free_newrec: put_device(dev); out_ida_put: ida_free(&nvmet_fc_tgtport_cnt, idx); out_fail_kfree: kfree(newrec); out_regtgt_failed: *portptr = NULL; return ret; } EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport); static void nvmet_fc_free_tgtport(struct kref *ref) { struct nvmet_fc_tgtport *tgtport = container_of(ref, struct nvmet_fc_tgtport, ref); struct device *dev = tgtport->dev; unsigned long flags; spin_lock_irqsave(&nvmet_fc_tgtlock, flags); list_del(&tgtport->tgt_list); spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); nvmet_fc_free_ls_iodlist(tgtport); /* let the LLDD know we've finished tearing it down */ tgtport->ops->targetport_delete(&tgtport->fc_target_port); ida_free(&nvmet_fc_tgtport_cnt, tgtport->fc_target_port.port_num); ida_destroy(&tgtport->assoc_cnt); kfree(tgtport); put_device(dev); } static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport) { kref_put(&tgtport->ref, nvmet_fc_free_tgtport); } static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport) { return kref_get_unless_zero(&tgtport->ref); } static void __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport) { struct nvmet_fc_tgt_assoc *assoc; rcu_read_lock(); list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { if (!nvmet_fc_tgt_a_get(assoc)) continue; if (!queue_work(nvmet_wq, &assoc->del_work)) /* already deleting - release local reference */ nvmet_fc_tgt_a_put(assoc); } rcu_read_unlock(); } /** * nvmet_fc_invalidate_host - transport entry point called by an LLDD * to remove references to a hosthandle for LS's. * * The nvmet-fc layer ensures that any references to the hosthandle * on the targetport are forgotten (set to NULL). The LLDD will * typically call this when a login with a remote host port has been * lost, thus LS's for the remote host port are no longer possible. * * If an LS request is outstanding to the targetport/hosthandle (or * issued concurrently with the call to invalidate the host), the * LLDD is responsible for terminating/aborting the LS and completing * the LS request. It is recommended that these terminations/aborts * occur after calling to invalidate the host handle to avoid additional * retries by the nvmet-fc transport. The nvmet-fc transport may * continue to reference host handle while it cleans up outstanding * NVME associations. The nvmet-fc transport will call the * ops->host_release() callback to notify the LLDD that all references * are complete and the related host handle can be recovered. * Note: if there are no references, the callback may be called before * the invalidate host call returns. * * @target_port: pointer to the (registered) target port that a prior * LS was received on and which supplied the transport the * hosthandle. * @hosthandle: the handle (pointer) that represents the host port * that no longer has connectivity and that LS's should * no longer be directed to. */ void nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port, void *hosthandle) { struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); struct nvmet_fc_tgt_assoc *assoc, *next; unsigned long flags; bool noassoc = true; spin_lock_irqsave(&tgtport->lock, flags); list_for_each_entry_safe(assoc, next, &tgtport->assoc_list, a_list) { if (!assoc->hostport || assoc->hostport->hosthandle != hosthandle) continue; if (!nvmet_fc_tgt_a_get(assoc)) continue; assoc->hostport->invalid = 1; noassoc = false; if (!queue_work(nvmet_wq, &assoc->del_work)) /* already deleting - release local reference */ nvmet_fc_tgt_a_put(assoc); } spin_unlock_irqrestore(&tgtport->lock, flags); /* if there's nothing to wait for - call the callback */ if (noassoc && tgtport->ops->host_release) tgtport->ops->host_release(hosthandle); } EXPORT_SYMBOL_GPL(nvmet_fc_invalidate_host); /* * nvmet layer has called to terminate an association */ static void nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) { struct nvmet_fc_tgtport *tgtport, *next; struct nvmet_fc_tgt_assoc *assoc; struct nvmet_fc_tgt_queue *queue; unsigned long flags; bool found_ctrl = false; /* this is a bit ugly, but don't want to make locks layered */ spin_lock_irqsave(&nvmet_fc_tgtlock, flags); list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list, tgt_list) { if (!nvmet_fc_tgtport_get(tgtport)) continue; spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); rcu_read_lock(); list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { queue = rcu_dereference(assoc->queues[0]); if (queue && queue->nvme_sq.ctrl == ctrl) { if (nvmet_fc_tgt_a_get(assoc)) found_ctrl = true; break; } } rcu_read_unlock(); nvmet_fc_tgtport_put(tgtport); if (found_ctrl) { if (!queue_work(nvmet_wq, &assoc->del_work)) /* already deleting - release local reference */ nvmet_fc_tgt_a_put(assoc); return; } spin_lock_irqsave(&nvmet_fc_tgtlock, flags); } spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); } /** * nvmet_fc_unregister_targetport - transport entry point called by an * LLDD to deregister/remove a previously * registered a local NVME subsystem FC port. * @target_port: pointer to the (registered) target port that is to be * deregistered. * * Returns: * a completion status. Must be 0 upon success; a negative errno * (ex: -ENXIO) upon failure. */ int nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port) { struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); nvmet_fc_portentry_unbind_tgt(tgtport); /* terminate any outstanding associations */ __nvmet_fc_free_assocs(tgtport); /* * should terminate LS's as well. However, LS's will be generated * at the tail end of association termination, so they likely don't * exist yet. And even if they did, it's worthwhile to just let * them finish and targetport ref counting will clean things up. */ nvmet_fc_tgtport_put(tgtport); return 0; } EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport); /* ********************** FC-NVME LS RCV Handling ************************* */ static void nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { struct fcnvme_ls_cr_assoc_rqst *rqst = &iod->rqstbuf->rq_cr_assoc; struct fcnvme_ls_cr_assoc_acc *acc = &iod->rspbuf->rsp_cr_assoc; struct nvmet_fc_tgt_queue *queue; int ret = 0; memset(acc, 0, sizeof(*acc)); /* * FC-NVME spec changes. There are initiators sending different * lengths as padding sizes for Create Association Cmd descriptor * was incorrect. * Accept anything of "minimum" length. Assume format per 1.15 * spec (with HOSTID reduced to 16 bytes), ignore how long the * trailing pad length is. */ if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN) ret = VERR_CR_ASSOC_LEN; else if (be32_to_cpu(rqst->desc_list_len) < FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN) ret = VERR_CR_ASSOC_RQST_LEN; else if (rqst->assoc_cmd.desc_tag != cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD)) ret = VERR_CR_ASSOC_CMD; else if (be32_to_cpu(rqst->assoc_cmd.desc_len) < FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN) ret = VERR_CR_ASSOC_CMD_LEN; else if (!rqst->assoc_cmd.ersp_ratio || (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >= be16_to_cpu(rqst->assoc_cmd.sqsize))) ret = VERR_ERSP_RATIO; else { /* new association w/ admin queue */ iod->assoc = nvmet_fc_alloc_target_assoc( tgtport, iod->hosthandle); if (!iod->assoc) ret = VERR_ASSOC_ALLOC_FAIL; else { queue = nvmet_fc_alloc_target_queue(iod->assoc, 0, be16_to_cpu(rqst->assoc_cmd.sqsize)); if (!queue) ret = VERR_QUEUE_ALLOC_FAIL; } } if (ret) { dev_err(tgtport->dev, "Create Association LS failed: %s\n", validation_errors[ret]); iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, sizeof(*acc), rqst->w0.ls_cmd, FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); return; } queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio); atomic_set(&queue->connected, 1); queue->sqhd = 0; /* best place to init value */ dev_info(tgtport->dev, "{%d:%d} Association created\n", tgtport->fc_target_port.port_num, iod->assoc->a_id); /* format a response */ iod->lsrsp->rsplen = sizeof(*acc); nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len( sizeof(struct fcnvme_ls_cr_assoc_acc)), FCNVME_LS_CREATE_ASSOCIATION); acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); acc->associd.desc_len = fcnvme_lsdesc_len( sizeof(struct fcnvme_lsdesc_assoc_id)); acc->associd.association_id = cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0)); acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); acc->connectid.desc_len = fcnvme_lsdesc_len( sizeof(struct fcnvme_lsdesc_conn_id)); acc->connectid.connection_id = acc->associd.association_id; } static void nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { struct fcnvme_ls_cr_conn_rqst *rqst = &iod->rqstbuf->rq_cr_conn; struct fcnvme_ls_cr_conn_acc *acc = &iod->rspbuf->rsp_cr_conn; struct nvmet_fc_tgt_queue *queue; int ret = 0; memset(acc, 0, sizeof(*acc)); if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst)) ret = VERR_CR_CONN_LEN; else if (rqst->desc_list_len != fcnvme_lsdesc_len( sizeof(struct fcnvme_ls_cr_conn_rqst))) ret = VERR_CR_CONN_RQST_LEN; else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) ret = VERR_ASSOC_ID; else if (rqst->associd.desc_len != fcnvme_lsdesc_len( sizeof(struct fcnvme_lsdesc_assoc_id))) ret = VERR_ASSOC_ID_LEN; else if (rqst->connect_cmd.desc_tag != cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD)) ret = VERR_CR_CONN_CMD; else if (rqst->connect_cmd.desc_len != fcnvme_lsdesc_len( sizeof(struct fcnvme_lsdesc_cr_conn_cmd))) ret = VERR_CR_CONN_CMD_LEN; else if (!rqst->connect_cmd.ersp_ratio || (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >= be16_to_cpu(rqst->connect_cmd.sqsize))) ret = VERR_ERSP_RATIO; else { /* new io queue */ iod->assoc = nvmet_fc_find_target_assoc(tgtport, be64_to_cpu(rqst->associd.association_id)); if (!iod->assoc) ret = VERR_NO_ASSOC; else { queue = nvmet_fc_alloc_target_queue(iod->assoc, be16_to_cpu(rqst->connect_cmd.qid), be16_to_cpu(rqst->connect_cmd.sqsize)); if (!queue) ret = VERR_QUEUE_ALLOC_FAIL; /* release get taken in nvmet_fc_find_target_assoc */ nvmet_fc_tgt_a_put(iod->assoc); } } if (ret) { dev_err(tgtport->dev, "Create Connection LS failed: %s\n", validation_errors[ret]); iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, sizeof(*acc), rqst->w0.ls_cmd, (ret == VERR_NO_ASSOC) ? FCNVME_RJT_RC_INV_ASSOC : FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); return; } queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio); atomic_set(&queue->connected, 1); queue->sqhd = 0; /* best place to init value */ /* format a response */ iod->lsrsp->rsplen = sizeof(*acc); nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)), FCNVME_LS_CREATE_CONNECTION); acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); acc->connectid.desc_len = fcnvme_lsdesc_len( sizeof(struct fcnvme_lsdesc_conn_id)); acc->connectid.connection_id = cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, be16_to_cpu(rqst->connect_cmd.qid))); } /* * Returns true if the LS response is to be transmit * Returns false if the LS response is to be delayed */ static int nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { struct fcnvme_ls_disconnect_assoc_rqst *rqst = &iod->rqstbuf->rq_dis_assoc; struct fcnvme_ls_disconnect_assoc_acc *acc = &iod->rspbuf->rsp_dis_assoc; struct nvmet_fc_tgt_assoc *assoc = NULL; struct nvmet_fc_ls_iod *oldls = NULL; unsigned long flags; int ret = 0; memset(acc, 0, sizeof(*acc)); ret = nvmefc_vldt_lsreq_discon_assoc(iod->rqstdatalen, rqst); if (!ret) { /* match an active association - takes an assoc ref if !NULL */ assoc = nvmet_fc_find_target_assoc(tgtport, be64_to_cpu(rqst->associd.association_id)); iod->assoc = assoc; if (!assoc) ret = VERR_NO_ASSOC; } if (ret || !assoc) { dev_err(tgtport->dev, "Disconnect LS failed: %s\n", validation_errors[ret]); iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, sizeof(*acc), rqst->w0.ls_cmd, (ret == VERR_NO_ASSOC) ? FCNVME_RJT_RC_INV_ASSOC : FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); return true; } /* format a response */ iod->lsrsp->rsplen = sizeof(*acc); nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len( sizeof(struct fcnvme_ls_disconnect_assoc_acc)), FCNVME_LS_DISCONNECT_ASSOC); /* release get taken in nvmet_fc_find_target_assoc */ nvmet_fc_tgt_a_put(assoc); /* * The rules for LS response says the response cannot * go back until ABTS's have been sent for all outstanding * I/O and a Disconnect Association LS has been sent. * So... save off the Disconnect LS to send the response * later. If there was a prior LS already saved, replace * it with the newer one and send a can't perform reject * on the older one. */ spin_lock_irqsave(&tgtport->lock, flags); oldls = assoc->rcv_disconn; assoc->rcv_disconn = iod; spin_unlock_irqrestore(&tgtport->lock, flags); nvmet_fc_delete_target_assoc(assoc); if (oldls) { dev_info(tgtport->dev, "{%d:%d} Multiple Disconnect Association LS's " "received\n", tgtport->fc_target_port.port_num, assoc->a_id); /* overwrite good response with bogus failure */ oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf, sizeof(*iod->rspbuf), /* ok to use rqst, LS is same */ rqst->w0.ls_cmd, FCNVME_RJT_RC_UNAB, FCNVME_RJT_EXP_NONE, 0); nvmet_fc_xmt_ls_rsp(tgtport, oldls); } return false; } /* *********************** NVME Ctrl Routines **************************** */ static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req); static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops; static void nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp) { struct nvmet_fc_ls_iod *iod = lsrsp->nvme_fc_private; struct nvmet_fc_tgtport *tgtport = iod->tgtport; fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma, sizeof(*iod->rspbuf), DMA_TO_DEVICE); nvmet_fc_free_ls_iod(tgtport, iod); nvmet_fc_tgtport_put(tgtport); } static void nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { int ret; fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma, sizeof(*iod->rspbuf), DMA_TO_DEVICE); ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsrsp); if (ret) nvmet_fc_xmt_ls_rsp_done(iod->lsrsp); } /* * Actual processing routine for received FC-NVME LS Requests from the LLD */ static void nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { struct fcnvme_ls_rqst_w0 *w0 = &iod->rqstbuf->rq_cr_assoc.w0; bool sendrsp = true; iod->lsrsp->nvme_fc_private = iod; iod->lsrsp->rspbuf = iod->rspbuf; iod->lsrsp->rspdma = iod->rspdma; iod->lsrsp->done = nvmet_fc_xmt_ls_rsp_done; /* Be preventative. handlers will later set to valid length */ iod->lsrsp->rsplen = 0; iod->assoc = NULL; /* * handlers: * parse request input, execute the request, and format the * LS response */ switch (w0->ls_cmd) { case FCNVME_LS_CREATE_ASSOCIATION: /* Creates Association and initial Admin Queue/Connection */ nvmet_fc_ls_create_association(tgtport, iod); break; case FCNVME_LS_CREATE_CONNECTION: /* Creates an IO Queue/Connection */ nvmet_fc_ls_create_connection(tgtport, iod); break; case FCNVME_LS_DISCONNECT_ASSOC: /* Terminate a Queue/Connection or the Association */ sendrsp = nvmet_fc_ls_disconnect(tgtport, iod); break; default: iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf, sizeof(*iod->rspbuf), w0->ls_cmd, FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); } if (sendrsp) nvmet_fc_xmt_ls_rsp(tgtport, iod); } /* * Actual processing routine for received FC-NVME LS Requests from the LLD */ static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work) { struct nvmet_fc_ls_iod *iod = container_of(work, struct nvmet_fc_ls_iod, work); struct nvmet_fc_tgtport *tgtport = iod->tgtport; nvmet_fc_handle_ls_rqst(tgtport, iod); } /** * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD * upon the reception of a NVME LS request. * * The nvmet-fc layer will copy payload to an internal structure for * processing. As such, upon completion of the routine, the LLDD may * immediately free/reuse the LS request buffer passed in the call. * * If this routine returns error, the LLDD should abort the exchange. * * @target_port: pointer to the (registered) target port the LS was * received on. * @hosthandle: pointer to the host specific data, gets stored in iod. * @lsrsp: pointer to a lsrsp structure to be used to reference * the exchange corresponding to the LS. * @lsreqbuf: pointer to the buffer containing the LS Request * @lsreqbuf_len: length, in bytes, of the received LS request */ int nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, void *hosthandle, struct nvmefc_ls_rsp *lsrsp, void *lsreqbuf, u32 lsreqbuf_len) { struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); struct nvmet_fc_ls_iod *iod; struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf; if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) { dev_info(tgtport->dev, "RCV %s LS failed: payload too large (%d)\n", (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? nvmefc_ls_names[w0->ls_cmd] : "", lsreqbuf_len); return -E2BIG; } if (!nvmet_fc_tgtport_get(tgtport)) { dev_info(tgtport->dev, "RCV %s LS failed: target deleting\n", (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? nvmefc_ls_names[w0->ls_cmd] : ""); return -ESHUTDOWN; } iod = nvmet_fc_alloc_ls_iod(tgtport); if (!iod) { dev_info(tgtport->dev, "RCV %s LS failed: context allocation failed\n", (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? nvmefc_ls_names[w0->ls_cmd] : ""); nvmet_fc_tgtport_put(tgtport); return -ENOENT; } iod->lsrsp = lsrsp; iod->fcpreq = NULL; memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len); iod->rqstdatalen = lsreqbuf_len; iod->hosthandle = hosthandle; queue_work(nvmet_wq, &iod->work); return 0; } EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req); /* * ********************** * Start of FCP handling * ********************** */ static int nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod) { struct scatterlist *sg; unsigned int nent; sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent); if (!sg) goto out; fod->data_sg = sg; fod->data_sg_cnt = nent; fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent, ((fod->io_dir == NVMET_FCP_WRITE) ? DMA_FROM_DEVICE : DMA_TO_DEVICE)); /* note: write from initiator perspective */ fod->next_sg = fod->data_sg; return 0; out: return NVME_SC_INTERNAL; } static void nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod) { if (!fod->data_sg || !fod->data_sg_cnt) return; fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt, ((fod->io_dir == NVMET_FCP_WRITE) ? DMA_FROM_DEVICE : DMA_TO_DEVICE)); sgl_free(fod->data_sg); fod->data_sg = NULL; fod->data_sg_cnt = 0; } static bool queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd) { u32 sqtail, used; /* egad, this is ugly. And sqtail is just a best guess */ sqtail = atomic_read(&q->sqtail) % q->sqsize; used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd); return ((used * 10) >= (((u32)(q->sqsize - 1) * 9))); } /* * Prep RSP payload. * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op */ static void nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_fcp_iod *fod) { struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf; struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common; struct nvme_completion *cqe = &ersp->cqe; u32 *cqewd = (u32 *)cqe; bool send_ersp = false; u32 rsn, rspcnt, xfr_length; if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP) xfr_length = fod->req.transfer_len; else xfr_length = fod->offset; /* * check to see if we can send a 0's rsp. * Note: to send a 0's response, the NVME-FC host transport will * recreate the CQE. The host transport knows: sq id, SQHD (last * seen in an ersp), and command_id. Thus it will create a * zero-filled CQE with those known fields filled in. Transport * must send an ersp for any condition where the cqe won't match * this. * * Here are the FC-NVME mandated cases where we must send an ersp: * every N responses, where N=ersp_ratio * force fabric commands to send ersp's (not in FC-NVME but good * practice) * normal cmds: any time status is non-zero, or status is zero * but words 0 or 1 are non-zero. * the SQ is 90% or more full * the cmd is a fused command * transferred data length not equal to cmd iu length */ rspcnt = atomic_inc_return(&fod->queue->zrspcnt); if (!(rspcnt % fod->queue->ersp_ratio) || nvme_is_fabrics((struct nvme_command *) sqe) || xfr_length != fod->req.transfer_len || (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] || (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) || queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head))) send_ersp = true; /* re-set the fields */ fod->fcpreq->rspaddr = ersp; fod->fcpreq->rspdma = fod->rspdma; if (!send_ersp) { memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP); fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP; } else { ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32)); rsn = atomic_inc_return(&fod->queue->rsn); ersp->rsn = cpu_to_be32(rsn); ersp->xfrd_len = cpu_to_be32(xfr_length); fod->fcpreq->rsplen = sizeof(*ersp); } fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma, sizeof(fod->rspiubuf), DMA_TO_DEVICE); } static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq); static void nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_fcp_iod *fod) { struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; /* data no longer needed */ nvmet_fc_free_tgt_pgs(fod); /* * if an ABTS was received or we issued the fcp_abort early * don't call abort routine again. */ /* no need to take lock - lock was taken earlier to get here */ if (!fod->aborted) tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq); nvmet_fc_free_fcp_iod(fod->queue, fod); } static void nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_fcp_iod *fod) { int ret; fod->fcpreq->op = NVMET_FCOP_RSP; fod->fcpreq->timeout = 0; nvmet_fc_prep_fcp_rsp(tgtport, fod); ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); if (ret) nvmet_fc_abort_op(tgtport, fod); } static void nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_fcp_iod *fod, u8 op) { struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; struct scatterlist *sg = fod->next_sg; unsigned long flags; u32 remaininglen = fod->req.transfer_len - fod->offset; u32 tlen = 0; int ret; fcpreq->op = op; fcpreq->offset = fod->offset; fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC; /* * for next sequence: * break at a sg element boundary * attempt to keep sequence length capped at * NVMET_FC_MAX_SEQ_LENGTH but allow sequence to * be longer if a single sg element is larger * than that amount. This is done to avoid creating * a new sg list to use for the tgtport api. */ fcpreq->sg = sg; fcpreq->sg_cnt = 0; while (tlen < remaininglen && fcpreq->sg_cnt < tgtport->max_sg_cnt && tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) { fcpreq->sg_cnt++; tlen += sg_dma_len(sg); sg = sg_next(sg); } if (tlen < remaininglen && fcpreq->sg_cnt == 0) { fcpreq->sg_cnt++; tlen += min_t(u32, sg_dma_len(sg), remaininglen); sg = sg_next(sg); } if (tlen < remaininglen) fod->next_sg = sg; else fod->next_sg = NULL; fcpreq->transfer_length = tlen; fcpreq->transferred_length = 0; fcpreq->fcp_error = 0; fcpreq->rsplen = 0; /* * If the last READDATA request: check if LLDD supports * combined xfr with response. */ if ((op == NVMET_FCOP_READDATA) && ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) && (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) { fcpreq->op = NVMET_FCOP_READDATA_RSP; nvmet_fc_prep_fcp_rsp(tgtport, fod); } ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); if (ret) { /* * should be ok to set w/o lock as its in the thread of * execution (not an async timer routine) and doesn't * contend with any clearing action */ fod->abort = true; if (op == NVMET_FCOP_WRITEDATA) { spin_lock_irqsave(&fod->flock, flags); fod->writedataactive = false; spin_unlock_irqrestore(&fod->flock, flags); nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ { fcpreq->fcp_error = ret; fcpreq->transferred_length = 0; nvmet_fc_xmt_fcp_op_done(fod->fcpreq); } } } static inline bool __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort) { struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; struct nvmet_fc_tgtport *tgtport = fod->tgtport; /* if in the middle of an io and we need to tear down */ if (abort) { if (fcpreq->op == NVMET_FCOP_WRITEDATA) { nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); return true; } nvmet_fc_abort_op(tgtport, fod); return true; } return false; } /* * actual done handler for FCP operations when completed by the lldd */ static void nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod) { struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; struct nvmet_fc_tgtport *tgtport = fod->tgtport; unsigned long flags; bool abort; spin_lock_irqsave(&fod->flock, flags); abort = fod->abort; fod->writedataactive = false; spin_unlock_irqrestore(&fod->flock, flags); switch (fcpreq->op) { case NVMET_FCOP_WRITEDATA: if (__nvmet_fc_fod_op_abort(fod, abort)) return; if (fcpreq->fcp_error || fcpreq->transferred_length != fcpreq->transfer_length) { spin_lock_irqsave(&fod->flock, flags); fod->abort = true; spin_unlock_irqrestore(&fod->flock, flags); nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); return; } fod->offset += fcpreq->transferred_length; if (fod->offset != fod->req.transfer_len) { spin_lock_irqsave(&fod->flock, flags); fod->writedataactive = true; spin_unlock_irqrestore(&fod->flock, flags); /* transfer the next chunk */ nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA); return; } /* data transfer complete, resume with nvmet layer */ fod->req.execute(&fod->req); break; case NVMET_FCOP_READDATA: case NVMET_FCOP_READDATA_RSP: if (__nvmet_fc_fod_op_abort(fod, abort)) return; if (fcpreq->fcp_error || fcpreq->transferred_length != fcpreq->transfer_length) { nvmet_fc_abort_op(tgtport, fod); return; } /* success */ if (fcpreq->op == NVMET_FCOP_READDATA_RSP) { /* data no longer needed */ nvmet_fc_free_tgt_pgs(fod); nvmet_fc_free_fcp_iod(fod->queue, fod); return; } fod->offset += fcpreq->transferred_length; if (fod->offset != fod->req.transfer_len) { /* transfer the next chunk */ nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_READDATA); return; } /* data transfer complete, send response */ /* data no longer needed */ nvmet_fc_free_tgt_pgs(fod); nvmet_fc_xmt_fcp_rsp(tgtport, fod); break; case NVMET_FCOP_RSP: if (__nvmet_fc_fod_op_abort(fod, abort)) return; nvmet_fc_free_fcp_iod(fod->queue, fod); break; default: break; } } static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq) { struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; nvmet_fc_fod_op_done(fod); } /* * actual completion handler after execution by the nvmet layer */ static void __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_fcp_iod *fod, int status) { struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common; struct nvme_completion *cqe = &fod->rspiubuf.cqe; unsigned long flags; bool abort; spin_lock_irqsave(&fod->flock, flags); abort = fod->abort; spin_unlock_irqrestore(&fod->flock, flags); /* if we have a CQE, snoop the last sq_head value */ if (!status) fod->queue->sqhd = cqe->sq_head; if (abort) { nvmet_fc_abort_op(tgtport, fod); return; } /* if an error handling the cmd post initial parsing */ if (status) { /* fudge up a failed CQE status for our transport error */ memset(cqe, 0, sizeof(*cqe)); cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */ cqe->sq_id = cpu_to_le16(fod->queue->qid); cqe->command_id = sqe->command_id; cqe->status = cpu_to_le16(status); } else { /* * try to push the data even if the SQE status is non-zero. * There may be a status where data still was intended to * be moved */ if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) { /* push the data over before sending rsp */ nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_READDATA); return; } /* writes & no data - fall thru */ } /* data no longer needed */ nvmet_fc_free_tgt_pgs(fod); nvmet_fc_xmt_fcp_rsp(tgtport, fod); } static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req) { struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req); struct nvmet_fc_tgtport *tgtport = fod->tgtport; __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0); } /* * Actual processing routine for received FC-NVME I/O Requests from the LLD */ static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_fcp_iod *fod) { struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf; u32 xfrlen = be32_to_cpu(cmdiu->data_len); int ret; /* * Fused commands are currently not supported in the linux * implementation. * * As such, the implementation of the FC transport does not * look at the fused commands and order delivery to the upper * layer until we have both based on csn. */ fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done; if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) { fod->io_dir = NVMET_FCP_WRITE; if (!nvme_is_write(&cmdiu->sqe)) goto transport_error; } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) { fod->io_dir = NVMET_FCP_READ; if (nvme_is_write(&cmdiu->sqe)) goto transport_error; } else { fod->io_dir = NVMET_FCP_NODATA; if (xfrlen) goto transport_error; } fod->req.cmd = &fod->cmdiubuf.sqe; fod->req.cqe = &fod->rspiubuf.cqe; if (tgtport->pe) fod->req.port = tgtport->pe->port; /* clear any response payload */ memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf)); fod->data_sg = NULL; fod->data_sg_cnt = 0; ret = nvmet_req_init(&fod->req, &fod->queue->nvme_cq, &fod->queue->nvme_sq, &nvmet_fc_tgt_fcp_ops); if (!ret) { /* bad SQE content or invalid ctrl state */ /* nvmet layer has already called op done to send rsp. */ return; } fod->req.transfer_len = xfrlen; /* keep a running counter of tail position */ atomic_inc(&fod->queue->sqtail); if (fod->req.transfer_len) { ret = nvmet_fc_alloc_tgt_pgs(fod); if (ret) { nvmet_req_complete(&fod->req, ret); return; } } fod->req.sg = fod->data_sg; fod->req.sg_cnt = fod->data_sg_cnt; fod->offset = 0; if (fod->io_dir == NVMET_FCP_WRITE) { /* pull the data over before invoking nvmet layer */ nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA); return; } /* * Reads or no data: * * can invoke the nvmet_layer now. If read data, cmd completion will * push the data */ fod->req.execute(&fod->req); return; transport_error: nvmet_fc_abort_op(tgtport, fod); } /** * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD * upon the reception of a NVME FCP CMD IU. * * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc * layer for processing. * * The nvmet_fc layer allocates a local job structure (struct * nvmet_fc_fcp_iod) from the queue for the io and copies the * CMD IU buffer to the job structure. As such, on a successful * completion (returns 0), the LLDD may immediately free/reuse * the CMD IU buffer passed in the call. * * However, in some circumstances, due to the packetized nature of FC * and the api of the FC LLDD which may issue a hw command to send the * response, but the LLDD may not get the hw completion for that command * and upcall the nvmet_fc layer before a new command may be * asynchronously received - its possible for a command to be received * before the LLDD and nvmet_fc have recycled the job structure. It gives * the appearance of more commands received than fits in the sq. * To alleviate this scenario, a temporary queue is maintained in the * transport for pending LLDD requests waiting for a queue job structure. * In these "overrun" cases, a temporary queue element is allocated * the LLDD request and CMD iu buffer information remembered, and the * routine returns a -EOVERFLOW status. Subsequently, when a queue job * structure is freed, it is immediately reallocated for anything on the * pending request list. The LLDDs defer_rcv() callback is called, * informing the LLDD that it may reuse the CMD IU buffer, and the io * is then started normally with the transport. * * The LLDD, when receiving an -EOVERFLOW completion status, is to treat * the completion as successful but must not reuse the CMD IU buffer * until the LLDD's defer_rcv() callback has been called for the * corresponding struct nvmefc_tgt_fcp_req pointer. * * If there is any other condition in which an error occurs, the * transport will return a non-zero status indicating the error. * In all cases other than -EOVERFLOW, the transport has not accepted the * request and the LLDD should abort the exchange. * * @target_port: pointer to the (registered) target port the FCP CMD IU * was received on. * @fcpreq: pointer to a fcpreq request structure to be used to reference * the exchange corresponding to the FCP Exchange. * @cmdiubuf: pointer to the buffer containing the FCP CMD IU * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU */ int nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port, struct nvmefc_tgt_fcp_req *fcpreq, void *cmdiubuf, u32 cmdiubuf_len) { struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); struct nvme_fc_cmd_iu *cmdiu = cmdiubuf; struct nvmet_fc_tgt_queue *queue; struct nvmet_fc_fcp_iod *fod; struct nvmet_fc_defer_fcp_req *deferfcp; unsigned long flags; /* validate iu, so the connection id can be used to find the queue */ if ((cmdiubuf_len != sizeof(*cmdiu)) || (cmdiu->format_id != NVME_CMD_FORMAT_ID) || (cmdiu->fc_id != NVME_CMD_FC_ID) || (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4))) return -EIO; queue = nvmet_fc_find_target_queue(tgtport, be64_to_cpu(cmdiu->connection_id)); if (!queue) return -ENOTCONN; /* * note: reference taken by find_target_queue * After successful fod allocation, the fod will inherit the * ownership of that reference and will remove the reference * when the fod is freed. */ spin_lock_irqsave(&queue->qlock, flags); fod = nvmet_fc_alloc_fcp_iod(queue); if (fod) { spin_unlock_irqrestore(&queue->qlock, flags); fcpreq->nvmet_fc_private = fod; fod->fcpreq = fcpreq; memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len); nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq); return 0; } if (!tgtport->ops->defer_rcv) { spin_unlock_irqrestore(&queue->qlock, flags); /* release the queue lookup reference */ nvmet_fc_tgt_q_put(queue); return -ENOENT; } deferfcp = list_first_entry_or_null(&queue->avail_defer_list, struct nvmet_fc_defer_fcp_req, req_list); if (deferfcp) { /* Just re-use one that was previously allocated */ list_del(&deferfcp->req_list); } else { spin_unlock_irqrestore(&queue->qlock, flags); /* Now we need to dynamically allocate one */ deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL); if (!deferfcp) { /* release the queue lookup reference */ nvmet_fc_tgt_q_put(queue); return -ENOMEM; } spin_lock_irqsave(&queue->qlock, flags); } /* For now, use rspaddr / rsplen to save payload information */ fcpreq->rspaddr = cmdiubuf; fcpreq->rsplen = cmdiubuf_len; deferfcp->fcp_req = fcpreq; /* defer processing till a fod becomes available */ list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list); /* NOTE: the queue lookup reference is still valid */ spin_unlock_irqrestore(&queue->qlock, flags); return -EOVERFLOW; } EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req); /** * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD * upon the reception of an ABTS for a FCP command * * Notify the transport that an ABTS has been received for a FCP command * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The * LLDD believes the command is still being worked on * (template_ops->fcp_req_release() has not been called). * * The transport will wait for any outstanding work (an op to the LLDD, * which the lldd should complete with error due to the ABTS; or the * completion from the nvmet layer of the nvme command), then will * stop processing and call the nvmet_fc_rcv_fcp_req() callback to * return the i/o context to the LLDD. The LLDD may send the BA_ACC * to the ABTS either after return from this function (assuming any * outstanding op work has been terminated) or upon the callback being * called. * * @target_port: pointer to the (registered) target port the FCP CMD IU * was received on. * @fcpreq: pointer to the fcpreq request structure that corresponds * to the exchange that received the ABTS. */ void nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port, struct nvmefc_tgt_fcp_req *fcpreq) { struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; struct nvmet_fc_tgt_queue *queue; unsigned long flags; if (!fod || fod->fcpreq != fcpreq) /* job appears to have already completed, ignore abort */ return; queue = fod->queue; spin_lock_irqsave(&queue->qlock, flags); if (fod->active) { /* * mark as abort. The abort handler, invoked upon completion * of any work, will detect the aborted status and do the * callback. */ spin_lock(&fod->flock); fod->abort = true; fod->aborted = true; spin_unlock(&fod->flock); } spin_unlock_irqrestore(&queue->qlock, flags); } EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort); struct nvmet_fc_traddr { u64 nn; u64 pn; }; static int __nvme_fc_parse_u64(substring_t *sstr, u64 *val) { u64 token64; if (match_u64(sstr, &token64)) return -EINVAL; *val = token64; return 0; } /* * This routine validates and extracts the WWN's from the TRADDR string. * As kernel parsers need the 0x to determine number base, universally * build string to parse with 0x prefix before parsing name strings. */ static int nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen) { char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1]; substring_t wwn = { name, &name[sizeof(name)-1] }; int nnoffset, pnoffset; /* validate if string is one of the 2 allowed formats */ if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH && !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) && !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET], "pn-0x", NVME_FC_TRADDR_OXNNLEN)) { nnoffset = NVME_FC_TRADDR_OXNNLEN; pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET + NVME_FC_TRADDR_OXNNLEN; } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH && !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) && !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET], "pn-", NVME_FC_TRADDR_NNLEN))) { nnoffset = NVME_FC_TRADDR_NNLEN; pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN; } else goto out_einval; name[0] = '0'; name[1] = 'x'; name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0; memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN); if (__nvme_fc_parse_u64(&wwn, &traddr->nn)) goto out_einval; memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN); if (__nvme_fc_parse_u64(&wwn, &traddr->pn)) goto out_einval; return 0; out_einval: pr_warn("%s: bad traddr string\n", __func__); return -EINVAL; } static int nvmet_fc_add_port(struct nvmet_port *port) { struct nvmet_fc_tgtport *tgtport; struct nvmet_fc_port_entry *pe; struct nvmet_fc_traddr traddr = { 0L, 0L }; unsigned long flags; int ret; /* validate the address info */ if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) || (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC)) return -EINVAL; /* map the traddr address info to a target port */ ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr, sizeof(port->disc_addr.traddr)); if (ret) return ret; pe = kzalloc(sizeof(*pe), GFP_KERNEL); if (!pe) return -ENOMEM; ret = -ENXIO; spin_lock_irqsave(&nvmet_fc_tgtlock, flags); list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) { if ((tgtport->fc_target_port.node_name == traddr.nn) && (tgtport->fc_target_port.port_name == traddr.pn)) { /* a FC port can only be 1 nvmet port id */ if (!tgtport->pe) { nvmet_fc_portentry_bind(tgtport, pe, port); ret = 0; } else ret = -EALREADY; break; } } spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); if (ret) kfree(pe); return ret; } static void nvmet_fc_remove_port(struct nvmet_port *port) { struct nvmet_fc_port_entry *pe = port->priv; nvmet_fc_portentry_unbind(pe); kfree(pe); } static void nvmet_fc_discovery_chg(struct nvmet_port *port) { struct nvmet_fc_port_entry *pe = port->priv; struct nvmet_fc_tgtport *tgtport = pe->tgtport; if (tgtport && tgtport->ops->discovery_event) tgtport->ops->discovery_event(&tgtport->fc_target_port); } static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = { .owner = THIS_MODULE, .type = NVMF_TRTYPE_FC, .msdbd = 1, .add_port = nvmet_fc_add_port, .remove_port = nvmet_fc_remove_port, .queue_response = nvmet_fc_fcp_nvme_cmd_done, .delete_ctrl = nvmet_fc_delete_ctrl, .discovery_chg = nvmet_fc_discovery_chg, }; static int __init nvmet_fc_init_module(void) { return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops); } static void __exit nvmet_fc_exit_module(void) { /* sanity check - all lports should be removed */ if (!list_empty(&nvmet_fc_target_list)) pr_warn("%s: targetport list not empty\n", __func__); nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops); ida_destroy(&nvmet_fc_tgtport_cnt); } module_init(nvmet_fc_init_module); module_exit(nvmet_fc_exit_module); MODULE_LICENSE("GPL v2");
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1