Author | Tokens | Token Proportion | Commits | Commit Proportion |
---|---|---|---|---|
Gary R Hook | 3212 | 93.07% | 7 | 31.82% |
Tom Lendacky | 150 | 4.35% | 2 | 9.09% |
Koba Ko | 30 | 0.87% | 2 | 9.09% |
Dāvis Mosāns | 27 | 0.78% | 1 | 4.55% |
amd at <amd@sosxen2.amd.com> | 8 | 0.23% | 2 | 9.09% |
Quentin Lambert | 8 | 0.23% | 1 | 4.55% |
Mark Salter | 8 | 0.23% | 1 | 4.55% |
Björn Helgaas | 3 | 0.09% | 1 | 4.55% |
Wei Yongjun | 2 | 0.06% | 2 | 9.09% |
Thomas Gleixner | 1 | 0.03% | 1 | 4.55% |
Baokun Li | 1 | 0.03% | 1 | 4.55% |
Colin Ian King | 1 | 0.03% | 1 | 4.55% |
Total | 3451 | 22 |
// SPDX-License-Identifier: GPL-2.0-only /* * AMD Cryptographic Coprocessor (CCP) driver * * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. * * Author: Gary R Hook <gary.hook@amd.com> */ #include <linux/module.h> #include <linux/kernel.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/spinlock.h> #include <linux/mutex.h> #include <linux/ccp.h> #include "ccp-dev.h" #include "../../dma/dmaengine.h" #define CCP_DMA_WIDTH(_mask) \ ({ \ u64 mask = _mask + 1; \ (mask == 0) ? 64 : fls64(mask); \ }) /* The CCP as a DMA provider can be configured for public or private * channels. Default is specified in the vdata for the device (PCI ID). * This module parameter will override for all channels on all devices: * dma_chan_attr = 0x2 to force all channels public * = 0x1 to force all channels private * = 0x0 to defer to the vdata setting * = any other value: warning, revert to 0x0 */ static unsigned int dma_chan_attr = CCP_DMA_DFLT; module_param(dma_chan_attr, uint, 0444); MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); static unsigned int dmaengine = 1; module_param(dmaengine, uint, 0444); MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)"); static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) { switch (dma_chan_attr) { case CCP_DMA_DFLT: return ccp->vdata->dma_chan_attr; case CCP_DMA_PRIV: return DMA_PRIVATE; case CCP_DMA_PUB: return 0; default: dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", dma_chan_attr); return ccp->vdata->dma_chan_attr; } } static void ccp_free_cmd_resources(struct ccp_device *ccp, struct list_head *list) { struct ccp_dma_cmd *cmd, *ctmp; list_for_each_entry_safe(cmd, ctmp, list, entry) { list_del(&cmd->entry); kmem_cache_free(ccp->dma_cmd_cache, cmd); } } static void ccp_free_desc_resources(struct ccp_device *ccp, struct list_head *list) { struct ccp_dma_desc *desc, *dtmp; list_for_each_entry_safe(desc, dtmp, list, entry) { ccp_free_cmd_resources(ccp, &desc->active); ccp_free_cmd_resources(ccp, &desc->pending); list_del(&desc->entry); kmem_cache_free(ccp->dma_desc_cache, desc); } } static void ccp_free_chan_resources(struct dma_chan *dma_chan) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); unsigned long flags; dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); spin_lock_irqsave(&chan->lock, flags); ccp_free_desc_resources(chan->ccp, &chan->complete); ccp_free_desc_resources(chan->ccp, &chan->active); ccp_free_desc_resources(chan->ccp, &chan->pending); ccp_free_desc_resources(chan->ccp, &chan->created); spin_unlock_irqrestore(&chan->lock, flags); } static void ccp_cleanup_desc_resources(struct ccp_device *ccp, struct list_head *list) { struct ccp_dma_desc *desc, *dtmp; list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { if (!async_tx_test_ack(&desc->tx_desc)) continue; dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); ccp_free_cmd_resources(ccp, &desc->active); ccp_free_cmd_resources(ccp, &desc->pending); list_del(&desc->entry); kmem_cache_free(ccp->dma_desc_cache, desc); } } static void ccp_do_cleanup(unsigned long data) { struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; unsigned long flags; dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, dma_chan_name(&chan->dma_chan)); spin_lock_irqsave(&chan->lock, flags); ccp_cleanup_desc_resources(chan->ccp, &chan->complete); spin_unlock_irqrestore(&chan->lock, flags); } static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) { struct ccp_dma_cmd *cmd; int ret; cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); list_move(&cmd->entry, &desc->active); dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, desc->tx_desc.cookie, cmd); ret = ccp_enqueue_cmd(&cmd->ccp_cmd); if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) return 0; dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, ret, desc->tx_desc.cookie, cmd); return ret; } static void ccp_free_active_cmd(struct ccp_dma_desc *desc) { struct ccp_dma_cmd *cmd; cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, entry); if (!cmd) return; dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", __func__, desc->tx_desc.cookie, cmd); list_del(&cmd->entry); kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); } static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, struct ccp_dma_desc *desc) { /* Move current DMA descriptor to the complete list */ if (desc) list_move(&desc->entry, &chan->complete); /* Get the next DMA descriptor on the active list */ desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, entry); return desc; } static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, struct ccp_dma_desc *desc) { struct dma_async_tx_descriptor *tx_desc; unsigned long flags; /* Loop over descriptors until one is found with commands */ do { if (desc) { /* Remove the DMA command from the list and free it */ ccp_free_active_cmd(desc); if (!list_empty(&desc->pending)) { /* No errors, keep going */ if (desc->status != DMA_ERROR) return desc; /* Error, free remaining commands and move on */ ccp_free_cmd_resources(desc->ccp, &desc->pending); } tx_desc = &desc->tx_desc; } else { tx_desc = NULL; } spin_lock_irqsave(&chan->lock, flags); if (desc) { if (desc->status != DMA_ERROR) desc->status = DMA_COMPLETE; dev_dbg(desc->ccp->dev, "%s - tx %d complete, status=%u\n", __func__, desc->tx_desc.cookie, desc->status); dma_cookie_complete(tx_desc); dma_descriptor_unmap(tx_desc); } desc = __ccp_next_dma_desc(chan, desc); spin_unlock_irqrestore(&chan->lock, flags); if (tx_desc) { dmaengine_desc_get_callback_invoke(tx_desc, NULL); dma_run_dependencies(tx_desc); } } while (desc); return NULL; } static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) { struct ccp_dma_desc *desc; if (list_empty(&chan->pending)) return NULL; desc = list_empty(&chan->active) ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) : NULL; list_splice_tail_init(&chan->pending, &chan->active); return desc; } static void ccp_cmd_callback(void *data, int err) { struct ccp_dma_desc *desc = data; struct ccp_dma_chan *chan; int ret; if (err == -EINPROGRESS) return; chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, dma_chan); dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", __func__, desc->tx_desc.cookie, err); if (err) desc->status = DMA_ERROR; while (true) { /* Check for DMA descriptor completion */ desc = ccp_handle_active_desc(chan, desc); /* Don't submit cmd if no descriptor or DMA is paused */ if (!desc || (chan->status == DMA_PAUSED)) break; ret = ccp_issue_next_cmd(desc); if (!ret) break; desc->status = DMA_ERROR; } tasklet_schedule(&chan->cleanup_tasklet); } static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) { struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, tx_desc); struct ccp_dma_chan *chan; dma_cookie_t cookie; unsigned long flags; chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); spin_lock_irqsave(&chan->lock, flags); cookie = dma_cookie_assign(tx_desc); list_move_tail(&desc->entry, &chan->pending); spin_unlock_irqrestore(&chan->lock, flags); dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", __func__, cookie); return cookie; } static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) { struct ccp_dma_cmd *cmd; cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); if (cmd) memset(cmd, 0, sizeof(*cmd)); return cmd; } static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, unsigned long flags) { struct ccp_dma_desc *desc; desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); if (!desc) return NULL; dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); desc->tx_desc.flags = flags; desc->tx_desc.tx_submit = ccp_tx_submit; desc->ccp = chan->ccp; INIT_LIST_HEAD(&desc->entry); INIT_LIST_HEAD(&desc->pending); INIT_LIST_HEAD(&desc->active); desc->status = DMA_IN_PROGRESS; return desc; } static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, struct scatterlist *dst_sg, unsigned int dst_nents, struct scatterlist *src_sg, unsigned int src_nents, unsigned long flags) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); struct ccp_device *ccp = chan->ccp; struct ccp_dma_desc *desc; struct ccp_dma_cmd *cmd; struct ccp_cmd *ccp_cmd; struct ccp_passthru_nomap_engine *ccp_pt; unsigned int src_offset, src_len; unsigned int dst_offset, dst_len; unsigned int len; unsigned long sflags; size_t total_len; if (!dst_sg || !src_sg) return NULL; if (!dst_nents || !src_nents) return NULL; desc = ccp_alloc_dma_desc(chan, flags); if (!desc) return NULL; total_len = 0; src_len = sg_dma_len(src_sg); src_offset = 0; dst_len = sg_dma_len(dst_sg); dst_offset = 0; while (true) { if (!src_len) { src_nents--; if (!src_nents) break; src_sg = sg_next(src_sg); if (!src_sg) break; src_len = sg_dma_len(src_sg); src_offset = 0; continue; } if (!dst_len) { dst_nents--; if (!dst_nents) break; dst_sg = sg_next(dst_sg); if (!dst_sg) break; dst_len = sg_dma_len(dst_sg); dst_offset = 0; continue; } len = min(dst_len, src_len); cmd = ccp_alloc_dma_cmd(chan); if (!cmd) goto err; ccp_cmd = &cmd->ccp_cmd; ccp_cmd->ccp = chan->ccp; ccp_pt = &ccp_cmd->u.passthru_nomap; ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; ccp_cmd->engine = CCP_ENGINE_PASSTHRU; ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; ccp_pt->src_len = len; ccp_pt->final = 1; ccp_cmd->callback = ccp_cmd_callback; ccp_cmd->data = desc; list_add_tail(&cmd->entry, &desc->pending); dev_dbg(ccp->dev, "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, cmd, &ccp_pt->src_dma, &ccp_pt->dst_dma, ccp_pt->src_len); total_len += len; src_len -= len; src_offset += len; dst_len -= len; dst_offset += len; } desc->len = total_len; if (list_empty(&desc->pending)) goto err; dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); spin_lock_irqsave(&chan->lock, sflags); list_add_tail(&desc->entry, &chan->created); spin_unlock_irqrestore(&chan->lock, sflags); return desc; err: ccp_free_cmd_resources(ccp, &desc->pending); kmem_cache_free(ccp->dma_desc_cache, desc); return NULL; } static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, unsigned long flags) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); struct ccp_dma_desc *desc; struct scatterlist dst_sg, src_sg; dev_dbg(chan->ccp->dev, "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", __func__, &src, &dst, len, flags); sg_init_table(&dst_sg, 1); sg_dma_address(&dst_sg) = dst; sg_dma_len(&dst_sg) = len; sg_init_table(&src_sg, 1); sg_dma_address(&src_sg) = src; sg_dma_len(&src_sg) = len; desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); if (!desc) return NULL; return &desc->tx_desc; } static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( struct dma_chan *dma_chan, unsigned long flags) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); struct ccp_dma_desc *desc; desc = ccp_alloc_dma_desc(chan, flags); if (!desc) return NULL; return &desc->tx_desc; } static void ccp_issue_pending(struct dma_chan *dma_chan) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); struct ccp_dma_desc *desc; unsigned long flags; dev_dbg(chan->ccp->dev, "%s\n", __func__); spin_lock_irqsave(&chan->lock, flags); desc = __ccp_pending_to_active(chan); spin_unlock_irqrestore(&chan->lock, flags); /* If there was nothing active, start processing */ if (desc) ccp_cmd_callback(desc, 0); } static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, dma_cookie_t cookie, struct dma_tx_state *state) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); struct ccp_dma_desc *desc; enum dma_status ret; unsigned long flags; if (chan->status == DMA_PAUSED) { ret = DMA_PAUSED; goto out; } ret = dma_cookie_status(dma_chan, cookie, state); if (ret == DMA_COMPLETE) { spin_lock_irqsave(&chan->lock, flags); /* Get status from complete chain, if still there */ list_for_each_entry(desc, &chan->complete, entry) { if (desc->tx_desc.cookie != cookie) continue; ret = desc->status; break; } spin_unlock_irqrestore(&chan->lock, flags); } out: dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); return ret; } static int ccp_pause(struct dma_chan *dma_chan) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); chan->status = DMA_PAUSED; /*TODO: Wait for active DMA to complete before returning? */ return 0; } static int ccp_resume(struct dma_chan *dma_chan) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); struct ccp_dma_desc *desc; unsigned long flags; spin_lock_irqsave(&chan->lock, flags); desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, entry); spin_unlock_irqrestore(&chan->lock, flags); /* Indicate the channel is running again */ chan->status = DMA_IN_PROGRESS; /* If there was something active, re-start */ if (desc) ccp_cmd_callback(desc, 0); return 0; } static int ccp_terminate_all(struct dma_chan *dma_chan) { struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, dma_chan); unsigned long flags; dev_dbg(chan->ccp->dev, "%s\n", __func__); /*TODO: Wait for active DMA to complete before continuing */ spin_lock_irqsave(&chan->lock, flags); /*TODO: Purge the complete list? */ ccp_free_desc_resources(chan->ccp, &chan->active); ccp_free_desc_resources(chan->ccp, &chan->pending); ccp_free_desc_resources(chan->ccp, &chan->created); spin_unlock_irqrestore(&chan->lock, flags); return 0; } static void ccp_dma_release(struct ccp_device *ccp) { struct ccp_dma_chan *chan; struct dma_chan *dma_chan; unsigned int i; for (i = 0; i < ccp->cmd_q_count; i++) { chan = ccp->ccp_dma_chan + i; dma_chan = &chan->dma_chan; tasklet_kill(&chan->cleanup_tasklet); list_del_rcu(&dma_chan->device_node); } } static void ccp_dma_release_channels(struct ccp_device *ccp) { struct ccp_dma_chan *chan; struct dma_chan *dma_chan; unsigned int i; for (i = 0; i < ccp->cmd_q_count; i++) { chan = ccp->ccp_dma_chan + i; dma_chan = &chan->dma_chan; if (dma_chan->client_count) dma_release_channel(dma_chan); } } int ccp_dmaengine_register(struct ccp_device *ccp) { struct ccp_dma_chan *chan; struct dma_device *dma_dev = &ccp->dma_dev; struct dma_chan *dma_chan; char *dma_cmd_cache_name; char *dma_desc_cache_name; unsigned int i; int ret; if (!dmaengine) return 0; ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, sizeof(*(ccp->ccp_dma_chan)), GFP_KERNEL); if (!ccp->ccp_dma_chan) return -ENOMEM; dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, "%s-dmaengine-cmd-cache", ccp->name); if (!dma_cmd_cache_name) return -ENOMEM; ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, sizeof(struct ccp_dma_cmd), sizeof(void *), SLAB_HWCACHE_ALIGN, NULL); if (!ccp->dma_cmd_cache) return -ENOMEM; dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, "%s-dmaengine-desc-cache", ccp->name); if (!dma_desc_cache_name) { ret = -ENOMEM; goto err_cache; } ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, sizeof(struct ccp_dma_desc), sizeof(void *), SLAB_HWCACHE_ALIGN, NULL); if (!ccp->dma_desc_cache) { ret = -ENOMEM; goto err_cache; } dma_dev->dev = ccp->dev; dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); dma_dev->directions = DMA_MEM_TO_MEM; dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); /* The DMA channels for this device can be set to public or private, * and overridden by the module parameter dma_chan_attr. * Default: according to the value in vdata (dma_chan_attr=0) * dma_chan_attr=0x1: all channels private (override vdata) * dma_chan_attr=0x2: all channels public (override vdata) */ if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); INIT_LIST_HEAD(&dma_dev->channels); for (i = 0; i < ccp->cmd_q_count; i++) { chan = ccp->ccp_dma_chan + i; dma_chan = &chan->dma_chan; chan->ccp = ccp; spin_lock_init(&chan->lock); INIT_LIST_HEAD(&chan->created); INIT_LIST_HEAD(&chan->pending); INIT_LIST_HEAD(&chan->active); INIT_LIST_HEAD(&chan->complete); tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, (unsigned long)chan); dma_chan->device = dma_dev; dma_cookie_init(dma_chan); list_add_tail(&dma_chan->device_node, &dma_dev->channels); } dma_dev->device_free_chan_resources = ccp_free_chan_resources; dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; dma_dev->device_issue_pending = ccp_issue_pending; dma_dev->device_tx_status = ccp_tx_status; dma_dev->device_pause = ccp_pause; dma_dev->device_resume = ccp_resume; dma_dev->device_terminate_all = ccp_terminate_all; ret = dma_async_device_register(dma_dev); if (ret) goto err_reg; return 0; err_reg: ccp_dma_release(ccp); kmem_cache_destroy(ccp->dma_desc_cache); err_cache: kmem_cache_destroy(ccp->dma_cmd_cache); return ret; } void ccp_dmaengine_unregister(struct ccp_device *ccp) { struct dma_device *dma_dev = &ccp->dma_dev; if (!dmaengine) return; ccp_dma_release_channels(ccp); dma_async_device_unregister(dma_dev); ccp_dma_release(ccp); kmem_cache_destroy(ccp->dma_desc_cache); kmem_cache_destroy(ccp->dma_cmd_cache); }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1