Contributors: 5
Author Tokens Token Proportion Commits Commit Proportion
Matt Sickler 1196 98.84% 1 16.67%
Vandana BN 9 0.74% 1 16.67%
Sam Muhammed 2 0.17% 2 33.33%
Simon Sandström 2 0.17% 1 16.67%
Puranjay Mohan 1 0.08% 1 16.67%
Total 1210 6 


// SPDX-License-Identifier: GPL-2.0+
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/io.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/platform_device.h>
#include <linux/fs.h>
#include <linux/rwsem.h>
#include "kpc_dma_driver.h"

/**********  IRQ Handlers  **********/
static
irqreturn_t  ndd_irq_handler(int irq, void *dev_id)
{
	struct kpc_dma_device *ldev = (struct kpc_dma_device *)dev_id;

	if ((GetEngineControl(ldev) & ENG_CTL_IRQ_ACTIVE) || (ldev->desc_completed->MyDMAAddr != GetEngineCompletePtr(ldev)))
		schedule_work(&ldev->irq_work);

	return IRQ_HANDLED;
}

static
void  ndd_irq_worker(struct work_struct *ws)
{
	struct kpc_dma_descriptor *cur;
	struct kpc_dma_device *eng = container_of(ws, struct kpc_dma_device, irq_work);

	lock_engine(eng);

	if (GetEngineCompletePtr(eng) == 0)
		goto out;

	if (eng->desc_completed->MyDMAAddr == GetEngineCompletePtr(eng))
		goto out;

	cur = eng->desc_completed;
	do {
		cur = cur->Next;
		dev_dbg(&eng->pldev->dev, "Handling completed descriptor %p (acd = %p)\n", cur, cur->acd);
		BUG_ON(cur == eng->desc_next); // Ordering failure.

		if (cur->DescControlFlags & DMA_DESC_CTL_SOP) {
			eng->accumulated_bytes = 0;
			eng->accumulated_flags = 0;
		}

		eng->accumulated_bytes += cur->DescByteCount;
		if (cur->DescStatusFlags & DMA_DESC_STS_ERROR)
			eng->accumulated_flags |= ACD_FLAG_ENG_ACCUM_ERROR;

		if (cur->DescStatusFlags & DMA_DESC_STS_SHORT)
			eng->accumulated_flags |= ACD_FLAG_ENG_ACCUM_SHORT;

		if (cur->DescControlFlags & DMA_DESC_CTL_EOP) {
			if (cur->acd)
				transfer_complete_cb(cur->acd, eng->accumulated_bytes, eng->accumulated_flags | ACD_FLAG_DONE);
		}

		eng->desc_completed = cur;
	} while (cur->MyDMAAddr != GetEngineCompletePtr(eng));

 out:
	SetClearEngineControl(eng, ENG_CTL_IRQ_ACTIVE, 0);

	unlock_engine(eng);
}

/**********  DMA Engine Init/Teardown  **********/
void  start_dma_engine(struct kpc_dma_device *eng)
{
	eng->desc_next       = eng->desc_pool_first;
	eng->desc_completed  = eng->desc_pool_last;

	// Setup the engine pointer registers
	SetEngineNextPtr(eng, eng->desc_pool_first);
	SetEngineSWPtr(eng, eng->desc_pool_first);
	ClearEngineCompletePtr(eng);

	WriteEngineControl(eng, ENG_CTL_DMA_ENABLE | ENG_CTL_IRQ_ENABLE);
}

int  setup_dma_engine(struct kpc_dma_device *eng, u32 desc_cnt)
{
	u32 caps;
	struct kpc_dma_descriptor *cur;
	struct kpc_dma_descriptor *next;
	dma_addr_t next_handle;
	dma_addr_t head_handle;
	unsigned int i;
	int rv;

	caps = GetEngineCapabilities(eng);

	if (WARN(!(caps & ENG_CAP_PRESENT), "%s() called for DMA Engine at %p which isn't present in hardware!\n", __func__, eng))
		return -ENXIO;

	if (caps & ENG_CAP_DIRECTION)
		eng->dir = DMA_FROM_DEVICE;
	else
		eng->dir = DMA_TO_DEVICE;

	eng->desc_pool_cnt = desc_cnt;
	eng->desc_pool = dma_pool_create("KPC DMA Descriptors", &eng->pldev->dev, sizeof(struct kpc_dma_descriptor), DMA_DESC_ALIGNMENT, 4096);

	eng->desc_pool_first = dma_pool_alloc(eng->desc_pool, GFP_KERNEL | GFP_DMA, &head_handle);
	if (!eng->desc_pool_first) {
		dev_err(&eng->pldev->dev, "%s: couldn't allocate desc_pool_first!\n", __func__);
		dma_pool_destroy(eng->desc_pool);
		return -ENOMEM;
	}

	eng->desc_pool_first->MyDMAAddr = head_handle;
	clear_desc(eng->desc_pool_first);

	cur = eng->desc_pool_first;
	for (i = 1 ; i < eng->desc_pool_cnt ; i++) {
		next = dma_pool_alloc(eng->desc_pool, GFP_KERNEL | GFP_DMA, &next_handle);
		if (!next)
			goto done_alloc;

		clear_desc(next);
		next->MyDMAAddr = next_handle;

		cur->DescNextDescPtr = next_handle;
		cur->Next = next;
		cur = next;
	}

 done_alloc:
	// Link the last descriptor back to the first, so it's a circular linked list
	cur->Next = eng->desc_pool_first;
	cur->DescNextDescPtr = eng->desc_pool_first->MyDMAAddr;

	eng->desc_pool_last = cur;
	eng->desc_completed = eng->desc_pool_last;

	// Setup work queue
	INIT_WORK(&eng->irq_work, ndd_irq_worker);

	// Grab IRQ line
	rv = request_irq(eng->irq, ndd_irq_handler, IRQF_SHARED, KP_DRIVER_NAME_DMA_CONTROLLER, eng);
	if (rv) {
		dev_err(&eng->pldev->dev, "%s: failed to request_irq: %d\n", __func__, rv);
		return rv;
	}

	// Turn on the engine!
	start_dma_engine(eng);
	unlock_engine(eng);

	return 0;
}

void  stop_dma_engine(struct kpc_dma_device *eng)
{
	unsigned long timeout;

	// Disable the descriptor engine
	WriteEngineControl(eng, 0);

	// Wait for descriptor engine to finish current operaion
	timeout = jiffies + (HZ / 2);
	while (GetEngineControl(eng) & ENG_CTL_DMA_RUNNING) {
		if (time_after(jiffies, timeout)) {
			dev_crit(&eng->pldev->dev, "DMA_RUNNING still asserted!\n");
			break;
		}
	}

	// Request a reset
	WriteEngineControl(eng, ENG_CTL_DMA_RESET_REQUEST);

	// Wait for reset request to be processed
	timeout = jiffies + (HZ / 2);
	while (GetEngineControl(eng) & (ENG_CTL_DMA_RUNNING | ENG_CTL_DMA_RESET_REQUEST)) {
		if (time_after(jiffies, timeout)) {
			dev_crit(&eng->pldev->dev, "ENG_CTL_DMA_RESET_REQUEST still asserted!\n");
			break;
		}
	}

	// Request a reset
	WriteEngineControl(eng, ENG_CTL_DMA_RESET);

	// And wait for reset to complete
	timeout = jiffies + (HZ / 2);
	while (GetEngineControl(eng) & ENG_CTL_DMA_RESET) {
		if (time_after(jiffies, timeout)) {
			dev_crit(&eng->pldev->dev, "DMA_RESET still asserted!\n");
			break;
		}
	}

	// Clear any persistent bits just to make sure there is no residue from the reset
	SetClearEngineControl(eng, (ENG_CTL_IRQ_ACTIVE | ENG_CTL_DESC_COMPLETE | ENG_CTL_DESC_ALIGN_ERR | ENG_CTL_DESC_FETCH_ERR | ENG_CTL_SW_ABORT_ERR | ENG_CTL_DESC_CHAIN_END | ENG_CTL_DMA_WAITING_PERSIST), 0);

	// Reset performance counters

	// Completely disable the engine
	WriteEngineControl(eng, 0);
}

void  destroy_dma_engine(struct kpc_dma_device *eng)
{
	struct kpc_dma_descriptor *cur;
	dma_addr_t cur_handle;
	unsigned int i;

	stop_dma_engine(eng);

	cur = eng->desc_pool_first;
	cur_handle = eng->desc_pool_first->MyDMAAddr;

	for (i = 0 ; i < eng->desc_pool_cnt ; i++) {
		struct kpc_dma_descriptor *next = cur->Next;
		dma_addr_t next_handle = cur->DescNextDescPtr;

		dma_pool_free(eng->desc_pool, cur, cur_handle);
		cur_handle = next_handle;
		cur = next;
	}

	dma_pool_destroy(eng->desc_pool);

	free_irq(eng->irq, eng);
}

/**********  Helper Functions  **********/
int  count_descriptors_available(struct kpc_dma_device *eng)
{
	u32 count = 0;
	struct kpc_dma_descriptor *cur = eng->desc_next;

	while (cur != eng->desc_completed) {
		BUG_ON(!cur);
		count++;
		cur = cur->Next;
	}
	return count;
}

void  clear_desc(struct kpc_dma_descriptor *desc)
{
	if (!desc)
		return;
	desc->DescByteCount         = 0;
	desc->DescStatusErrorFlags  = 0;
	desc->DescStatusFlags       = 0;
	desc->DescUserControlLS     = 0;
	desc->DescUserControlMS     = 0;
	desc->DescCardAddrLS        = 0;
	desc->DescBufferByteCount   = 0;
	desc->DescCardAddrMS        = 0;
	desc->DescControlFlags      = 0;
	desc->DescSystemAddrLS      = 0;
	desc->DescSystemAddrMS      = 0;
	desc->acd = NULL;
}