Release 4.11 drivers/dma/dmaengine.c

/*
 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * The full GNU General Public License is included in this distribution in the
 * file called COPYING.
 */

/*
 * This code implements the DMA subsystem. It provides a HW-neutral interface
 * for other kernel code to use asynchronous memory copy capabilities,
 * if present, and allows different HW DMA drivers to register as providing
 * this capability.
 *
 * Due to the fact we are accelerating what is already a relatively fast
 * operation, the code goes to great lengths to avoid additional overhead,
 * such as locking.
 *
 * LOCKING:
 *
 * The subsystem keeps a global list of dma_device structs it is protected by a
 * mutex, dma_list_mutex.
 *
 * A subsystem can get access to a channel by calling dmaengine_get() followed
 * by dma_find_channel(), or if it has need for an exclusive channel it can call
 * dma_request_channel().  Once a channel is allocated a reference is taken
 * against its corresponding driver to disable removal.
 *
 * Each device has a channels list, which runs unlocked but is never modified
 * once the device is registered, it's just setup by the driver.
 *
 * See Documentation/dmaengine.txt for more details
 */


#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/platform_device.h>
#include <linux/dma-mapping.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/hardirq.h>
#include <linux/spinlock.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/mutex.h>
#include <linux/jiffies.h>
#include <linux/rculist.h>
#include <linux/idr.h>
#include <linux/slab.h>
#include <linux/acpi.h>
#include <linux/acpi_dma.h>
#include <linux/of_dma.h>
#include <linux/mempool.h>

static DEFINE_MUTEX(dma_list_mutex);
static DEFINE_IDA(dma_ida);
static LIST_HEAD(dma_device_list);

static long dmaengine_ref_count;

/* --- sysfs implementation --- */

/**
 * dev_to_dma_chan - convert a device pointer to the its sysfs container object
 * @dev - device node
 *
 * Must be called under dma_list_mutex
 */


static struct dma_chan *dev_to_dma_chan(struct device *dev)
{
	struct dma_chan_dev *chan_dev;

	chan_dev = container_of(dev, typeof(*chan_dev), device);
	return chan_dev->chan;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
38
100.00%
1
100.00%
Total
38
100.00%
1
100.00%



static ssize_t memcpy_count_show(struct device *dev,
				 struct device_attribute *attr, char *buf)
{
	struct dma_chan *chan;
	unsigned long count = 0;
	int i;
	int err;

	mutex_lock(&dma_list_mutex);
	chan = dev_to_dma_chan(dev);
	if (chan) {
		for_each_possible_cpu(i)
			count += per_cpu_ptr(chan->local, i)->memcpy_count;
		err = sprintf(buf, "%lu\n", count);
	} else
		err = -ENODEV;
	mutex_unlock(&dma_list_mutex);

	return err;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Christopher Leech
51
51.52%
1
20.00%
Dan J Williams
39
39.39%
1
20.00%
Tony Jones
7
7.07%
1
20.00%
Andrew Morton
1
1.01%
1
20.00%
Greg Kroah-Hartman
1
1.01%
1
20.00%
Total
99
100.00%
5
100.00%

static DEVICE_ATTR_RO(memcpy_count);



static ssize_t bytes_transferred_show(struct device *dev,
				      struct device_attribute *attr, char *buf)
{
	struct dma_chan *chan;
	unsigned long count = 0;
	int i;
	int err;

	mutex_lock(&dma_list_mutex);
	chan = dev_to_dma_chan(dev);
	if (chan) {
		for_each_possible_cpu(i)
			count += per_cpu_ptr(chan->local, i)->bytes_transferred;
		err = sprintf(buf, "%lu\n", count);
	} else
		err = -ENODEV;
	mutex_unlock(&dma_list_mutex);

	return err;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Christopher Leech
51
51.52%
1
20.00%
Dan J Williams
39
39.39%
1
20.00%
Tony Jones
7
7.07%
1
20.00%
Andrew Morton
1
1.01%
1
20.00%
Greg Kroah-Hartman
1
1.01%
1
20.00%
Total
99
100.00%
5
100.00%

static DEVICE_ATTR_RO(bytes_transferred);



static ssize_t in_use_show(struct device *dev, struct device_attribute *attr,
			   char *buf)
{
	struct dma_chan *chan;
	int err;

	mutex_lock(&dma_list_mutex);
	chan = dev_to_dma_chan(dev);
	if (chan)
		err = sprintf(buf, "%d\n", chan->client_count);
	else
		err = -ENODEV;
	mutex_unlock(&dma_list_mutex);

	return err;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
38
52.05%
2
40.00%
Christopher Leech
26
35.62%
1
20.00%
Tony Jones
8
10.96%
1
20.00%
Greg Kroah-Hartman
1
1.37%
1
20.00%
Total
73
100.00%
5
100.00%

static DEVICE_ATTR_RO(in_use);


static struct attribute *dma_dev_attrs[] = {
	&dev_attr_memcpy_count.attr,
	&dev_attr_bytes_transferred.attr,
	&dev_attr_in_use.attr,
	NULL,
};

ATTRIBUTE_GROUPS(dma_dev);



static void chan_dev_release(struct device *dev)
{
	struct dma_chan_dev *chan_dev;

	chan_dev = container_of(dev, typeof(*chan_dev), device);
	if (atomic_dec_and_test(chan_dev->idr_ref)) {
		mutex_lock(&dma_list_mutex);
		ida_remove(&dma_ida, chan_dev->dev_id);
		mutex_unlock(&dma_list_mutex);
		kfree(chan_dev->idr_ref);
	}
	kfree(chan_dev);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
74
97.37%
2
66.67%
Matthew Wilcox
2
2.63%
1
33.33%
Total
76
100.00%
3
100.00%


static struct class dma_devclass = {
	.name		= "dma",
	.dev_groups	= dma_dev_groups,
	.dev_release	= chan_dev_release,
};

/* --- client and device registration --- */


#define dma_device_satisfies_mask(device, mask) \
	__dma_device_satisfies_mask((device), &(mask))

static int

__dma_device_satisfies_mask(struct dma_device *device,
			    const dma_cap_mask_t *want)
{
	dma_cap_mask_t has;

	bitmap_and(has.bits, want->bits, device->cap_mask.bits,
		DMA_TX_TYPE_END);
	return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
48
92.31%
2
50.00%
Christopher Leech
3
5.77%
1
25.00%
Lars-Peter Clausen
1
1.92%
1
25.00%
Total
52
100.00%
4
100.00%



static struct module *dma_chan_to_owner(struct dma_chan *chan)
{
	return chan->device->dev->driver->owner;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
24
100.00%
1
100.00%
Total
24
100.00%
1
100.00%

/**
 * balance_ref_count - catch up the channel reference count
 * @chan - channel to balance ->client_count versus dmaengine_ref_count
 *
 * balance_ref_count must be called under dma_list_mutex
 */


static void balance_ref_count(struct dma_chan *chan)
{
	struct module *owner = dma_chan_to_owner(chan);

	while (chan->client_count < dmaengine_ref_count) {
		__module_get(owner);
		chan->client_count++;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
41
100.00%
1
100.00%
Total
41
100.00%
1
100.00%

/**
 * dma_chan_get - try to grab a dma channel's parent driver module
 * @chan - channel to grab
 *
 * Must be called under dma_list_mutex
 */


static int dma_chan_get(struct dma_chan *chan)
{
	struct module *owner = dma_chan_to_owner(chan);
	int ret;

	/* The channel is already in use, update client count */
	if (chan->client_count) {
		__module_get(owner);
		goto out;
	}

	if (!try_module_get(owner))
		return -ENODEV;

	/* allocate upon first client reference */
	if (chan->device->device_alloc_chan_resources) {
		ret = chan->device->device_alloc_chan_resources(chan);
		if (ret < 0)
			goto err_out;
	}

	if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
		balance_ref_count(chan);

out:
	chan->client_count++;
	return 0;

err_out:
	module_put(owner);
	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
76
61.79%
2
50.00%
Maxime Ripard
47
38.21%
2
50.00%
Total
123
100.00%
4
100.00%

/**
 * dma_chan_put - drop a reference to a dma channel's parent driver module
 * @chan - channel to release
 *
 * Must be called under dma_list_mutex
 */


static void dma_chan_put(struct dma_chan *chan)
{
	/* This channel is not in use, bail out */
	if (!chan->client_count)
		return;

	chan->client_count--;
	module_put(dma_chan_to_owner(chan));

	/* This channel is not in use anymore, free it */
	if (!chan->client_count && chan->device->device_free_chan_resources) {
		/* Make sure all operations have completed */
		dmaengine_synchronize(chan);
		chan->device->device_free_chan_resources(chan);
	}

	/* If the channel is used via a DMA request router, free the mapping */
	if (chan->router && chan->router->route_free) {
		chan->router->route_free(chan->router->dev, chan->route_data);
		chan->router = NULL;
		chan->route_data = NULL;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
47
43.52%
1
25.00%
Peter Ujfalusi
44
40.74%
1
25.00%
Maxime Ripard
9
8.33%
1
25.00%
Lars-Peter Clausen
8
7.41%
1
25.00%
Total
108
100.00%
4
100.00%



enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
{
	enum dma_status status;
	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);

	dma_async_issue_pending(chan);
	do {
		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
			dev_err(chan->device->dev, "%s: timeout!\n", __func__);
			return DMA_ERROR;
		}
		if (status != DMA_IN_PROGRESS)
			break;
		cpu_relax();
	} while (1);

	return status;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
74
77.89%
1
25.00%
Bartlomiej Zolnierkiewicz
11
11.58%
1
25.00%
Jarkko Nikula
7
7.37%
1
25.00%
Joe Perches
3
3.16%
1
25.00%
Total
95
100.00%
4
100.00%


EXPORT_SYMBOL(dma_sync_wait);

/**
 * dma_cap_mask_all - enable iteration over all operation types
 */

static dma_cap_mask_t dma_cap_mask_all;

/**
 * dma_chan_tbl_ent - tracks channel allocations per core/operation
 * @chan - associated channel for this entry
 */

struct dma_chan_tbl_ent {
	
struct dma_chan *chan;
};

/**
 * channel_table - percpu lookup table for memory-to-memory offload providers
 */

static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];



static int __init dma_channel_table_init(void)
{
	enum dma_transaction_type cap;
	int err = 0;

	bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);

	/* 'interrupt', 'private', and 'slave' are channel capabilities,
         * but are not associated with an operation so they do not need
         * an entry in the channel_table
         */
	clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
	clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits);
	clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);

	for_each_dma_cap_mask(cap, dma_cap_mask_all) {
		channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
		if (!channel_table[cap]) {
			err = -ENOMEM;
			break;
		}
	}

	if (err) {
		pr_err("initialization failure\n");
		for_each_dma_cap_mask(cap, dma_cap_mask_all)
			free_percpu(channel_table[cap]);
	}

	return err;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
117
99.15%
2
66.67%
Joe Perches
1
0.85%
1
33.33%
Total
118
100.00%
3
100.00%


arch_initcall(dma_channel_table_init);

/**
 * dma_find_channel - find a channel to carry out the operation
 * @tx_type: transaction type
 */


struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
{
	return this_cpu_read(channel_table[tx_type]->chan);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
19
86.36%
1
50.00%
Christoph Lameter
3
13.64%
1
50.00%
Total
22
100.00%
2
100.00%


EXPORT_SYMBOL(dma_find_channel);

/**
 * dma_issue_pending_all - flush all pending operations across all channels
 */


void dma_issue_pending_all(void)
{
	struct dma_device *device;
	struct dma_chan *chan;

	rcu_read_lock();
	list_for_each_entry_rcu(device, &dma_device_list, global_node) {
		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
		list_for_each_entry(chan, &device->channels, device_node)
			if (chan->client_count)
				device->device_issue_pending(chan);
	}
	rcu_read_unlock();
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
66
100.00%
2
100.00%
Total
66
100.00%
2
100.00%


EXPORT_SYMBOL(dma_issue_pending_all);

/**
 * dma_chan_is_local - returns true if the channel is in the same numa-node as the cpu
 */


static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
{
	int node = dev_to_node(chan->device->dev);
	return node == -1 || cpumask_test_cpu(cpu, cpumask_of_node(node));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Brice Goglin
38
90.48%
1
50.00%
Dan J Williams
4
9.52%
1
50.00%
Total
42
100.00%
2
100.00%

/**
 * min_chan - returns the channel with min count and in the same numa-node as the cpu
 * @cap: capability to match
 * @cpu: cpu index which the channel should be close to
 *
 * If some channels are close to the given cpu, the one with the lowest
 * reference count is returned. Otherwise, cpu is ignored and only the
 * reference count is taken into account.
 * Must be called under dma_list_mutex.
 */


static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)
{
	struct dma_device *device;
	struct dma_chan *chan;
	struct dma_chan *min = NULL;
	struct dma_chan *localmin = NULL;

	list_for_each_entry(device, &dma_device_list, global_node) {
		if (!dma_has_cap(cap, device->cap_mask) ||
		    dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
		list_for_each_entry(chan, &device->channels, device_node) {
			if (!chan->client_count)
				continue;
			if (!min || chan->table_count < min->table_count)
				min = chan;

			if (dma_chan_is_local(chan, cpu))
				if (!localmin ||
				    chan->table_count < localmin->table_count)
					localmin = chan;
		}
	}

	chan = localmin ? localmin : min;

	if (chan)
		chan->table_count++;

	return chan;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
115
75.66%
2
66.67%
Brice Goglin
37
24.34%
1
33.33%
Total
152
100.00%
3
100.00%

/**
 * dma_channel_rebalance - redistribute the available channels
 *
 * Optimize for cpu isolation (each cpu gets a dedicated channel for an
 * operation type) in the SMP case,  and operation isolation (avoid
 * multi-tasking channels) in the non-SMP case.  Must be called under
 * dma_list_mutex.
 */


static void dma_channel_rebalance(void)
{
	struct dma_chan *chan;
	struct dma_device *device;
	int cpu;
	int cap;

	/* undo the last distribution */
	for_each_dma_cap_mask(cap, dma_cap_mask_all)
		for_each_possible_cpu(cpu)
			per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;

	list_for_each_entry(device, &dma_device_list, global_node) {
		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
		list_for_each_entry(chan, &device->channels, device_node)
			chan->table_count = 0;
	}

	/* don't populate the channel_table if no clients are available */
	if (!dmaengine_ref_count)
		return;

	/* redistribute available channels */
	for_each_dma_cap_mask(cap, dma_cap_mask_all)
		for_each_online_cpu(cpu) {
			chan = min_chan(cap, cpu);
			per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
		}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
126
98.44%
2
66.67%
Brice Goglin
2
1.56%
1
33.33%
Total
128
100.00%
3
100.00%



int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
{
	struct dma_device *device;

	if (!chan || !caps)
		return -EINVAL;

	device = chan->device;

	/* check if the channel supports slave transactions */
	if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||
	      test_bit(DMA_CYCLIC, device->cap_mask.bits)))
		return -ENXIO;

	/*
         * Check whether it reports it uses the generic slave
         * capabilities, if not, that means it doesn't support any
         * kind of slave capabilities reporting.
         */
	if (!device->directions)
		return -ENXIO;

	caps->src_addr_widths = device->src_addr_widths;
	caps->dst_addr_widths = device->dst_addr_widths;
	caps->directions = device->directions;
	caps->max_burst = device->max_burst;
	caps->residue_granularity = device->residue_granularity;
	caps->descriptor_reuse = device->descriptor_reuse;

	/*
         * Some devices implement only pause (e.g. to get residuum) but no
         * resume. However cmd_pause is advertised as pause AND resume.
         */
	caps->cmd_pause = !!(device->device_pause && device->device_resume);
	caps->cmd_terminate = !!device->device_terminate_all;

	return 0;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Laurent Pinchart
124
77.50%
1
16.67%
Vinod Koul
12
7.50%
1
16.67%
Robert Jarzmik
8
5.00%
1
16.67%
Shawn Lin
8
5.00%
1
16.67%
Krzysztof Kozlowski
7
4.38%
1
16.67%
Andy Shevchenko
1
0.62%
1
16.67%
Total
160
100.00%
6
100.00%


EXPORT_SYMBOL_GPL(dma_get_slave_caps);



static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
					  struct dma_device *dev,
					  dma_filter_fn fn, void *fn_param)
{
	struct dma_chan *chan;

	if (mask && !__dma_device_satisfies_mask(dev, mask)) {
		dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);
		return NULL;
	}
	/* devices with multiple channels need special handling as we need to
         * ensure that all channels are either private or public.
         */
	if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))
		list_for_each_entry(chan, &dev->channels, device_node) {
			/* some channels are already publicly allocated */
			if (chan->client_count)
				return NULL;
		}

	list_for_each_entry(chan, &dev->channels, device_node) {
		if (chan->client_count) {
			dev_dbg(dev->dev, "%s: %s busy\n",
				 __func__, dma_chan_name(chan));
			continue;
		}
		if (fn && !fn(chan, fn_param)) {
			dev_dbg(dev->dev, "%s: %s filter said false\n",
				 __func__, dma_chan_name(chan));
			continue;
		}
		return chan;
	}

	return NULL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
131
77.51%
4
50.00%
Christopher Leech
20
11.83%
1
12.50%
Jarkko Nikula
15
8.88%
1
12.50%
Peter Ujfalusi
2
1.18%
1
12.50%
Lars-Peter Clausen
1
0.59%
1
12.50%
Total
169
100.00%
8
100.00%



static struct dma_chan *find_candidate(struct dma_device *device,
				       const dma_cap_mask_t *mask,
				       dma_filter_fn fn, void *fn_param)
{
	struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);
	int err;

	if (chan) {
		/* Found a suitable channel, try to grab, prep, and return it.
                 * We first set DMA_PRIVATE to disable balance_ref_count as this
                 * channel will not be published in the general-purpose
                 * allocator
                 */
		dma_cap_set(DMA_PRIVATE, device->cap_mask);
		device->privatecnt++;
		err = dma_chan_get(chan);

		if (err) {
			if (err == -ENODEV) {
				dev_dbg(device->dev, "%s: %s module removed\n",
					__func__, dma_chan_name(chan));
				list_del_rcu(&device->global_node);
			} else
				dev_dbg(device->dev,
					"%s: failed to get %s: (%d)\n",
					 __func__, dma_chan_name(chan), err);

			if (--device->privatecnt == 0)
				dma_cap_clear(DMA_PRIVATE, device->cap_mask);

			chan = ERR_PTR(err);
		}
	}

	return chan ? chan : ERR_PTR(-EPROBE_DEFER);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Ujfalusi
156
93.98%
1
50.00%
Jarkko Nikula
10
6.02%
1
50.00%
Total
166
100.00%
2
100.00%

/**
 * dma_get_slave_channel - try to get specific channel exclusively
 * @chan: target channel
 */


struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
{
	int err = -EBUSY;

	/* lock against __dma_request_channel */
	mutex_lock(&dma_list_mutex);

	if (chan->client_count == 0) {
		struct dma_device *device = chan->device;

		dma_cap_set(DMA_PRIVATE, device->cap_mask);
		device->privatecnt++;
		err = dma_chan_get(chan);
		if (err) {
			dev_dbg(chan->device->dev,
				"%s: failed to get %s: (%d)\n",
				__func__, dma_chan_name(chan), err);
			chan = NULL;
			if (--device->privatecnt == 0)
				dma_cap_clear(DMA_PRIVATE, device->cap_mask);
		}
	} else
		chan = NULL;

	mutex_unlock(&dma_list_mutex);


	return chan;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Zhangfei Gao
60
47.24%
1
25.00%
Peter Ujfalusi
47
37.01%
1
25.00%
Vinod Koul
13
10.24%
1
25.00%
Jarkko Nikula
7
5.51%
1
25.00%
Total
127
100.00%
4
100.00%


EXPORT_SYMBOL_GPL(dma_get_slave_channel);



struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)
{
	dma_cap_mask_t mask;
	struct dma_chan *chan;

	dma_cap_zero(mask);
	dma_cap_set(DMA_SLAVE, mask);

	/* lock against __dma_request_channel */
	mutex_lock(&dma_list_mutex);

	chan = find_candidate(device, &mask, NULL, NULL);

	mutex_unlock(&dma_list_mutex);

	return IS_ERR(chan) ? NULL : chan;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Stephen Warren
57
82.61%
1
33.33%
Peter Ujfalusi
10
14.49%
1
33.33%
Christopher Freeman
2
2.90%
1
33.33%
Total
69
100.00%
3
100.00%


EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);

/**
 * __dma_request_channel - try to allocate an exclusive channel
 * @mask: capabilities that the channel must satisfy
 * @fn: optional callback to disposition available channels
 * @fn_param: opaque parameter to pass to dma_filter_fn
 *
 * Returns pointer to appropriate DMA channel on success or NULL.
 */


struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
				       dma_filter_fn fn, void *fn_param)
{
	struct dma_device *device, *_d;
	struct dma_chan *chan = NULL;

	/* Find a channel */
	mutex_lock(&dma_list_mutex);
	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
		chan = find_candidate(device, mask, fn, fn_param);
		if (!IS_ERR(chan))
			break;

		chan = NULL;
	}
	mutex_unlock(&dma_list_mutex);

	pr_debug("%s: %s (%s)\n",
		 __func__,
		 chan ? "success" : "fail",
		 chan ? dma_chan_name(chan) : NULL);

	return chan;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
99
90.00%
5
55.56%
Peter Ujfalusi
5
4.55%
1
11.11%
Christopher Leech
4
3.64%
1
11.11%
Lars-Peter Clausen
1
0.91%
1
11.11%
Jarkko Nikula
1
0.91%
1
11.11%
Total
110
100.00%
9
100.00%


EXPORT_SYMBOL_GPL(__dma_request_channel);



static const struct dma_slave_map *dma_filter_match(struct dma_device *device,
						    const char *name,
						    struct device *dev)
{
	int i;

	if (!device->filter.mapcnt)
		return NULL;

	for (i = 0; i < device->filter.mapcnt; i++) {
		const struct dma_slave_map *map = &device->filter.map[i];

		if (!strcmp(map->devname, dev_name(dev)) &&
		    !strcmp(map->slave, name))
			return map;
	}

	return NULL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Ujfalusi
105
100.00%
1
100.00%
Total
105
100.00%
1
100.00%

/**
 * dma_request_chan - try to allocate an exclusive slave channel
 * @dev:        pointer to client device structure
 * @name:       slave channel name
 *
 * Returns pointer to appropriate DMA channel on success or an error pointer.
 */


struct dma_chan *dma_request_chan(struct device *dev, const char *name)
{
	struct dma_device *d, *_d;
	struct dma_chan *chan = NULL;

	/* If device-tree is present get slave info from here */
	if (dev->of_node)
		chan = of_dma_request_slave_channel(dev->of_node, name);

	/* If device was enumerated by ACPI get slave info from here */
	if (has_acpi_companion(dev) && !chan)
		chan = acpi_dma_request_slave_chan_by_name(dev, name);

	if (chan) {
		/* Valid channel found or requester need to be deferred */
		if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER)
			return chan;
	}

	/* Try to find the channel via the DMA filter map(s) */
	mutex_lock(&dma_list_mutex);
	list_for_each_entry_safe(d, _d, &dma_device_list, global_node) {
		dma_cap_mask_t mask;
		const struct dma_slave_map *map = dma_filter_match(d, name, dev);

		if (!map)
			continue;

		dma_cap_zero(mask);
		dma_cap_set(DMA_SLAVE, mask);

		chan = find_candidate(d, &mask, d->filter.fn, map->param);
		if (!IS_ERR(chan))
			break;
	}
	mutex_unlock(&dma_list_mutex);

	return chan ? chan : ERR_PTR(-EPROBE_DEFER);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Ujfalusi
146
74.11%
1
20.00%
Jon Hunter
29
14.72%
1
20.00%
Andy Shevchenko
14
7.11%
1
20.00%
Stephen Warren
7
3.55%
1
20.00%
Markus Pargmann
1
0.51%
1
20.00%
Total
197
100.00%
5
100.00%


EXPORT_SYMBOL_GPL(dma_request_chan);

/**
 * dma_request_slave_channel - try to allocate an exclusive slave channel
 * @dev:        pointer to client device structure
 * @name:       slave channel name
 *
 * Returns pointer to appropriate DMA channel on success or NULL.
 */


struct dma_chan *dma_request_slave_channel(struct device *dev,
					   const char *name)
{
	struct dma_chan *ch = dma_request_chan(dev, name);
	if (IS_ERR(ch))
		return NULL;

	return ch;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Stephen Warren
34
80.95%
1
33.33%
Peter Ujfalusi
5
11.90%
1
33.33%
Jon Hunter
3
7.14%
1
33.33%
Total
42
100.00%
3
100.00%


EXPORT_SYMBOL_GPL(dma_request_slave_channel);

/**
 * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
 * @mask: capabilities that the channel must satisfy
 *
 * Returns pointer to appropriate DMA channel on success or an error pointer.
 */


struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)
{
	struct dma_chan *chan;

	if (!mask)
		return ERR_PTR(-ENODEV);

	chan = __dma_request_channel(mask, NULL, NULL);
	if (!chan)
		chan = ERR_PTR(-ENODEV);

	return chan;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Ujfalusi
49
87.50%
1
25.00%
Robert Baldyga
4
7.14%
1
25.00%
Stephen Warren
2
3.57%
1
25.00%
Jon Hunter
1
1.79%
1
25.00%
Total
56
100.00%
4
100.00%


EXPORT_SYMBOL_GPL(dma_request_chan_by_mask);



void dma_release_channel(struct dma_chan *chan)
{
	mutex_lock(&dma_list_mutex);
	WARN_ONCE(chan->client_count != 1,
		  "chan reference count %d != 1\n", chan->client_count);
	dma_chan_put(chan);
	/* drop PRIVATE cap enabled by __dma_request_channel() */
	if (--chan->device->privatecnt == 0)
		dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);
	mutex_unlock(&dma_list_mutex);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
42
64.62%
1
50.00%
Atsushi Nemoto
23
35.38%
1
50.00%
Total
65
100.00%
2
100.00%


EXPORT_SYMBOL_GPL(dma_release_channel);

/**
 * dmaengine_get - register interest in dma_channels
 */


void dmaengine_get(void)
{
	struct dma_device *device, *_d;
	struct dma_chan *chan;
	int err;

	mutex_lock(&dma_list_mutex);
	dmaengine_ref_count++;

	/* try to grab channels */
	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
		list_for_each_entry(chan, &device->channels, device_node) {
			err = dma_chan_get(chan);
			if (err == -ENODEV) {
				/* module removed before we could use it */
				list_del_rcu(&device->global_node);
				break;
			} else if (err)
				dev_dbg(chan->device->dev,
					"%s: failed to get %s: (%d)\n",
					__func__, dma_chan_name(chan), err);
		}
	}

	/* if this is the first reference and there were channels
         * waiting we need to rebalance to get those channels
         * incorporated into the channel table
         */
	if (dmaengine_ref_count == 1)
		dma_channel_rebalance();
	mutex_unlock(&dma_list_mutex);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
100
74.63%
8
66.67%
Christopher Leech
22
16.42%
1
8.33%
Jarkko Nikula
7
5.22%
1
8.33%
Fabio Estevam
3
2.24%
1
8.33%
Håvard Skinnemoen
2
1.49%
1
8.33%
Total
134
100.00%
12
100.00%


EXPORT_SYMBOL(dmaengine_get);

/**
 * dmaengine_put - let dma drivers be removed when ref_count == 0
 */


void dmaengine_put(void)
{
	struct dma_device *device;
	struct dma_chan *chan;

	mutex_lock(&dma_list_mutex);
	dmaengine_ref_count--;
	BUG_ON(dmaengine_ref_count < 0);
	/* drop channel references */
	list_for_each_entry(device, &dma_device_list, global_node) {
		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
		list_for_each_entry(chan, &device->channels, device_node)
			dma_chan_put(chan);
	}
	mutex_unlock(&dma_list_mutex);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
52
69.33%
4
80.00%
Christopher Leech
23
30.67%
1
20.00%
Total
75
100.00%
5
100.00%


EXPORT_SYMBOL(dmaengine_put);



static bool device_has_all_tx_types(struct dma_device *device)
{
	/* A device that satisfies this test has channels that will never cause
         * an async_tx channel switch event as all possible operation types can
         * be handled.
         */
	#ifdef CONFIG_ASYNC_TX_DMA
	if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
		return false;
	#endif

	#if IS_ENABLED(CONFIG_ASYNC_MEMCPY)
	if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
		return false;
	#endif

	#if IS_ENABLED(CONFIG_ASYNC_XOR)
	if (!dma_has_cap(DMA_XOR, device->cap_mask))
		return false;

	#ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
	if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))
		return false;
	#endif
	#endif

	#if IS_ENABLED(CONFIG_ASYNC_PQ)
	if (!dma_has_cap(DMA_PQ, device->cap_mask))
		return false;

	#ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
	if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))
		return false;
	#endif
	#endif

	return true;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
141
97.92%
3
75.00%
Javier Martinez Canillas
3
2.08%
1
25.00%
Total
144
100.00%
4
100.00%



static int get_dma_id(struct dma_device *device)
{
	int rc;

	do {
		if (!ida_pre_get(&dma_ida, GFP_KERNEL))
			return -ENOMEM;
		mutex_lock(&dma_list_mutex);
		rc = ida_get_new(&dma_ida, &device->dev_id);
		mutex_unlock(&dma_list_mutex);
	} while (rc == -EAGAIN);

	return rc;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
37
54.41%
1
33.33%
Matthew Wilcox
30
44.12%
1
33.33%
Tejun Heo
1
1.47%
1
33.33%
Total
68
100.00%
3
100.00%

/**
 * dma_async_device_register - registers DMA devices found
 * @device: &dma_device
 */


int dma_async_device_register(struct dma_device *device)
{
	int chancnt = 0, rc;
	struct dma_chan* chan;
	atomic_t *idr_ref;

	if (!device)
		return -ENODEV;

	/* validate device routines */
	BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) &&
		!device->device_prep_dma_memcpy);
	BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
		!device->device_prep_dma_xor);
	BUG_ON(dma_has_cap(DMA_XOR_VAL, device->cap_mask) &&
		!device->device_prep_dma_xor_val);
	BUG_ON(dma_has_cap(DMA_PQ, device->cap_mask) &&
		!device->device_prep_dma_pq);
	BUG_ON(dma_has_cap(DMA_PQ_VAL, device->cap_mask) &&
		!device->device_prep_dma_pq_val);
	BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
		!device->device_prep_dma_memset);
	BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
		!device->device_prep_dma_interrupt);
	BUG_ON(dma_has_cap(DMA_SG, device->cap_mask) &&
		!device->device_prep_dma_sg);
	BUG_ON(dma_has_cap(DMA_CYCLIC, device->cap_mask) &&
		!device->device_prep_dma_cyclic);
	BUG_ON(dma_has_cap(DMA_INTERLEAVE, device->cap_mask) &&
		!device->device_prep_interleaved_dma);

	BUG_ON(!device->device_tx_status);
	BUG_ON(!device->device_issue_pending);
	BUG_ON(!device->dev);

	/* note: this only matters in the
         * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
         */
	if (device_has_all_tx_types(device))
		dma_cap_set(DMA_ASYNC_TX, device->cap_mask);

	idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
	if (!idr_ref)
		return -ENOMEM;
	rc = get_dma_id(device);
	if (rc != 0) {
		kfree(idr_ref);
		return rc;
	}

	atomic_set(idr_ref, 0);

	/* represent channels in sysfs. Probably want devs too */
	list_for_each_entry(chan, &device->channels, device_node) {
		rc = -ENOMEM;
		chan->local = alloc_percpu(typeof(*chan->local));
		if (chan->local == NULL)
			goto err_out;
		chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);
		if (chan->dev == NULL) {
			free_percpu(chan->local);
			chan->local = NULL;
			goto err_out;
		}

		chan->chan_id = chancnt++;
		chan->dev->device.class = &dma_devclass;
		chan->dev->device.parent = device->dev;
		chan->dev->chan = chan;
		chan->dev->idr_ref = idr_ref;
		chan->dev->dev_id = device->dev_id;
		atomic_inc(idr_ref);
		dev_set_name(&chan->dev->device, "dma%dchan%d",
			     device->dev_id, chan->chan_id);

		rc = device_register(&chan->dev->device);
		if (rc) {
			free_percpu(chan->local);
			chan->local = NULL;
			kfree(chan->dev);
			atomic_dec(idr_ref);
			goto err_out;
		}
		chan->client_count = 0;
	}

	if (!chancnt) {
		dev_err(device->dev, "%s: device has no channels!\n", __func__);
		rc = -ENODEV;
		goto err_out;
	}

	device->chancnt = chancnt;

	mutex_lock(&dma_list_mutex);
	/* take references on public channels */
	if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))
		list_for_each_entry(chan, &device->channels, device_node) {
			/* if clients are already waiting for channels we need
                         * to take references on their behalf
                         */
			if (dma_chan_get(chan) == -ENODEV) {
				/* note we can only get here for the first
                                 * channel as the remaining channels are
                                 * guaranteed to get a reference
                                 */
				rc = -ENODEV;
				mutex_unlock(&dma_list_mutex);
				goto err_out;
			}
		}
	list_add_tail_rcu(&device->global_node, &dma_device_list);
	if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
		device->privatecnt++;	/* Always private */
	dma_channel_rebalance();
	mutex_unlock(&dma_list_mutex);

	return 0;

err_out:
	/* if we never registered a channel just release the idr */
	if (atomic_read(idr_ref) == 0) {
		mutex_lock(&dma_list_mutex);
		ida_remove(&dma_ida, device->dev_id);
		mutex_unlock(&dma_list_mutex);
		kfree(idr_ref);
		return rc;
	}

	list_for_each_entry(chan, &device->channels, device_node) {
		if (chan->local == NULL)
			continue;
		mutex_lock(&dma_list_mutex);
		chan->dev->chan = NULL;
		mutex_unlock(&dma_list_mutex);
		device_unregister(&chan->dev->device);
		free_percpu(chan->local);
	}
	return rc;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
430
57.26%
15
51.72%
Christopher Leech
132
17.58%
1
3.45%
Jeff Garzik
61
8.12%
1
3.45%
Håvard Skinnemoen
33
4.39%
2
6.90%
Viresh Kumar
26
3.46%
1
3.45%
Maxime Ripard
17
2.26%
1
3.45%
Ira W. Snyder
17
2.26%
1
3.45%
Atsushi Nemoto
17
2.26%
1
3.45%
Tony Jones
8
1.07%
1
3.45%
Jassi Brar
4
0.53%
1
3.45%
Matthew Wilcox
2
0.27%
1
3.45%
Kay Sievers
2
0.27%
1
3.45%
Linus Walleij
1
0.13%
1
3.45%
Zhang Wei
1
0.13%
1
3.45%
Total
751
100.00%
29
100.00%


EXPORT_SYMBOL(dma_async_device_register);

/**
 * dma_async_device_unregister - unregister a DMA device
 * @device: &dma_device
 *
 * This routine is called by dma driver exit routines, dmaengine holds module
 * references to prevent it being called while channels are in use.
 */


void dma_async_device_unregister(struct dma_device *device)
{
	struct dma_chan *chan;

	mutex_lock(&dma_list_mutex);
	list_del_rcu(&device->global_node);
	dma_channel_rebalance();
	mutex_unlock(&dma_list_mutex);

	list_for_each_entry(chan, &device->channels, device_node) {
		WARN_ONCE(chan->client_count,
			  "%s called while %d clients hold a reference\n",
			  __func__, chan->client_count);
		mutex_lock(&dma_list_mutex);
		chan->dev->chan = NULL;
		mutex_unlock(&dma_list_mutex);
		device_unregister(&chan->dev->device);
		free_percpu(chan->local);
	}
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Christopher Leech
54
54.00%
1
14.29%
Dan J Williams
37
37.00%
4
57.14%
Anatolij Gustschin
7
7.00%
1
14.29%
Tony Jones
2
2.00%
1
14.29%
Total
100
100.00%
7
100.00%


EXPORT_SYMBOL(dma_async_device_unregister);


struct dmaengine_unmap_pool {
	
struct kmem_cache *cache;
	
const char *name;
	
mempool_t *pool;
	
size_t size;
};


#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }

static struct dmaengine_unmap_pool unmap_pool[] = {
	__UNMAP_POOL(2),
	#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
	__UNMAP_POOL(16),
	__UNMAP_POOL(128),
	__UNMAP_POOL(256),
	#endif
};



static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)
{
	int order = get_count_order(nr);

	switch (order) {
	case 0 ... 1:
		return &unmap_pool[0];
#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
	case 2 ... 4:
		return &unmap_pool[1];
	case 5 ... 7:
		return &unmap_pool[2];
	case 8:
		return &unmap_pool[3];
#endif
	default:
		BUG();
		return NULL;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
78
90.70%
3
75.00%
Matthias Kaehlcke
8
9.30%
1
25.00%
Total
86
100.00%
4
100.00%



static void dmaengine_unmap(struct kref *kref)
{
	struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);
	struct device *dev = unmap->dev;
	int cnt, i;

	cnt = unmap->to_cnt;
	for (i = 0; i < cnt; i++)
		dma_unmap_page(dev, unmap->addr[i], unmap->len,
			       DMA_TO_DEVICE);
	cnt += unmap->from_cnt;
	for (; i < cnt; i++)
		dma_unmap_page(dev, unmap->addr[i], unmap->len,
			       DMA_FROM_DEVICE);
	cnt += unmap->bidi_cnt;
	for (; i < cnt; i++) {
		if (unmap->addr[i] == 0)
			continue;
		dma_unmap_page(dev, unmap->addr[i], unmap->len,
			       DMA_BIDIRECTIONAL);
	}
	cnt = unmap->map_cnt;
	mempool_free(unmap, __get_unmap_pool(cnt)->pool);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
173
96.11%
5
71.43%
Xuelin Shi
6
3.33%
1
14.29%
Maciej Sosnowski
1
0.56%
1
14.29%
Total
180
100.00%
7
100.00%



void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
{
	if (unmap)
		kref_put(&unmap->kref, dmaengine_unmap);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
22
91.67%
2
66.67%
Christoph Lameter
2
8.33%
1
33.33%
Total
24
100.00%
3
100.00%


EXPORT_SYMBOL_GPL(dmaengine_unmap_put);



static void dmaengine_destroy_unmap_pool(void)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
		struct dmaengine_unmap_pool *p = &unmap_pool[i];

		mempool_destroy(p->pool);
		p->pool = NULL;
		kmem_cache_destroy(p->cache);
		p->cache = NULL;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
63
95.45%
3
75.00%
Maciej Sosnowski
3
4.55%
1
25.00%
Total
66
100.00%
4
100.00%



static int __init dmaengine_init_unmap_pool(void)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
		struct dmaengine_unmap_pool *p = &unmap_pool[i];
		size_t size;

		size = sizeof(struct dmaengine_unmap_data) +
		       sizeof(dma_addr_t) * p->size;

		p->cache = kmem_cache_create(p->name, size, 0,
					     SLAB_HWCACHE_ALIGN, NULL);
		if (!p->cache)
			break;
		p->pool = mempool_create_slab_pool(1, p->cache);
		if (!p->pool)
			break;
	}

	if (i == ARRAY_SIZE(unmap_pool))
		return 0;

	dmaengine_destroy_unmap_pool();
	return -ENOMEM;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
128
100.00%
3
100.00%
Total
128
100.00%
3
100.00%


struct dmaengine_unmap_data *

dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
{
	struct dmaengine_unmap_data *unmap;

	unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);
	if (!unmap)
		return NULL;

	memset(unmap, 0, sizeof(*unmap));
	kref_init(&unmap->kref);
	unmap->dev = dev;
	unmap->map_cnt = nr;

	return unmap;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
72
88.89%
2
50.00%
Xuelin Shi
6
7.41%
1
25.00%
Christoph Lameter
3
3.70%
1
25.00%
Total
81
100.00%
4
100.00%


EXPORT_SYMBOL(dmaengine_get_unmap_data);



void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
	struct dma_chan *chan)
{
	tx->chan = chan;
	#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
	spin_lock_init(&tx->lock);
	#endif
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
34
100.00%
3
100.00%
Total
34
100.00%
3
100.00%


EXPORT_SYMBOL(dma_async_tx_descriptor_init);

/* dma_wait_for_async_tx - spin wait for a transaction to complete
 * @tx: in-flight transaction to wait on
 */

enum dma_status

dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
{
	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);

	if (!tx)
		return DMA_COMPLETE;

	while (tx->cookie == -EBUSY) {
		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
			dev_err(tx->chan->device->dev,
				"%s timeout waiting for descriptor submission\n",
				__func__);
			return DMA_ERROR;
		}
		cpu_relax();
	}
	return dma_sync_wait(tx->chan, tx->cookie);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
75
88.24%
2
50.00%
Jarkko Nikula
9
10.59%
1
25.00%
Vinod Koul
1
1.18%
1
25.00%
Total
85
100.00%
4
100.00%


EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);

/* dma_run_dependencies - helper routine for dma drivers to process
 *      (start) dependent operations on their target channel
 * @tx: transaction with dependencies
 */


void dma_run_dependencies(struct dma_async_tx_descriptor *tx)
{
	struct dma_async_tx_descriptor *dep = txd_next(tx);
	struct dma_async_tx_descriptor *dep_next;
	struct dma_chan *chan;

	if (!dep)
		return;

	/* we'll submit tx->next now, so clear the link */
	txd_clear_next(tx);
	chan = dep->chan;

	/* keep submitting up until a channel switch is detected
         * in that case we will be called again as a result of
         * processing the interrupt from async_tx_channel_switch
         */
	for (; dep; dep = dep_next) {
		txd_lock(dep);
		txd_clear_parent(dep);
		dep_next = txd_next(dep);
		if (dep_next && dep_next->chan == chan)
			txd_clear_next(dep); /* ->next will be submitted */
		else
			dep_next = NULL; /* submit current dep and terminate */
		txd_unlock(dep);

		dep->tx_submit(dep);
	}

	chan->device->device_issue_pending(chan);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
117
97.50%
2
66.67%
Yuri Tikhonov
3
2.50%
1
33.33%
Total
120
100.00%
3
100.00%


EXPORT_SYMBOL_GPL(dma_run_dependencies);



static int __init dma_bus_init(void)
{
	int err = dmaengine_init_unmap_pool();

	if (err)
		return err;
	return class_register(&dma_devclass);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Christopher Leech
16
55.17%
1
50.00%
Dan J Williams
13
44.83%
1
50.00%
Total
29
100.00%
2
100.00%


arch_initcall(dma_bus_init);


Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
3026
59.06%
30
32.26%
Peter Ujfalusi
580
11.32%
5
5.38%
Christopher Leech
471
9.19%
1
1.08%
Laurent Pinchart
129
2.52%
1
1.08%
Stephen Warren
111
2.17%
2
2.15%
Brice Goglin
79
1.54%
1
1.08%
Maxime Ripard
73
1.42%
3
3.23%
Zhangfei Gao
65
1.27%
1
1.08%
Jeff Garzik
61
1.19%
1
1.08%
Jarkko Nikula
57
1.11%
3
3.23%
Greg Kroah-Hartman
45
0.88%
1
1.08%
Atsushi Nemoto
40
0.78%
1
1.08%
Jon Hunter
40
0.78%
1
1.08%
Matthew Wilcox
36
0.70%
1
1.08%
Håvard Skinnemoen
35
0.68%
2
2.15%
Tony Jones
32
0.62%
1
1.08%
Vinod Koul
26
0.51%
3
3.23%
Viresh Kumar
26
0.51%
1
1.08%
Andy Shevchenko
21
0.41%
2
2.15%
David Brownell
18
0.35%
1
1.08%
Ira W. Snyder
17
0.33%
1
1.08%
Xuelin Shi
12
0.23%
1
1.08%
Joe Perches
11
0.21%
1
1.08%
Lars-Peter Clausen
11
0.21%
2
2.15%
Bartlomiej Zolnierkiewicz
11
0.21%
1
1.08%
Matthias Kaehlcke
8
0.16%
1
1.08%
Christoph Lameter
8
0.16%
1
1.08%
Robert Jarzmik
8
0.16%
1
1.08%
Shawn Lin
8
0.16%
1
1.08%
Anatolij Gustschin
7
0.14%
1
1.08%
Krzysztof Kozlowski
7
0.14%
1
1.08%
Tejun Heo
5
0.10%
3
3.23%
Robert Baldyga
5
0.10%
1
1.08%
Jassi Brar
4
0.08%
1
1.08%
Maciej Sosnowski
4
0.08%
1
1.08%
Axel Lin
4
0.08%
1
1.08%
Yuri Tikhonov
3
0.06%
1
1.08%
Javier Martinez Canillas
3
0.06%
1
1.08%
Alexey Dobriyan
3
0.06%
1
1.08%
Fabio Estevam
3
0.06%
1
1.08%
Andrew Morton
2
0.04%
1
1.08%
Kay Sievers
2
0.04%
1
1.08%
Christopher Freeman
2
0.04%
1
1.08%
Markus Pargmann
1
0.02%
1
1.08%
Stefan Agner
1
0.02%
1
1.08%
Linus Walleij
1
0.02%
1
1.08%
Zhang Wei
1
0.02%
1
1.08%
Randy Dunlap
1
0.02%
1
1.08%
Total
5124
100.00%
93
100.00%
Directory: drivers/dma

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.