Release 4.16 arch/powerpc/kernel/dma.c
/*
* Copyright (C) 2006 Benjamin Herrenschmidt, IBM Corporation
*
* Provide default implementations of the DMA mapping callbacks for
* directly mapped busses.
*/
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/dma-debug.h>
#include <linux/gfp.h>
#include <linux/memblock.h>
#include <linux/export.h>
#include <linux/pci.h>
#include <asm/vio.h>
#include <asm/bug.h>
#include <asm/machdep.h>
#include <asm/swiotlb.h>
#include <asm/iommu.h>
/*
* Generic direct DMA implementation
*
* This implementation supports a per-device offset that can be applied if
* the address at which memory is visible to devices is not 0. Platform code
* can set archdata.dma_data to an unsigned long holding the offset. By
* default the offset is PCI_DRAM_OFFSET.
*/
static u64 __maybe_unused get_pfn_limit(struct device *dev)
{
u64 pfn = (dev->coherent_dma_mask >> PAGE_SHIFT) + 1;
struct dev_archdata __maybe_unused *sd = &dev->archdata;
#ifdef CONFIG_SWIOTLB
if (sd->max_direct_dma_addr && dev->dma_ops == &powerpc_swiotlb_dma_ops)
pfn = min_t(u64, pfn, sd->max_direct_dma_addr >> PAGE_SHIFT);
#endif
return pfn;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Scott Wood | 70 | 97.22% | 1 | 33.33% |
| Christoph Hellwig | 1 | 1.39% | 1 | 33.33% |
| Bart Van Assche | 1 | 1.39% | 1 | 33.33% |
| Total | 72 | 100.00% | 3 | 100.00% |
static int dma_nommu_dma_supported(struct device *dev, u64 mask)
{
#ifdef CONFIG_PPC64
u64 limit = get_dma_offset(dev) + (memblock_end_of_DRAM() - 1);
/* Limit fits in the mask, we are good */
if (mask >= limit)
return 1;
#ifdef CONFIG_FSL_SOC
/* Freescale gets another chance via ZONE_DMA/ZONE_DMA32, however
* that will have to be refined if/when they support iommus
*/
return 1;
#endif
/* Sorry ... */
return 0;
#else
return 1;
#endif
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 61 | 98.39% | 1 | 50.00% |
| Christoph Hellwig | 1 | 1.61% | 1 | 50.00% |
| Total | 62 | 100.00% | 2 | 100.00% |
void *__dma_nommu_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
unsigned long attrs)
{
void *ret;
#ifdef CONFIG_NOT_COHERENT_CACHE
ret = __dma_alloc_coherent(dev, size, dma_handle, flag);
if (ret == NULL)
return NULL;
*dma_handle += get_dma_offset(dev);
return ret;
#else
struct page *page;
int node = dev_to_node(dev);
#ifdef CONFIG_FSL_SOC
u64 pfn = get_pfn_limit(dev);
int zone;
/*
* This code should be OK on other platforms, but we have drivers that
* don't set coherent_dma_mask. As a workaround we just ifdef it. This
* whole routine needs some serious cleanup.
*/
zone = dma_pfn_limit_to_zone(pfn);
if (zone < 0) {
dev_err(dev, "%s: No suitable zone for pfn %#llx\n",
__func__, pfn);
return NULL;
}
switch (zone) {
case ZONE_DMA:
flag |= GFP_DMA;
break;
#ifdef CONFIG_ZONE_DMA32
case ZONE_DMA32:
flag |= GFP_DMA32;
break;
#endif
};
#endif /* CONFIG_FSL_SOC */
page = alloc_pages_node(node, flag, get_order(size));
if (page == NULL)
return NULL;
ret = page_address(page);
memset(ret, 0, size);
*dma_handle = __pa(ret) + get_dma_offset(dev);
return ret;
#endif
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 87 | 41.04% | 4 | 23.53% |
| Scott Wood | 68 | 32.08% | 2 | 11.76% |
| Becky Bruce | 21 | 9.91% | 3 | 17.65% |
| Andrew Morton | 16 | 7.55% | 1 | 5.88% |
| Michael Ellerman | 12 | 5.66% | 3 | 17.65% |
| Jeremy Kerr | 3 | 1.42% | 1 | 5.88% |
| Krzysztof Kozlowski | 2 | 0.94% | 1 | 5.88% |
| Andrzej Pietrasiewicz | 2 | 0.94% | 1 | 5.88% |
| Christoph Hellwig | 1 | 0.47% | 1 | 5.88% |
| Total | 212 | 100.00% | 17 | 100.00% |
void __dma_nommu_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
#ifdef CONFIG_NOT_COHERENT_CACHE
__dma_free_coherent(size, vaddr);
#else
free_pages((unsigned long)vaddr, get_order(size));
#endif
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 30 | 57.69% | 1 | 16.67% |
| Becky Bruce | 14 | 26.92% | 1 | 16.67% |
| Andrew Morton | 3 | 5.77% | 1 | 16.67% |
| Andrzej Pietrasiewicz | 2 | 3.85% | 1 | 16.67% |
| Krzysztof Kozlowski | 2 | 3.85% | 1 | 16.67% |
| Christoph Hellwig | 1 | 1.92% | 1 | 16.67% |
| Total | 52 | 100.00% | 6 | 100.00% |
static void *dma_nommu_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
unsigned long attrs)
{
struct iommu_table *iommu;
/* The coherent mask may be smaller than the real mask, check if
* we can really use the direct ops
*/
if (dma_nommu_dma_supported(dev, dev->coherent_dma_mask))
return __dma_nommu_alloc_coherent(dev, size, dma_handle,
flag, attrs);
/* Ok we can't ... do we have an iommu ? If not, fail */
iommu = get_iommu_table_base(dev);
if (!iommu)
return NULL;
/* Try to use the iommu */
return iommu_alloc_coherent(dev, iommu, size, dma_handle,
dev->coherent_dma_mask, flag,
dev_to_node(dev));
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 92 | 94.85% | 1 | 33.33% |
| Christoph Hellwig | 3 | 3.09% | 1 | 33.33% |
| Krzysztof Kozlowski | 2 | 2.06% | 1 | 33.33% |
| Total | 97 | 100.00% | 3 | 100.00% |
static void dma_nommu_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
struct iommu_table *iommu;
/* See comments in dma_nommu_alloc_coherent() */
if (dma_nommu_dma_supported(dev, dev->coherent_dma_mask))
return __dma_nommu_free_coherent(dev, size, vaddr, dma_handle,
attrs);
/* Maybe we used an iommu ... */
iommu = get_iommu_table_base(dev);
/* If we hit that we should have never allocated in the first
* place so how come we are freeing ?
*/
if (WARN_ON(!iommu))
return;
iommu_free_coherent(iommu, size, vaddr, dma_handle);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 79 | 92.94% | 1 | 33.33% |
| Christoph Hellwig | 4 | 4.71% | 1 | 33.33% |
| Krzysztof Kozlowski | 2 | 2.35% | 1 | 33.33% |
| Total | 85 | 100.00% | 3 | 100.00% |
int dma_nommu_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t handle, size_t size,
unsigned long attrs)
{
unsigned long pfn;
#ifdef CONFIG_NOT_COHERENT_CACHE
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
pfn = __dma_get_coherent_pfn((unsigned long)cpu_addr);
#else
pfn = page_to_pfn(virt_to_page(cpu_addr));
#endif
return remap_pfn_range(vma, vma->vm_start,
pfn + vma->vm_pgoff,
vma->vm_end - vma->vm_start,
vma->vm_page_prot);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Marek Szyprowski | 97 | 97.00% | 1 | 33.33% |
| Krzysztof Kozlowski | 2 | 2.00% | 1 | 33.33% |
| Christoph Hellwig | 1 | 1.00% | 1 | 33.33% |
| Total | 100 | 100.00% | 3 | 100.00% |
static int dma_nommu_map_sg(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction direction,
unsigned long attrs)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nents, i) {
sg->dma_address = sg_phys(sg) + get_dma_offset(dev);
sg->dma_length = sg->length;
if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
continue;
__dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
}
return nents;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 41 | 42.27% | 2 | 15.38% |
| Jens Axboe | 17 | 17.53% | 2 | 15.38% |
| Andrew Morton | 17 | 17.53% | 1 | 7.69% |
| Alexander Duyck | 7 | 7.22% | 1 | 7.69% |
| Michael Ellerman | 4 | 4.12% | 1 | 7.69% |
| Jeremy Kerr | 3 | 3.09% | 1 | 7.69% |
| Krzysztof Kozlowski | 2 | 2.06% | 1 | 7.69% |
| Stephen Rothwell | 2 | 2.06% | 1 | 7.69% |
| Mark Nelson | 2 | 2.06% | 1 | 7.69% |
| Becky Bruce | 1 | 1.03% | 1 | 7.69% |
| Christoph Hellwig | 1 | 1.03% | 1 | 7.69% |
| Total | 97 | 100.00% | 13 | 100.00% |
static void dma_nommu_unmap_sg(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction direction,
unsigned long attrs)
{
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 18 | 69.23% | 1 | 20.00% |
| Andrew Morton | 3 | 11.54% | 1 | 20.00% |
| Mark Nelson | 2 | 7.69% | 1 | 20.00% |
| Krzysztof Kozlowski | 2 | 7.69% | 1 | 20.00% |
| Christoph Hellwig | 1 | 3.85% | 1 | 20.00% |
| Total | 26 | 100.00% | 5 | 100.00% |
static u64 dma_nommu_get_required_mask(struct device *dev)
{
u64 end, mask;
end = memblock_end_of_DRAM() + get_dma_offset(dev);
mask = 1ULL << (fls64(end) - 1);
mask += mask - 1;
return mask;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Milton D. Miller II | 47 | 97.92% | 1 | 50.00% |
| Christoph Hellwig | 1 | 2.08% | 1 | 50.00% |
| Total | 48 | 100.00% | 2 | 100.00% |
static inline dma_addr_t dma_nommu_map_page(struct device *dev,
struct page *page,
unsigned long offset,
size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
BUG_ON(dir == DMA_NONE);
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
__dma_sync_page(page, offset, size, dir);
return page_to_phys(page) + offset + get_dma_offset(dev);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Becky Bruce | 60 | 83.33% | 2 | 40.00% |
| Alexander Duyck | 9 | 12.50% | 1 | 20.00% |
| Krzysztof Kozlowski | 2 | 2.78% | 1 | 20.00% |
| Christoph Hellwig | 1 | 1.39% | 1 | 20.00% |
| Total | 72 | 100.00% | 5 | 100.00% |
static inline void dma_nommu_unmap_page(struct device *dev,
dma_addr_t dma_address,
size_t size,
enum dma_data_direction direction,
unsigned long attrs)
{
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Becky Bruce | 22 | 88.00% | 1 | 33.33% |
| Krzysztof Kozlowski | 2 | 8.00% | 1 | 33.33% |
| Christoph Hellwig | 1 | 4.00% | 1 | 33.33% |
| Total | 25 | 100.00% | 3 | 100.00% |
#ifdef CONFIG_NOT_COHERENT_CACHE
static inline void dma_nommu_sync_sg(struct device *dev,
struct scatterlist *sgl, int nents,
enum dma_data_direction direction)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nents, i)
__dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Becky Bruce | 59 | 98.33% | 1 | 50.00% |
| Christoph Hellwig | 1 | 1.67% | 1 | 50.00% |
| Total | 60 | 100.00% | 2 | 100.00% |
static inline void dma_nommu_sync_single(struct device *dev,
dma_addr_t dma_handle, size_t size,
enum dma_data_direction direction)
{
__dma_sync(bus_to_virt(dma_handle), size, direction);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Becky Bruce | 33 | 97.06% | 1 | 50.00% |
| Christoph Hellwig | 1 | 2.94% | 1 | 50.00% |
| Total | 34 | 100.00% | 2 | 100.00% |
#endif
const struct dma_map_ops dma_nommu_ops = {
.alloc = dma_nommu_alloc_coherent,
.free = dma_nommu_free_coherent,
.mmap = dma_nommu_mmap_coherent,
.map_sg = dma_nommu_map_sg,
.unmap_sg = dma_nommu_unmap_sg,
.dma_supported = dma_nommu_dma_supported,
.map_page = dma_nommu_map_page,
.unmap_page = dma_nommu_unmap_page,
.get_required_mask = dma_nommu_get_required_mask,
#ifdef CONFIG_NOT_COHERENT_CACHE
.sync_single_for_cpu = dma_nommu_sync_single,
.sync_single_for_device = dma_nommu_sync_single,
.sync_sg_for_cpu = dma_nommu_sync_sg,
.sync_sg_for_device = dma_nommu_sync_sg,
#endif
};
EXPORT_SYMBOL(dma_nommu_ops);
int dma_set_coherent_mask(struct device *dev, u64 mask)
{
if (!dma_supported(dev, mask)) {
/*
* We need to special case the direct DMA ops which can
* support a fallback for coherent allocations. There
* is no dma_op->set_coherent_mask() so we have to do
* things the hard way:
*/
if (get_dma_ops(dev) != &dma_nommu_ops ||
get_iommu_table_base(dev) == NULL ||
!dma_iommu_dma_supported(dev, mask))
return -EIO;
}
dev->coherent_dma_mask = mask;
return 0;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 63 | 98.44% | 1 | 50.00% |
| Christoph Hellwig | 1 | 1.56% | 1 | 50.00% |
| Total | 64 | 100.00% | 2 | 100.00% |
EXPORT_SYMBOL(dma_set_coherent_mask);
#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
int dma_set_mask(struct device *dev, u64 dma_mask)
{
if (ppc_md.dma_set_mask)
return ppc_md.dma_set_mask(dev, dma_mask);
if (dev_is_pci(dev)) {
struct pci_dev *pdev = to_pci_dev(dev);
struct pci_controller *phb = pci_bus_to_host(pdev->bus);
if (phb->controller_ops.dma_set_mask)
return phb->controller_ops.dma_set_mask(pdev, dma_mask);
}
if (!dev->dma_mask || !dma_supported(dev, dma_mask))
return -EIO;
*dev->dma_mask = dma_mask;
return 0;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Daniel Axtens | 51 | 46.79% | 1 | 33.33% |
| Benjamin Herrenschmidt | 35 | 32.11% | 1 | 33.33% |
| Christoph Hellwig | 23 | 21.10% | 1 | 33.33% |
| Total | 109 | 100.00% | 3 | 100.00% |
EXPORT_SYMBOL(dma_set_mask);
u64 __dma_get_required_mask(struct device *dev)
{
const struct dma_map_ops *dma_ops = get_dma_ops(dev);
if (unlikely(dma_ops == NULL))
return 0;
if (dma_ops->get_required_mask)
return dma_ops->get_required_mask(dev);
return DMA_BIT_MASK(8 * sizeof(dma_addr_t));
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Milton D. Miller II | 56 | 96.55% | 2 | 50.00% |
| Bart Van Assche | 1 | 1.72% | 1 | 25.00% |
| Gavin Shan | 1 | 1.72% | 1 | 25.00% |
| Total | 58 | 100.00% | 4 | 100.00% |
u64 dma_get_required_mask(struct device *dev)
{
if (ppc_md.dma_get_required_mask)
return ppc_md.dma_get_required_mask(dev);
if (dev_is_pci(dev)) {
struct pci_dev *pdev = to_pci_dev(dev);
struct pci_controller *phb = pci_bus_to_host(pdev->bus);
if (phb->controller_ops.dma_get_required_mask)
return phb->controller_ops.dma_get_required_mask(pdev);
}
return __dma_get_required_mask(dev);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Andrew Donnellan | 49 | 62.03% | 1 | 50.00% |
| Gavin Shan | 30 | 37.97% | 1 | 50.00% |
| Total | 79 | 100.00% | 2 | 100.00% |
EXPORT_SYMBOL_GPL(dma_get_required_mask);
static int __init dma_init(void)
{
dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
#ifdef CONFIG_PCI
dma_debug_add_bus(&pci_bus_type);
#endif
#ifdef CONFIG_IBMVIO
dma_debug_add_bus(&vio_bus_type);
#endif
return 0;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Anton Blanchard | 22 | 56.41% | 1 | 50.00% |
| FUJITA Tomonori | 17 | 43.59% | 1 | 50.00% |
| Total | 39 | 100.00% | 2 | 100.00% |
fs_initcall(dma_init);
Overall Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| Benjamin Herrenschmidt | 544 | 35.30% | 10 | 21.28% |
| Becky Bruce | 246 | 15.96% | 5 | 10.64% |
| Scott Wood | 141 | 9.15% | 2 | 4.26% |
| Milton D. Miller II | 112 | 7.27% | 2 | 4.26% |
| Marek Szyprowski | 101 | 6.55% | 1 | 2.13% |
| Christoph Hellwig | 58 | 3.76% | 3 | 6.38% |
| Andrew Morton | 53 | 3.44% | 1 | 2.13% |
| Daniel Axtens | 51 | 3.31% | 1 | 2.13% |
| Andrew Donnellan | 49 | 3.18% | 1 | 2.13% |
| FUJITA Tomonori | 32 | 2.08% | 3 | 6.38% |
| Gavin Shan | 31 | 2.01% | 1 | 2.13% |
| Anton Blanchard | 28 | 1.82% | 1 | 2.13% |
| Krzysztof Kozlowski | 18 | 1.17% | 1 | 2.13% |
| Jens Axboe | 17 | 1.10% | 2 | 4.26% |
| Michael Ellerman | 16 | 1.04% | 3 | 6.38% |
| Alexander Duyck | 16 | 1.04% | 1 | 2.13% |
| Jeremy Kerr | 6 | 0.39% | 1 | 2.13% |
| Andrzej Pietrasiewicz | 6 | 0.39% | 1 | 2.13% |
| Mark Nelson | 4 | 0.26% | 1 | 2.13% |
| Tejun Heo | 3 | 0.19% | 1 | 2.13% |
| Paul Gortmaker | 3 | 0.19% | 1 | 2.13% |
| Bart Van Assche | 3 | 0.19% | 2 | 4.26% |
| Stephen Rothwell | 2 | 0.13% | 1 | 2.13% |
| Yinghai Lu | 1 | 0.06% | 1 | 2.13% |
| Total | 1541 | 100.00% | 47 | 100.00% |
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.