Release 4.11 drivers/iommu/tegra-smmu.c

/*
 * Copyright (C) 2011-2014 NVIDIA CORPORATION.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/bitops.h>
#include <linux/debugfs.h>
#include <linux/err.h>
#include <linux/iommu.h>
#include <linux/kernel.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/slab.h>

#include <soc/tegra/ahb.h>
#include <soc/tegra/mc.h>


struct tegra_smmu {
	
void __iomem *regs;
	
struct device *dev;

	
struct tegra_mc *mc;
	
const struct tegra_smmu_soc *soc;

	
unsigned long pfn_mask;
	
unsigned long tlb_mask;

	
unsigned long *asids;
	
struct mutex lock;

	
struct list_head list;

	
struct dentry *debugfs;
};


struct tegra_smmu_as {
	
struct iommu_domain domain;
	
struct tegra_smmu *smmu;
	
unsigned int use_count;
	
u32 *count;
	
struct page **pts;
	
struct page *pd;
	
dma_addr_t pd_dma;
	
unsigned id;
	
u32 attr;
};



static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom)
{
	return container_of(dom, struct tegra_smmu_as, domain);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Joerg Roedel
24
100.00%
1
100.00%
Total
24
100.00%
1
100.00%



static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,
			       unsigned long offset)
{
	writel(value, smmu->regs + offset);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
30
100.00%
1
100.00%
Total
30
100.00%
1
100.00%



static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
{
	return readl(smmu->regs + offset);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
26
100.00%
1
100.00%
Total
26
100.00%
1
100.00%


#define SMMU_CONFIG 0x010

#define  SMMU_CONFIG_ENABLE (1 << 0)


#define SMMU_TLB_CONFIG 0x14

#define  SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)

#define  SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)

#define  SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \
	((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)


#define SMMU_PTC_CONFIG 0x18

#define  SMMU_PTC_CONFIG_ENABLE (1 << 29)

#define  SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24)

#define  SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f)


#define SMMU_PTB_ASID 0x01c

#define  SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)


#define SMMU_PTB_DATA 0x020

#define  SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))


#define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))


#define SMMU_TLB_FLUSH 0x030

#define  SMMU_TLB_FLUSH_VA_MATCH_ALL     (0 << 0)

#define  SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)

#define  SMMU_TLB_FLUSH_VA_MATCH_GROUP   (3 << 0)

#define  SMMU_TLB_FLUSH_ASID(x)          (((x) & 0x7f) << 24)

#define  SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
                                          SMMU_TLB_FLUSH_VA_MATCH_SECTION)

#define  SMMU_TLB_FLUSH_VA_GROUP(addr)   ((((addr) & 0xffffc000) >> 12) | \
                                          SMMU_TLB_FLUSH_VA_MATCH_GROUP)

#define  SMMU_TLB_FLUSH_ASID_MATCH       (1 << 31)


#define SMMU_PTC_FLUSH 0x034

#define  SMMU_PTC_FLUSH_TYPE_ALL (0 << 0)

#define  SMMU_PTC_FLUSH_TYPE_ADR (1 << 0)


#define SMMU_PTC_FLUSH_HI 0x9b8

#define  SMMU_PTC_FLUSH_HI_MASK 0x3

/* per-SWGROUP SMMU_*_ASID register */

#define SMMU_ASID_ENABLE (1 << 31)

#define SMMU_ASID_MASK 0x7f

#define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK)

/* page table definitions */

#define SMMU_NUM_PDE 1024

#define SMMU_NUM_PTE 1024


#define SMMU_SIZE_PD (SMMU_NUM_PDE * 4)

#define SMMU_SIZE_PT (SMMU_NUM_PTE * 4)


#define SMMU_PDE_SHIFT 22

#define SMMU_PTE_SHIFT 12


#define SMMU_PD_READABLE	(1 << 31)

#define SMMU_PD_WRITABLE	(1 << 30)

#define SMMU_PD_NONSECURE	(1 << 29)


#define SMMU_PDE_READABLE	(1 << 31)

#define SMMU_PDE_WRITABLE	(1 << 30)

#define SMMU_PDE_NONSECURE	(1 << 29)

#define SMMU_PDE_NEXT		(1 << 28)


#define SMMU_PTE_READABLE	(1 << 31)

#define SMMU_PTE_WRITABLE	(1 << 30)

#define SMMU_PTE_NONSECURE	(1 << 29)


#define SMMU_PDE_ATTR		(SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
                                 SMMU_PDE_NONSECURE)

#define SMMU_PTE_ATTR		(SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \
                                 SMMU_PTE_NONSECURE)



static unsigned int iova_pd_index(unsigned long iova)
{
	return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
24
100.00%
1
100.00%
Total
24
100.00%
1
100.00%



static unsigned int iova_pt_index(unsigned long iova)
{
	return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
24
100.00%
1
100.00%
Total
24
100.00%
1
100.00%



static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)
{
	addr >>= 12;
	return (addr & smmu->pfn_mask) == addr;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
29
100.00%
2
100.00%
Total
29
100.00%
2
100.00%



static dma_addr_t smmu_pde_to_dma(u32 pde)
{
	return pde << 12;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
14
100.00%
2
100.00%
Total
14
100.00%
2
100.00%



static void smmu_flush_ptc_all(struct tegra_smmu *smmu)
{
	smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
20
100.00%
1
100.00%
Total
20
100.00%
1
100.00%



static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,
				  unsigned long offset)
{
	u32 value;

	offset &= ~(smmu->mc->soc->atom_size - 1);

	if (smmu->mc->soc->num_address_bits > 32) {
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
		value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK;
#else
		value = 0;
#endif
		smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
	}

	value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
	smmu_writel(smmu, value, SMMU_PTC_FLUSH);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
66
66.00%
1
33.33%
Hiroshi Doyu
29
29.00%
1
33.33%
Russell King
5
5.00%
1
33.33%
Total
100
100.00%
3
100.00%



static inline void smmu_flush_tlb(struct tegra_smmu *smmu)
{
	smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
15
71.43%
1
50.00%
Hiroshi Doyu
6
28.57%
1
50.00%
Total
21
100.00%
2
100.00%



static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,
				       unsigned long asid)
{
	u32 value;

	value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
		SMMU_TLB_FLUSH_VA_MATCH_ALL;
	smmu_writel(smmu, value, SMMU_TLB_FLUSH);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
30
76.92%
1
50.00%
Hiroshi Doyu
9
23.08%
1
50.00%
Total
39
100.00%
2
100.00%



static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,
					  unsigned long asid,
					  unsigned long iova)
{
	u32 value;

	value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
		SMMU_TLB_FLUSH_VA_SECTION(iova);
	smmu_writel(smmu, value, SMMU_TLB_FLUSH);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
33
71.74%
1
50.00%
Hiroshi Doyu
13
28.26%
1
50.00%
Total
46
100.00%
2
100.00%



static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,
					unsigned long asid,
					unsigned long iova)
{
	u32 value;

	value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
		SMMU_TLB_FLUSH_VA_GROUP(iova);
	smmu_writel(smmu, value, SMMU_TLB_FLUSH);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
36
78.26%
1
50.00%
Hiroshi Doyu
10
21.74%
1
50.00%
Total
46
100.00%
2
100.00%



static inline void smmu_flush(struct tegra_smmu *smmu)
{
	smmu_readl(smmu, SMMU_CONFIG);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
14
73.68%
1
50.00%
Hiroshi Doyu
5
26.32%
1
50.00%
Total
19
100.00%
2
100.00%



static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)
{
	unsigned long id;

	mutex_lock(&smmu->lock);

	id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);
	if (id >= smmu->soc->num_asids) {
		mutex_unlock(&smmu->lock);
		return -ENOSPC;
	}

	set_bit(id, smmu->asids);
	*idp = id;

	mutex_unlock(&smmu->lock);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
54
58.70%
1
50.00%
Hiroshi Doyu
38
41.30%
1
50.00%
Total
92
100.00%
2
100.00%



static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)
{
	mutex_lock(&smmu->lock);
	clear_bit(id, smmu->asids);
	mutex_unlock(&smmu->lock);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Hiroshi Doyu
26
65.00%
1
50.00%
Thierry Reding
14
35.00%
1
50.00%
Total
40
100.00%
2
100.00%



static bool tegra_smmu_capable(enum iommu_cap cap)
{
	return false;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Hiroshi Doyu
8
61.54%
1
33.33%
Joerg Roedel
4
30.77%
1
33.33%
Thierry Reding
1
7.69%
1
33.33%
Total
13
100.00%
3
100.00%



static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
{
	struct tegra_smmu_as *as;

	if (type != IOMMU_DOMAIN_UNMANAGED)
		return NULL;

	as = kzalloc(sizeof(*as), GFP_KERNEL);
	if (!as)
		return NULL;

	as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;

	as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
	if (!as->pd) {
		kfree(as);
		return NULL;
	}

	as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);
	if (!as->count) {
		__free_page(as->pd);
		kfree(as);
		return NULL;
	}

	as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);
	if (!as->pts) {
		kfree(as->count);
		__free_page(as->pd);
		kfree(as);
		return NULL;
	}

	/* setup aperture */
	as->domain.geometry.aperture_start = 0;
	as->domain.geometry.aperture_end = 0xffffffff;
	as->domain.geometry.force_aperture = true;

	return &as->domain;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
64
30.05%
2
25.00%
Hiroshi Doyu
62
29.11%
1
12.50%
Russell King
60
28.17%
3
37.50%
Joerg Roedel
27
12.68%
2
25.00%
Total
213
100.00%
8
100.00%



static void tegra_smmu_domain_free(struct iommu_domain *domain)
{
	struct tegra_smmu_as *as = to_smmu_as(domain);

	/* TODO: free page directory and page tables */

	kfree(as);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Hiroshi Doyu
15
55.56%
1
33.33%
Thierry Reding
8
29.63%
1
33.33%
Joerg Roedel
4
14.81%
1
33.33%
Total
27
100.00%
3
100.00%


static const struct tegra_smmu_swgroup *

tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)
{
	const struct tegra_smmu_swgroup *group = NULL;
	unsigned int i;

	for (i = 0; i < smmu->soc->num_swgroups; i++) {
		if (smmu->soc->swgroups[i].swgroup == swgroup) {
			group = &smmu->soc->swgroups[i];
			break;
		}
	}

	return group;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
76
92.68%
1
50.00%
Hiroshi Doyu
6
7.32%
1
50.00%
Total
82
100.00%
2
100.00%



static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,
			      unsigned int asid)
{
	const struct tegra_smmu_swgroup *group;
	unsigned int i;
	u32 value;

	for (i = 0; i < smmu->soc->num_clients; i++) {
		const struct tegra_mc_client *client = &smmu->soc->clients[i];

		if (client->swgroup != swgroup)
			continue;

		value = smmu_readl(smmu, client->smmu.reg);
		value |= BIT(client->smmu.bit);
		smmu_writel(smmu, value, client->smmu.reg);
	}

	group = tegra_smmu_find_swgroup(smmu, swgroup);
	if (group) {
		value = smmu_readl(smmu, group->reg);
		value &= ~SMMU_ASID_MASK;
		value |= SMMU_ASID_VALUE(asid);
		value |= SMMU_ASID_ENABLE;
		smmu_writel(smmu, value, group->reg);
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
128
77.11%
1
50.00%
Hiroshi Doyu
38
22.89%
1
50.00%
Total
166
100.00%
2
100.00%



static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,
			       unsigned int asid)
{
	const struct tegra_smmu_swgroup *group;
	unsigned int i;
	u32 value;

	group = tegra_smmu_find_swgroup(smmu, swgroup);
	if (group) {
		value = smmu_readl(smmu, group->reg);
		value &= ~SMMU_ASID_MASK;
		value |= SMMU_ASID_VALUE(asid);
		value &= ~SMMU_ASID_ENABLE;
		smmu_writel(smmu, value, group->reg);
	}

	for (i = 0; i < smmu->soc->num_clients; i++) {
		const struct tegra_mc_client *client = &smmu->soc->clients[i];

		if (client->swgroup != swgroup)
			continue;

		value = smmu_readl(smmu, client->smmu.reg);
		value &= ~BIT(client->smmu.bit);
		smmu_writel(smmu, value, client->smmu.reg);
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
111
66.07%
1
25.00%
Hiroshi Doyu
57
33.93%
3
75.00%
Total
168
100.00%
4
100.00%



static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
				 struct tegra_smmu_as *as)
{
	u32 value;
	int err;

	if (as->use_count > 0) {
		as->use_count++;
		return 0;
	}

	as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,
				  DMA_TO_DEVICE);
	if (dma_mapping_error(smmu->dev, as->pd_dma))
		return -ENOMEM;

	/* We can't handle 64-bit DMA addresses */
	if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {
		err = -ENOMEM;
		goto err_unmap;
	}

	err = tegra_smmu_alloc_asid(smmu, &as->id);
	if (err < 0)
		goto err_unmap;

	smmu_flush_ptc(smmu, as->pd_dma, 0);
	smmu_flush_tlb_asid(smmu, as->id);

	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
	value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);
	smmu_writel(smmu, value, SMMU_PTB_DATA);
	smmu_flush(smmu);

	as->smmu = smmu;
	as->use_count++;

	return 0;

err_unmap:
	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
	return err;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
85
39.35%
1
20.00%
Thierry Reding
80
37.04%
1
20.00%
Hiroshi Doyu
50
23.15%
2
40.00%
Joerg Roedel
1
0.46%
1
20.00%
Total
216
100.00%
5
100.00%



static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
				    struct tegra_smmu_as *as)
{
	if (--as->use_count > 0)
		return;

	tegra_smmu_free_asid(smmu, as->id);

	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);

	as->smmu = NULL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
25
44.64%
1
33.33%
Hiroshi Doyu
16
28.57%
1
33.33%
Russell King
15
26.79%
1
33.33%
Total
56
100.00%
3
100.00%



static int tegra_smmu_attach_dev(struct iommu_domain *domain,
				 struct device *dev)
{
	struct tegra_smmu *smmu = dev->archdata.iommu;
	struct tegra_smmu_as *as = to_smmu_as(domain);
	struct device_node *np = dev->of_node;
	struct of_phandle_args args;
	unsigned int index = 0;
	int err = 0;

	while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
					   &args)) {
		unsigned int swgroup = args.args[0];

		if (args.np != smmu->dev->of_node) {
			of_node_put(args.np);
			continue;
		}

		of_node_put(args.np);

		err = tegra_smmu_as_prepare(smmu, as);
		if (err < 0)
			return err;

		tegra_smmu_enable(smmu, swgroup, as->id);
		index++;
	}

	if (index == 0)
		return -ENODEV;

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
109
66.06%
1
14.29%
Hiroshi Doyu
51
30.91%
4
57.14%
Joerg Roedel
3
1.82%
1
14.29%
Al Viro
2
1.21%
1
14.29%
Total
165
100.00%
7
100.00%



static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
{
	struct tegra_smmu_as *as = to_smmu_as(domain);
	struct device_node *np = dev->of_node;
	struct tegra_smmu *smmu = as->smmu;
	struct of_phandle_args args;
	unsigned int index = 0;

	while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
					   &args)) {
		unsigned int swgroup = args.args[0];

		if (args.np != smmu->dev->of_node) {
			of_node_put(args.np);
			continue;
		}

		of_node_put(args.np);

		tegra_smmu_disable(smmu, swgroup, as->id);
		tegra_smmu_as_unprepare(smmu, as);
		index++;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
111
82.84%
1
33.33%
Hiroshi Doyu
20
14.93%
1
33.33%
Joerg Roedel
3
2.24%
1
33.33%
Total
134
100.00%
3
100.00%



static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
			       u32 value)
{
	unsigned int pd_index = iova_pd_index(iova);
	struct tegra_smmu *smmu = as->smmu;
	u32 *pd = page_address(as->pd);
	unsigned long offset = pd_index * sizeof(*pd);

	/* Set the page directory entry first */
	pd[pd_index] = value;

	/* The flush the page directory entry from caches */
	dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,
					 sizeof(*pd), DMA_TO_DEVICE);

	/* And flush the iommu */
	smmu_flush_ptc(smmu, as->pd_dma, offset);
	smmu_flush_tlb_section(smmu, as->id, iova);
	smmu_flush(smmu);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
117
100.00%
1
100.00%
Total
117
100.00%
1
100.00%



static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova)
{
	u32 *pt = page_address(pt_page);

	return pt + iova_pt_index(iova);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
33
100.00%
1
100.00%
Total
33
100.00%
1
100.00%



static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,
				  dma_addr_t *dmap)
{
	unsigned int pd_index = iova_pd_index(iova);
	struct page *pt_page;
	u32 *pd;

	pt_page = as->pts[pd_index];
	if (!pt_page)
		return NULL;

	pd = page_address(as->pd);
	*dmap = smmu_pde_to_dma(pd[pd_index]);

	return tegra_smmu_pte_offset(pt_page, iova);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
83
100.00%
3
100.00%
Total
83
100.00%
3
100.00%



static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
		       dma_addr_t *dmap)
{
	unsigned int pde = iova_pd_index(iova);
	struct tegra_smmu *smmu = as->smmu;

	if (!as->pts[pde]) {
		struct page *page;
		dma_addr_t dma;

		page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
		if (!page)
			return NULL;

		dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,
				   DMA_TO_DEVICE);
		if (dma_mapping_error(smmu->dev, dma)) {
			__free_page(page);
			return NULL;
		}

		if (!smmu_dma_addr_valid(smmu, dma)) {
			dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,
				       DMA_TO_DEVICE);
			__free_page(page);
			return NULL;
		}

		as->pts[pde] = page;

		tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |
							      SMMU_PDE_NEXT));

		*dmap = dma;
	} else {
		u32 *pd = page_address(as->pd);

		*dmap = smmu_pde_to_dma(pd[pde]);
	}

	return tegra_smmu_pte_offset(as->pts[pde], iova);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
143
66.51%
7
63.64%
Thierry Reding
48
22.33%
1
9.09%
Hiroshi Doyu
21
9.77%
2
18.18%
Al Viro
3
1.40%
1
9.09%
Total
215
100.00%
11
100.00%



static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)
{
	unsigned int pd_index = iova_pd_index(iova);

	as->count[pd_index]++;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
26
81.25%
2
50.00%
Thierry Reding
5
15.62%
1
25.00%
Hiroshi Doyu
1
3.12%
1
25.00%
Total
32
100.00%
4
100.00%



static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)
{
	unsigned int pde = iova_pd_index(iova);
	struct page *page = as->pts[pde];

	/*
         * When no entries in this page table are used anymore, return the
         * memory page to the system.
         */
	if (--as->count[pde] == 0) {
		struct tegra_smmu *smmu = as->smmu;
		u32 *pd = page_address(as->pd);
		dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]);

		tegra_smmu_set_pde(as, iova, 0);

		dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);
		__free_page(page);
		as->pts[pde] = NULL;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
80
67.80%
6
66.67%
Thierry Reding
26
22.03%
1
11.11%
Hiroshi Doyu
12
10.17%
2
22.22%
Total
118
100.00%
9
100.00%



static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
			       u32 *pte, dma_addr_t pte_dma, u32 val)
{
	struct tegra_smmu *smmu = as->smmu;
	unsigned long offset = offset_in_page(pte);

	*pte = val;

	dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,
					 4, DMA_TO_DEVICE);
	smmu_flush_ptc(smmu, pte_dma, offset);
	smmu_flush_tlb_group(smmu, as->id, iova);
	smmu_flush(smmu);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Russell King
88
100.00%
2
100.00%
Total
88
100.00%
2
100.00%



static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
			  phys_addr_t paddr, size_t size, int prot)
{
	struct tegra_smmu_as *as = to_smmu_as(domain);
	dma_addr_t pte_dma;
	u32 *pte;

	pte = as_get_pte(as, iova, &pte_dma);
	if (!pte)
		return -ENOMEM;

	/* If we aren't overwriting a pre-existing entry, increment use */
	if (*pte == 0)
		tegra_smmu_pte_get_use(as, iova);

	tegra_smmu_set_pte(as, iova, pte, pte_dma,
			   __phys_to_pfn(paddr) | SMMU_PTE_ATTR);

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
43
43.88%
1
14.29%
Russell King
27
27.55%
3
42.86%
Hiroshi Doyu
25
25.51%
2
28.57%
Joerg Roedel
3
3.06%
1
14.29%
Total
98
100.00%
7
100.00%



static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
			       size_t size)
{
	struct tegra_smmu_as *as = to_smmu_as(domain);
	dma_addr_t pte_dma;
	u32 *pte;

	pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
	if (!pte || !*pte)
		return 0;

	tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);
	tegra_smmu_pte_put_use(as, iova);

	return size;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
42
51.22%
1
12.50%
Russell King
21
25.61%
4
50.00%
Hiroshi Doyu
16
19.51%
2
25.00%
Joerg Roedel
3
3.66%
1
12.50%
Total
82
100.00%
8
100.00%



static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
					   dma_addr_t iova)
{
	struct tegra_smmu_as *as = to_smmu_as(domain);
	unsigned long pfn;
	dma_addr_t pte_dma;
	u32 *pte;

	pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
	if (!pte || !*pte)
		return 0;

	pfn = *pte & as->smmu->pfn_mask;

	return PFN_PHYS(pfn);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
35
46.05%
2
22.22%
Hiroshi Doyu
21
27.63%
3
33.33%
Russell King
17
22.37%
3
33.33%
Joerg Roedel
3
3.95%
1
11.11%
Total
76
100.00%
9
100.00%



static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
{
	struct platform_device *pdev;
	struct tegra_mc *mc;

	pdev = of_find_device_by_node(np);
	if (!pdev)
		return NULL;

	mc = platform_get_drvdata(pdev);
	if (!mc)
		return NULL;

	return mc->smmu;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
37
63.79%
1
25.00%
Hiroshi Doyu
21
36.21%
3
75.00%
Total
58
100.00%
4
100.00%



static int tegra_smmu_add_device(struct device *dev)
{
	struct device_node *np = dev->of_node;
	struct of_phandle_args args;
	unsigned int index = 0;

	while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
					  &args) == 0) {
		struct tegra_smmu *smmu;

		smmu = tegra_smmu_find(args.np);
		if (smmu) {
			/*
                         * Only a single IOMMU master interface is currently
                         * supported by the Linux kernel, so abort after the
                         * first match.
                         */
			dev->archdata.iommu = smmu;
			break;
		}

		index++;
	}

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
64
74.42%
1
25.00%
Hiroshi Doyu
22
25.58%
3
75.00%
Total
86
100.00%
4
100.00%



static void tegra_smmu_remove_device(struct device *dev)
{
	dev->archdata.iommu = NULL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
18
94.74%
1
50.00%
Hiroshi Doyu
1
5.26%
1
50.00%
Total
19
100.00%
2
100.00%


static const struct iommu_ops tegra_smmu_ops = {
	.capable = tegra_smmu_capable,
	.domain_alloc = tegra_smmu_domain_alloc,
	.domain_free = tegra_smmu_domain_free,
	.attach_dev = tegra_smmu_attach_dev,
	.detach_dev = tegra_smmu_detach_dev,
	.add_device = tegra_smmu_add_device,
	.remove_device = tegra_smmu_remove_device,
	.map = tegra_smmu_map,
	.unmap = tegra_smmu_unmap,
	.map_sg = default_iommu_map_sg,
	.iova_to_phys = tegra_smmu_iova_to_phys,

	.pgsize_bitmap = SZ_4K,
};



static void tegra_smmu_ahb_enable(void)
{
	static const struct of_device_id ahb_match[] = {
		{ .compatible = "nvidia,tegra30-ahb", },
		{ }
	};
	struct device_node *ahb;

	ahb = of_find_matching_node(NULL, ahb_match);
	if (ahb) {
		tegra_ahb_enable_smmu(ahb);
		of_node_put(ahb);
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
45
80.36%
1
25.00%
Hiroshi Doyu
11
19.64%
3
75.00%
Total
56
100.00%
4
100.00%



static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)
{
	struct tegra_smmu *smmu = s->private;
	unsigned int i;
	u32 value;

	seq_printf(s, "swgroup    enabled  ASID\n");
	seq_printf(s, "------------------------\n");

	for (i = 0; i < smmu->soc->num_swgroups; i++) {
		const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];
		const char *status;
		unsigned int asid;

		value = smmu_readl(smmu, group->reg);

		if (value & SMMU_ASID_ENABLE)
			status = "yes";
		else
			status = "no";

		asid = value & SMMU_ASID_MASK;

		seq_printf(s, "%-9s  %-7s  %#04x\n", group->name, status,
			   asid);
	}

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
139
100.00%
1
100.00%
Total
139
100.00%
1
100.00%



static int tegra_smmu_swgroups_open(struct inode *inode, struct file *file)
{
	return single_open(file, tegra_smmu_swgroups_show, inode->i_private);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
28
100.00%
1
100.00%
Total
28
100.00%
1
100.00%


static const struct file_operations tegra_smmu_swgroups_fops = {
	.open = tegra_smmu_swgroups_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};



static int tegra_smmu_clients_show(struct seq_file *s, void *data)
{
	struct tegra_smmu *smmu = s->private;
	unsigned int i;
	u32 value;

	seq_printf(s, "client       enabled\n");
	seq_printf(s, "--------------------\n");

	for (i = 0; i < smmu->soc->num_clients; i++) {
		const struct tegra_mc_client *client = &smmu->soc->clients[i];
		const char *status;

		value = smmu_readl(smmu, client->smmu.reg);

		if (value & BIT(client->smmu.bit))
			status = "yes";
		else
			status = "no";

		seq_printf(s, "%-12s %s\n", client->name, status);
	}

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
136
100.00%
1
100.00%
Total
136
100.00%
1
100.00%



static int tegra_smmu_clients_open(struct inode *inode, struct file *file)
{
	return single_open(file, tegra_smmu_clients_show, inode->i_private);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
28
100.00%
1
100.00%
Total
28
100.00%
1
100.00%


static const struct file_operations tegra_smmu_clients_fops = {
	.open = tegra_smmu_clients_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};



static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
{
	smmu->debugfs = debugfs_create_dir("smmu", NULL);
	if (!smmu->debugfs)
		return;

	debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
			    &tegra_smmu_swgroups_fops);
	debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,
			    &tegra_smmu_clients_fops);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
62
100.00%
1
100.00%
Total
62
100.00%
1
100.00%



static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)
{
	debugfs_remove_recursive(smmu->debugfs);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
18
100.00%
1
100.00%
Total
18
100.00%
1
100.00%



struct tegra_smmu *tegra_smmu_probe(struct device *dev,
				    const struct tegra_smmu_soc *soc,
				    struct tegra_mc *mc)
{
	struct tegra_smmu *smmu;
	size_t size;
	u32 value;
	int err;

	/* This can happen on Tegra20 which doesn't have an SMMU */
	if (!soc)
		return NULL;

	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
	if (!smmu)
		return ERR_PTR(-ENOMEM);

	/*
         * This is a bit of a hack. Ideally we'd want to simply return this
         * value. However the IOMMU registration process will attempt to add
         * all devices to the IOMMU when bus_set_iommu() is called. In order
         * not to rely on global variables to track the IOMMU instance, we
         * set it here so that it can be looked up from the .add_device()
         * callback via the IOMMU device's .drvdata field.
         */
	mc->smmu = smmu;

	size = BITS_TO_LONGS(soc->num_asids) * sizeof(long);

	smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);
	if (!smmu->asids)
		return ERR_PTR(-ENOMEM);

	mutex_init(&smmu->lock);

	smmu->regs = mc->regs;
	smmu->soc = soc;
	smmu->dev = dev;
	smmu->mc = mc;

	smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1;
	dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
		mc->soc->num_address_bits, smmu->pfn_mask);
	smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1;
	dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,
		smmu->tlb_mask);

	value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);

	if (soc->supports_request_limit)
		value |= SMMU_PTC_CONFIG_REQ_LIMIT(8);

	smmu_writel(smmu, value, SMMU_PTC_CONFIG);

	value = SMMU_TLB_CONFIG_HIT_UNDER_MISS |
		SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);

	if (soc->supports_round_robin_arbitration)
		value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION;

	smmu_writel(smmu, value, SMMU_TLB_CONFIG);

	smmu_flush_ptc_all(smmu);
	smmu_flush_tlb(smmu);
	smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
	smmu_flush(smmu);

	tegra_smmu_ahb_enable();

	err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
	if (err < 0)
		return ERR_PTR(err);

	if (IS_ENABLED(CONFIG_DEBUG_FS))
		tegra_smmu_debugfs_init(smmu);

	return smmu;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
247
71.39%
4
40.00%
Hiroshi Doyu
98
28.32%
5
50.00%
Russell King
1
0.29%
1
10.00%
Total
346
100.00%
10
100.00%



void tegra_smmu_remove(struct tegra_smmu *smmu)
{
	if (IS_ENABLED(CONFIG_DEBUG_FS))
		tegra_smmu_debugfs_exit(smmu);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
22
100.00%
1
100.00%
Total
22
100.00%
1
100.00%

Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
Thierry Reding
2415
56.70%
7
18.42%
Russell King
926
21.74%
13
34.21%
Hiroshi Doyu
801
18.81%
11
28.95%
Joerg Roedel
79
1.85%
4
10.53%
Stephen Warren
33
0.77%
1
2.63%
Al Viro
5
0.12%
2
5.26%
Total
4259
100.00%
38
100.00%
Directory: drivers/iommu

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.