cregit-Linux how code gets into the kernel

Release 4.14 drivers/iommu/arm-smmu.c

Directory: drivers/iommu
/*
 * IOMMU API for ARM architected SMMU implementations.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) 2013 ARM Limited
 *
 * Author: Will Deacon <will.deacon@arm.com>
 *
 * This driver currently supports:
 *      - SMMUv1 and v2 implementations
 *      - Stream-matching and stream-indexing
 *      - v7/v8 long-descriptor format
 *      - Non-secure access to the SMMU
 *      - Context fault reporting
 *      - Extended Stream ID (16 bit)
 */


#define pr_fmt(fmt) "arm-smmu: " fmt

#include <linux/acpi.h>
#include <linux/acpi_iort.h>
#include <linux/atomic.h>
#include <linux/delay.h>
#include <linux/dma-iommu.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/io-64-nonatomic-hi-lo.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_device.h>
#include <linux/of_iommu.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>

#include <linux/amba/bus.h>

#include "io-pgtable.h"
#include "arm-smmu-regs.h"


#define ARM_MMU500_ACTLR_CPRE		(1 << 1)


#define ARM_MMU500_ACR_CACHE_LOCK	(1 << 26)

#define ARM_MMU500_ACR_SMTNMB_TLBEN	(1 << 8)


#define TLB_LOOP_TIMEOUT		1000000	
/* 1s! */

#define TLB_SPIN_COUNT			10

/* Maximum number of context banks per SMMU */

#define ARM_SMMU_MAX_CBS		128

/* SMMU global address space */

#define ARM_SMMU_GR0(smmu)		((smmu)->base)

#define ARM_SMMU_GR1(smmu)		((smmu)->base + (1 << (smmu)->pgshift))

/*
 * SMMU global address space with conditional offset to access secure
 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
 * nsGFSYNR0: 0x450)
 */

#define ARM_SMMU_GR0_NS(smmu)						\
	((smmu)->base +                                                 \
                ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
                        ? 0x400 : 0))

/*
 * Some 64-bit registers only make sense to write atomically, but in such
 * cases all the data relevant to AArch32 formats lies within the lower word,
 * therefore this actually makes more sense than it might first appear.
 */
#ifdef CONFIG_64BIT

#define smmu_write_atomic_lq		writeq_relaxed
#else

#define smmu_write_atomic_lq		writel_relaxed
#endif

/* Translation context bank */

#define ARM_SMMU_CB(smmu, n)	((smmu)->cb_base + ((n) << (smmu)->pgshift))


#define MSI_IOVA_BASE			0x8000000

#define MSI_IOVA_LENGTH			0x100000


static int force_stage;
module_param(force_stage, int, S_IRUGO);
MODULE_PARM_DESC(force_stage,
	"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");

static bool disable_bypass;
module_param(disable_bypass, bool, S_IRUGO);
MODULE_PARM_DESC(disable_bypass,
	"Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");


enum arm_smmu_arch_version {
	
ARM_SMMU_V1,
	
ARM_SMMU_V1_64K,
	
ARM_SMMU_V2,
};


enum arm_smmu_implementation {
	
GENERIC_SMMU,
	
ARM_MMU500,
	
CAVIUM_SMMUV2,
};

/* Until ACPICA headers cover IORT rev. C */
#ifndef ACPI_IORT_SMMU_CORELINK_MMU401

#define ACPI_IORT_SMMU_CORELINK_MMU401	0x4
#endif
#ifndef ACPI_IORT_SMMU_CAVIUM_THUNDERX

#define ACPI_IORT_SMMU_CAVIUM_THUNDERX	0x5
#endif


struct arm_smmu_s2cr {
	
struct iommu_group		*group;
	
int				count;
	
enum arm_smmu_s2cr_type		type;
	
enum arm_smmu_s2cr_privcfg	privcfg;
	
u8				cbndx;
};


#define s2cr_init_val (struct arm_smmu_s2cr){                         \
        .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
}


struct arm_smmu_smr {
	
u16				mask;
	
u16				id;
	
bool				valid;
};


struct arm_smmu_cb {
	
u64				ttbr[2];
	
u32				tcr[2];
	
u32				mair[2];
	
struct arm_smmu_cfg		*cfg;
};


struct arm_smmu_master_cfg {
	
struct arm_smmu_device		*smmu;
	
s16				smendx[];
};

#define INVALID_SMENDX			-1

#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)

#define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)

#define fwspec_smendx(fw, i) \
	(i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])

#define for_each_cfg_sme(fw, i, idx) \
	for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)


struct arm_smmu_device {
	
struct device			*dev;

	
void __iomem			*base;
	
void __iomem			*cb_base;
	
unsigned long			pgshift;


#define ARM_SMMU_FEAT_COHERENT_WALK	(1 << 0)

#define ARM_SMMU_FEAT_STREAM_MATCH	(1 << 1)

#define ARM_SMMU_FEAT_TRANS_S1		(1 << 2)

#define ARM_SMMU_FEAT_TRANS_S2		(1 << 3)

#define ARM_SMMU_FEAT_TRANS_NESTED	(1 << 4)

#define ARM_SMMU_FEAT_TRANS_OPS		(1 << 5)

#define ARM_SMMU_FEAT_VMID16		(1 << 6)

#define ARM_SMMU_FEAT_FMT_AARCH64_4K	(1 << 7)

#define ARM_SMMU_FEAT_FMT_AARCH64_16K	(1 << 8)

#define ARM_SMMU_FEAT_FMT_AARCH64_64K	(1 << 9)

#define ARM_SMMU_FEAT_FMT_AARCH32_L	(1 << 10)

#define ARM_SMMU_FEAT_FMT_AARCH32_S	(1 << 11)

#define ARM_SMMU_FEAT_EXIDS		(1 << 12)
	
u32				features;


#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
	
u32				options;
	
enum arm_smmu_arch_version	version;
	
enum arm_smmu_implementation	model;

	
u32				num_context_banks;
	
u32				num_s2_context_banks;
	DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
	
struct arm_smmu_cb		*cbs;
	
atomic_t			irptndx;

	
u32				num_mapping_groups;
	
u16				streamid_mask;
	
u16				smr_mask_mask;
	
struct arm_smmu_smr		*smrs;
	
struct arm_smmu_s2cr		*s2crs;
	
struct mutex			stream_map_mutex;

	
unsigned long			va_size;
	
unsigned long			ipa_size;
	
unsigned long			pa_size;
	
unsigned long			pgsize_bitmap;

	
u32				num_global_irqs;
	
u32				num_context_irqs;
	
unsigned int			*irqs;

	
u32				cavium_id_base; /* Specific to Cavium */

	
spinlock_t			global_sync_lock;

	/* IOMMU core code handle */
	
struct iommu_device		iommu;
};


enum arm_smmu_context_fmt {
	
ARM_SMMU_CTX_FMT_NONE,
	
ARM_SMMU_CTX_FMT_AARCH64,
	
ARM_SMMU_CTX_FMT_AARCH32_L,
	
ARM_SMMU_CTX_FMT_AARCH32_S,
};


struct arm_smmu_cfg {
	
u8				cbndx;
	
u8				irptndx;
	union {
		
u16			asid;
		
u16			vmid;
	};
	
u32				cbar;
	
enum arm_smmu_context_fmt	fmt;
};

#define INVALID_IRPTNDX			0xff


enum arm_smmu_domain_stage {
	
ARM_SMMU_DOMAIN_S1 = 0,
	
ARM_SMMU_DOMAIN_S2,
	
ARM_SMMU_DOMAIN_NESTED,
	
ARM_SMMU_DOMAIN_BYPASS,
};


struct arm_smmu_domain {
	
struct arm_smmu_device		*smmu;
	
struct io_pgtable_ops		*pgtbl_ops;
	
struct arm_smmu_cfg		cfg;
	
enum arm_smmu_domain_stage	stage;
	
struct mutex			init_mutex; /* Protects smmu pointer */
	
spinlock_t			cb_lock; /* Serialises ATS1* ops and TLB syncs */
	
struct iommu_domain		domain;
};


struct arm_smmu_option_prop {
	
u32 opt;
	
const char *prop;
};


static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);



static bool using_legacy_binding, using_generic_binding;


static struct arm_smmu_option_prop arm_smmu_options[] = {
	{ ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
	{ 0, NULL},
};


static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) { return container_of(dom, struct arm_smmu_domain, domain); }

Contributors

PersonTokensPropCommitsCommitProp
Joerg Roedel24100.00%1100.00%
Total24100.00%1100.00%


static void parse_driver_options(struct arm_smmu_device *smmu) { int i = 0; do { if (of_property_read_bool(smmu->dev->of_node, arm_smmu_options[i].prop)) { smmu->options |= arm_smmu_options[i].opt; dev_notice(smmu->dev, "option %s\n", arm_smmu_options[i].prop); } } while (arm_smmu_options[++i].opt); }

Contributors

PersonTokensPropCommitsCommitProp
Andreas Herrmann77100.00%1100.00%
Total77100.00%1100.00%


static struct device_node *dev_get_dev_node(struct device *dev) { if (dev_is_pci(dev)) { struct pci_bus *bus = to_pci_dev(dev)->bus; while (!pci_is_root_bus(bus)) bus = bus->parent; return of_node_get(bus->bridge->parent->of_node); } return of_node_get(dev->of_node); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon6291.18%266.67%
Robin Murphy68.82%133.33%
Total68100.00%3100.00%


static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) { *((__be32 *)data) = cpu_to_be32(alias); return 0; /* Continue walking */ }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy3391.67%150.00%
Will Deacon38.33%150.00%
Total36100.00%2100.00%


static int __find_legacy_master_phandle(struct device *dev, void *data) { struct of_phandle_iterator *it = *(void **)data; struct device_node *np = it->node; int err; of_for_each_phandle(it, err, dev->of_node, "mmu-masters", "#stream-id-cells", 0) if (it->node == np) { *(void **)data = dev; return 1; } it->node = np; return err == -ENOENT ? 0 : err; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy6266.67%150.00%
Will Deacon3133.33%150.00%
Total93100.00%2100.00%

static struct platform_driver arm_smmu_driver; static struct iommu_ops arm_smmu_ops;
static int arm_smmu_register_legacy_master(struct device *dev, struct arm_smmu_device **smmu) { struct device *smmu_dev; struct device_node *np; struct of_phandle_iterator it; void *data = &it; u32 *sids; __be32 pci_sid; int err; np = dev_get_dev_node(dev); if (!np || !of_find_property(np, "#stream-id-cells", NULL)) { of_node_put(np); return -ENODEV; } it.node = np; err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, __find_legacy_master_phandle); smmu_dev = data; of_node_put(np); if (err == 0) return -ENODEV; if (err < 0) return err; if (dev_is_pci(dev)) { /* "mmu-masters" assumes Stream ID == Requester ID */ pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, &pci_sid); it.cur = &pci_sid; it.cur_count = 1; } err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, &arm_smmu_ops); if (err) return err; sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); if (!sids) return -ENOMEM; *smmu = dev_get_drvdata(smmu_dev); of_phandle_iterator_args(&it, sids, it.cur_count); err = iommu_fwspec_add_ids(dev, sids, it.cur_count); kfree(sids); return err; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy18972.97%342.86%
Will Deacon6625.48%342.86%
Olav Haugan41.54%114.29%
Total259100.00%7100.00%


static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end) { int idx; do { idx = find_next_zero_bit(map, end, start); if (idx == end) return -ENOSPC; } while (test_and_set_bit(idx, map)); return idx; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon5087.72%133.33%
Robin Murphy712.28%266.67%
Total57100.00%3100.00%


static void __arm_smmu_free_bitmap(unsigned long *map, int idx) { clear_bit(idx, map); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon2095.24%150.00%
Olav Haugan14.76%150.00%
Total21100.00%2100.00%

/* Wait for any pending TLB invalidations to complete */
static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, void __iomem *sync, void __iomem *status) { unsigned int spin_cnt, delay; writel_relaxed(0, sync); for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE)) return; cpu_relax(); } udelay(delay); } dev_err_ratelimited(smmu->dev, "TLB sync timed out -- SMMU may be deadlocked\n"); }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy6366.32%457.14%
Will Deacon3233.68%342.86%
Total95100.00%7100.00%


static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) { void __iomem *base = ARM_SMMU_GR0(smmu); unsigned long flags; spin_lock_irqsave(&smmu->global_sync_lock, flags); __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC, base + ARM_SMMU_GR0_sTLBGSTATUS); spin_unlock_irqrestore(&smmu->global_sync_lock, flags); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon3356.90%457.14%
Robin Murphy2543.10%342.86%
Total58100.00%7100.00%


static void arm_smmu_tlb_sync_context(void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_device *smmu = smmu_domain->smmu; void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx); unsigned long flags; spin_lock_irqsave(&smmu_domain->cb_lock, flags); __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC, base + ARM_SMMU_CB_TLBSTATUS); spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon5772.15%555.56%
Robin Murphy2126.58%333.33%
Tirumalesh Chalamarla11.27%111.11%
Total79100.00%9100.00%


static void arm_smmu_tlb_sync_vmid(void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; arm_smmu_tlb_sync_global(smmu_domain->smmu); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon1979.17%266.67%
Robin Murphy520.83%133.33%
Total24100.00%3100.00%


static void arm_smmu_tlb_inv_context_s1(void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx); writel_relaxed(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID); arm_smmu_tlb_sync_context(cookie); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon4271.19%466.67%
Robin Murphy1728.81%233.33%
Total59100.00%6100.00%


static void arm_smmu_tlb_inv_context_s2(void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_device *smmu = smmu_domain->smmu; void __iomem *base = ARM_SMMU_GR0(smmu); writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID); arm_smmu_tlb_sync_global(smmu); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon3259.26%350.00%
Robin Murphy2240.74%350.00%
Total54100.00%6100.00%


static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, size_t granule, bool leaf, void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx); if (stage1) { reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { iova &= ~12UL; iova |= cfg->asid; do { writel_relaxed(iova, reg); iova += granule; } while (size -= granule); } else { iova >>= 12; iova |= (u64)cfg->asid << 48; do { writeq_relaxed(iova, reg); iova += granule >> 12; } while (size -= granule); } } else { reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2; iova >>= 12; do { smmu_write_atomic_lq(iova, reg); iova += granule >> 12; } while (size -= granule); } }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon11959.50%541.67%
Robin Murphy8140.50%758.33%
Total200100.00%12100.00%

/* * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears * almost negligible, but the benefit of getting the first one in as far ahead * of the sync as possible is significant, hence we don't just make this a * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think. */
static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size, size_t granule, bool leaf, void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu); writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon3665.45%360.00%
Robin Murphy1934.55%240.00%
Total55100.00%5100.00%

static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context_s1, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync_context, }; static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = { .tlb_flush_all = arm_smmu_tlb_inv_context_s2, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync_context, }; static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = { .tlb_flush_all = arm_smmu_tlb_inv_context_s2, .tlb_add_flush = arm_smmu_tlb_inv_vmid_nosync, .tlb_sync = arm_smmu_tlb_sync_vmid, };
static irqreturn_t arm_smmu_context_fault(int irq, void *dev) { u32 fsr, fsynr; unsigned long iova; struct iommu_domain *domain = dev; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_device *smmu = smmu_domain->smmu; void __iomem *cb_base; cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); if (!(fsr & FSR_FAULT)) return IRQ_NONE; fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); dev_err_ratelimited(smmu->dev, "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n", fsr, iova, fsynr, cfg->cbndx); writel(fsr, cb_base + ARM_SMMU_CB_FSR); return IRQ_HANDLED; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon12888.89%555.56%
Robin Murphy139.03%333.33%
Joerg Roedel32.08%111.11%
Total144100.00%9100.00%


static irqreturn_t arm_smmu_global_fault(int irq, void *dev) { u32 gfsr, gfsynr0, gfsynr1, gfsynr2; struct arm_smmu_device *smmu = dev; void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu); gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR); gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0); gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1); gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2); if (!gfsr) return IRQ_NONE; dev_err_ratelimited(smmu->dev, "Unexpected global fault, this could be serious\n"); dev_err_ratelimited(smmu->dev, "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", gfsr, gfsynr0, gfsynr1, gfsynr2); writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR); return IRQ_HANDLED; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon10284.30%583.33%
Robin Murphy1915.70%116.67%
Total121100.00%6100.00%


static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg) { struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; cb->cfg = cfg; /* TTBCR */ if (stage1) { if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr; } else { cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr; cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; cb->tcr[1] |= TTBCR2_SEP_UPSTREAM; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) cb->tcr[1] |= TTBCR2_AS; } } else { cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; } /* TTBRs */ if (stage1) { if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0]; cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1]; } else { cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT; cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT; } } else { cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; } /* MAIRs (stage-1 only) */ if (stage1) { if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr; cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr; } else { cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0]; cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1]; } } }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy20053.91%323.08%
Will Deacon11731.54%753.85%
Sunil Goutham4512.13%17.69%
Tirumalesh Chalamarla92.43%215.38%
Total371100.00%13100.00%


static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) { u32 reg; bool stage1; struct arm_smmu_cb *cb = &smmu->cbs[idx]; struct arm_smmu_cfg *cfg = cb->cfg; void __iomem *cb_base, *gr1_base; cb_base = ARM_SMMU_CB(smmu, idx); /* Unassigned context banks only need disabling */ if (!cfg) { writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); return; } gr1_base = ARM_SMMU_GR1(smmu); stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; /* CBA2R */ if (smmu->version > ARM_SMMU_V1) { if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) reg = CBA2R_RW64_64BIT; else reg = CBA2R_RW64_32BIT; /* 16-bit VMIDs live in CBA2R */ if (smmu->features & ARM_SMMU_FEAT_VMID16) reg |= cfg->vmid << CBA2R_VMID_SHIFT; writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx)); } /* CBAR */ reg = cfg->cbar; if (smmu->version < ARM_SMMU_V2) reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT; /* * Use the weakest shareability/memory types, so they are * overridden by the ttbcr/pte. */ if (stage1) { reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) | (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { /* 8-bit VMIDs live in CBAR */ reg |= cfg->vmid << CBAR_VMID_SHIFT; } writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx)); /* * TTBCR * We must write this before the TTBRs, since it determines the * access behaviour of some fields (in particular, ASID[15:8]). */ if (stage1 && smmu->version > ARM_SMMU_V1) writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TTBCR2); writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TTBCR); /* TTBRs */ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR); writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0); writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1); } else { writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0); if (stage1) writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1); } /* MAIRs (stage-1 only) */ if (stage1) { writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0); writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1); } /* SCTLR */ reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M; if (stage1) reg |= SCTLR_S1_ASIDPNE; if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) reg |= SCTLR_E; writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon22251.87%426.67%
Robin Murphy15335.75%640.00%
Tirumalesh Chalamarla368.41%320.00%
Sunil Goutham143.27%16.67%
Mitchel Humpherys30.70%16.67%
Total428100.00%15100.00%


static int arm_smmu_init_domain_context(struct iommu_domain *domain, struct arm_smmu_device *smmu) { int irq, start, ret = 0; unsigned long ias, oas; struct io_pgtable_ops *pgtbl_ops; struct io_pgtable_cfg pgtbl_cfg; enum io_pgtable_fmt fmt; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; const struct iommu_gather_ops *tlb_ops; mutex_lock(&smmu_domain->init_mutex); if (smmu_domain->smmu) goto out_unlock; if (domain->type == IOMMU_DOMAIN_IDENTITY) { smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; smmu_domain->smmu = smmu; goto out_unlock; } /* * Mapping the requested stage onto what we support is surprisingly * complicated, mainly because the spec allows S1+S2 SMMUs without * support for nested translation. That means we end up with the * following table: * * Requested Supported Actual * S1 N S1 * S1 S1+S2 S1 * S1 S2 S2 * S1 S1 S1 * N N N * N S1+S2 S2 * N S2 S2 * N S1 S1 * * Note that you can't actually request stage-2 mappings. */ if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) smmu_domain->stage = ARM_SMMU_DOMAIN_S2; if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) smmu_domain->stage = ARM_SMMU_DOMAIN_S1; /* * Choosing a suitable context format is even more fiddly. Until we * grow some way for the caller to express a preference, and/or move * the decision into the io-pgtable code where it arguably belongs, * just aim for the closest thing to the rest of the system, and hope * that the hardware isn't esoteric enough that we can't assume AArch64 * support to be a superset of AArch32 support... */ if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | ARM_SMMU_FEAT_FMT_AARCH64_16K | ARM_SMMU_FEAT_FMT_AARCH64_4K))) cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { ret = -EINVAL; goto out_unlock; } switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; start = smmu->num_s2_context_banks; ias = smmu->va_size; oas = smmu->ipa_size; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { fmt = ARM_64_LPAE_S1; } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { fmt = ARM_32_LPAE_S1; ias = min(ias, 32UL); oas = min(oas, 40UL); } else { fmt = ARM_V7S; ias = min(ias, 32UL); oas = min(oas, 32UL); } tlb_ops = &arm_smmu_s1_tlb_ops; break; case ARM_SMMU_DOMAIN_NESTED: /* * We will likely want to change this if/when KVM gets * involved. */ case ARM_SMMU_DOMAIN_S2: cfg->cbar = CBAR_TYPE_S2_TRANS; start = 0; ias = smmu->ipa_size; oas = smmu->pa_size; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { fmt = ARM_64_LPAE_S2; } else { fmt = ARM_32_LPAE_S2; ias = min(ias, 40UL); oas = min(oas, 40UL); } if (smmu->version == ARM_SMMU_V2) tlb_ops = &arm_smmu_s2_tlb_ops_v2; else tlb_ops = &arm_smmu_s2_tlb_ops_v1; break; default: ret = -EINVAL; goto out_unlock; } ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); if (ret < 0) goto out_unlock; cfg->cbndx = ret; if (smmu->version < ARM_SMMU_V2) { cfg->irptndx = atomic_inc_return(&smmu->irptndx); cfg->irptndx %= smmu->num_context_irqs; } else { cfg->irptndx = cfg->cbndx; } if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base; else cfg->asid = cfg->cbndx + smmu->cavium_id_base; pgtbl_cfg = (struct io_pgtable_cfg) { .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, .tlb = tlb_ops, .iommu_dev = smmu->dev, }; if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA; smmu_domain->smmu = smmu; pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); if (!pgtbl_ops) { ret = -ENOMEM; goto out_clear_smmu; } /* Update the domain's page sizes to reflect the page table format */ domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; domain->geometry.aperture_end = (1UL << ias) - 1; domain->geometry.force_aperture = true; /* Initialise the context bank with our page table cfg */ arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); arm_smmu_write_context_bank(smmu, cfg->cbndx); /* * Request context fault interrupt. Do this last to avoid the * handler seeing a half-initialised domain state. */ irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault, IRQF_SHARED, "arm-smmu-context-fault", domain); if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); cfg->irptndx = INVALID_IRPTNDX; } mutex_unlock(&smmu_domain->init_mutex); /* Publish page table ops for map/unmap */ smmu_domain->pgtbl_ops = pgtbl_ops; return 0; out_clear_smmu: smmu_domain->smmu = NULL; out_unlock: mutex_unlock(&smmu_domain->init_mutex); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon47358.54%526.32%
Robin Murphy32239.85%1052.63%
Peng Fan50.62%15.26%
Arnd Bergmann40.50%15.26%
Joerg Roedel30.37%15.26%
Mitchel Humpherys10.12%15.26%
Total808100.00%19100.00%


static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; int irq; if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) return; /* * Disable the context bank and free the page tables before freeing * it. */ smmu->cbs[cfg->cbndx].cfg = NULL; arm_smmu_write_context_bank(smmu, cfg->cbndx); if (cfg->irptndx != INVALID_IRPTNDX) { irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; devm_free_irq(smmu->dev, irq, domain); } free_io_pgtable_ops(smmu_domain->pgtbl_ops); __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon11184.09%350.00%
Robin Murphy139.85%116.67%
Peng Fan53.79%116.67%
Joerg Roedel32.27%116.67%
Total132100.00%6100.00%


static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) { struct arm_smmu_domain *smmu_domain; if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_IDENTITY) return NULL; /* * Allocate the domain and initialise some of its data structures. * We can't really do anything meaningful until we've added a * master. */ smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); if (!smmu_domain) return NULL; if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding || iommu_get_dma_cookie(&smmu_domain->domain))) { kfree(smmu_domain); return NULL; } mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->cb_lock); return &smmu_domain->domain; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon5451.43%342.86%
Robin Murphy3331.43%342.86%
Joerg Roedel1817.14%114.29%
Total105100.00%7100.00%


static void arm_smmu_domain_free(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); /* * Free the domain resources. We assume that all devices have * already been detached. */ iommu_put_dma_cookie(domain); arm_smmu_destroy_domain_context(domain); kfree(smmu_domain); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon2875.68%250.00%
Robin Murphy513.51%125.00%
Joerg Roedel410.81%125.00%
Total37100.00%4100.00%


static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) { struct arm_smmu_smr *smr = smmu->smrs + idx; u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT; if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) reg |= SMR_VALID; writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx)); }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy5168.92%133.33%
Will Deacon1418.92%133.33%
Aleksey Makarov912.16%133.33%
Total74100.00%3100.00%


static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) { struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT | (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT | (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT; if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && smmu->smrs[idx].valid) reg |= S2CR_EXIDVALID; writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx)); }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy7374.49%150.00%
Aleksey Makarov2525.51%150.00%
Total98100.00%2100.00%


static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) { arm_smmu_write_s2cr(smmu, idx); if (smmu->smrs) arm_smmu_write_smr(smmu, idx); }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy34100.00%1100.00%
Total34100.00%1100.00%

/* * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function * should be called after sCR0 is written. */
static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) { void __iomem *gr0_base = ARM_SMMU_GR0(smmu); u32 smr; if (!smmu->smrs) return; /* * SMR.ID bits may not be preserved if the corresponding MASK * bits are set, so check each one separately. We can reject * masters later if they try to claim IDs outside these masks. */ smr = smmu->streamid_mask << SMR_ID_SHIFT; writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); smmu->streamid_mask = smr >> SMR_ID_SHIFT; smr = smmu->streamid_mask << SMR_MASK_SHIFT; writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT; }

Contributors

PersonTokensPropCommitsCommitProp
Aleksey Makarov113100.00%1100.00%
Total113100.00%1100.00%


static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) { struct arm_smmu_smr *smrs = smmu->smrs; int i, free_idx = -ENOSPC; /* Stream indexing is blissfully easy */ if (!smrs) return id; /* Validating SMRs is... less so */ for (i = 0; i < smmu->num_mapping_groups; ++i) { if (!smrs[i].valid) { /* * Note the first free entry we come across, which * we'll claim in the end if nothing else matches. */ if (free_idx < 0) free_idx = i; continue; } /* * If the new entry is _entirely_ matched by an existing entry, * then reuse that, with the guarantee that there also cannot * be any subsequent conflicting entries. In normal use we'd * expect simply identical entries for this case, but there's * no harm in accommodating the generalisation. */ if ((mask & smrs[i].mask) == mask && !((id ^ smrs[i].id) & ~smrs[i].mask)) return i; /* * If the new entry has any other overlap with an existing one, * though, then there always exists at least one stream ID * which would cause a conflict, and we can't allow that risk. */ if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) return -EINVAL; } return free_idx; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy15595.68%266.67%
Will Deacon74.32%133.33%
Total162100.00%3100.00%


static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) { if (--smmu->s2crs[idx].count) return false; smmu->s2crs[idx] = s2cr_init_val; if (smmu->smrs) smmu->smrs[idx].valid = false; return true; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy58100.00%1100.00%
Total58100.00%1100.00%


static int arm_smmu_master_alloc_smes(struct device *dev) { struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; struct arm_smmu_device *smmu = cfg->smmu; struct arm_smmu_smr *smrs = smmu->smrs; struct iommu_group *group; int i, idx, ret; mutex_lock(&smmu->stream_map_mutex); /* Figure out a viable stream map entry allocation */ for_each_cfg_sme(fwspec, i, idx) { u16 sid = fwspec->ids[i]; u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT; if (idx != INVALID_SMENDX) { ret = -EEXIST; goto out_err; } ret = arm_smmu_find_sme(smmu, sid, mask); if (ret < 0) goto out_err; idx = ret; if (smrs && smmu->s2crs[idx].count == 0) { smrs[idx].id = sid; smrs[idx].mask = mask; smrs[idx].valid = true; } smmu->s2crs[idx].count++; cfg->smendx[i] = (s16)idx; } group = iommu_group_get_for_dev(dev); if (!group) group = ERR_PTR(-ENOMEM); if (IS_ERR(group)) { ret = PTR_ERR(group); goto out_err; } iommu_group_put(group); /* It worked! Now, poke the actual hardware */ for_each_cfg_sme(fwspec, i, idx) { arm_smmu_write_sme(smmu, idx); smmu->s2crs[idx].group = group; } mutex_unlock(&smmu->stream_map_mutex); return 0; out_err: while (i--) { arm_smmu_free_sme(smmu, cfg->smendx[i]); cfg->smendx[i] = INVALID_SMENDX; } mutex_unlock(&smmu->stream_map_mutex); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy30792.75%583.33%
Will Deacon247.25%116.67%
Total331100.00%6100.00%


static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec) { struct arm_smmu_device *smmu = fwspec_smmu(fwspec); struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; int i, idx; mutex_lock(&smmu->stream_map_mutex); for_each_cfg_sme(fwspec, i, idx) { if (arm_smmu_free_sme(smmu, idx)) arm_smmu_write_sme(smmu, idx); cfg->smendx[i] = INVALID_SMENDX; } mutex_unlock(&smmu->stream_map_mutex); }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy6575.58%675.00%
Will Deacon2124.42%225.00%
Total86100.00%8100.00%


static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, struct iommu_fwspec *fwspec) { struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_s2cr *s2cr = smmu->s2crs; u8 cbndx = smmu_domain->cfg.cbndx; enum arm_smmu_s2cr_type type; int i, idx; if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) type = S2CR_TYPE_BYPASS; else type = S2CR_TYPE_TRANS; for_each_cfg_sme(fwspec, i, idx) { if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) continue; s2cr[idx].type = type; s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; s2cr[idx].cbndx = cbndx; arm_smmu_write_s2cr(smmu, idx); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy7454.01%545.45%
Will Deacon6245.26%545.45%
R Sricharan10.73%19.09%
Total137100.00%11100.00%


static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); if (!fwspec || fwspec->ops != &arm_smmu_ops) { dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); return -ENXIO; } /* * FIXME: The arch/arm DMA API code tries to attach devices to its own * domains between of_xlate() and add_device() - we have no way to cope * with that, so until ARM gets converted to rely on groups and default * domains, just say no (but more politely than by dereferencing NULL). * This should be at least a WARN_ON once that's sorted. */ if (!fwspec->iommu_priv) return -ENODEV; smmu = fwspec_smmu(fwspec); /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); if (ret < 0) return ret; /* * Sanity check the domain. We don't support domains across * different SMMUs. */ if (smmu_domain->smmu != smmu) { dev_err(dev, "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); return -EINVAL; } /* Looks ok, so add the device to the domain */ return arm_smmu_domain_add_master(smmu_domain, fwspec); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon10568.63%550.00%
Robin Murphy4328.10%330.00%
Joerg Roedel31.96%110.00%
Arnd Bergmann21.31%110.00%
Total153100.00%10100.00%


static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; if (!ops) return -ENODEV; return ops->map(ops, iova, paddr, size, prot); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon5590.16%360.00%
Robin Murphy34.92%120.00%
Joerg Roedel34.92%120.00%
Total61100.00%5100.00%


static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; if (!ops) return 0; return ops->unmap(ops, iova, size); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon4488.00%250.00%
Robin Murphy36.00%125.00%
Joerg Roedel36.00%125.00%
Total50100.00%4100.00%


static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dma_addr_t iova) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; struct device *dev = smmu->dev; void __iomem *cb_base; u32 tmp; u64 phys; unsigned long va, flags; cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); spin_lock_irqsave(&smmu_domain->cb_lock, flags); /* ATS1 registers can only be written atomically */ va = iova & ~0xfffUL; if (smmu->version == ARM_SMMU_V2) smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR); else /* Register is only 32-bit in v1 */ writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) { spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); dev_err(dev, "iova to phys timed out on %pad. Falling back to software table walk.\n", &iova); return ops->iova_to_phys(ops, iova); } phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR); spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); if (phys & CB_PAR_F) { dev_err(dev, "translation fault!\n"); dev_err(dev, "PAR = 0x%llx\n", phys); return 0; } return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); }

Contributors

PersonTokensPropCommitsCommitProp
Mitchel Humpherys19677.47%116.67%
Robin Murphy5320.95%350.00%
Joerg Roedel31.19%116.67%
Fabio Estevam10.40%116.67%
Total253100.00%6100.00%


static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; if (domain->type == IOMMU_DOMAIN_IDENTITY) return iova; if (!ops) return 0; if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && smmu_domain->stage == ARM_SMMU_DOMAIN_S1) return arm_smmu_iova_to_phys_hard(domain, iova); return ops->iova_to_phys(ops, iova); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon4754.65%337.50%
Mitchel Humpherys1618.60%112.50%
Sunil Goutham1112.79%112.50%
Baptiste Reynal66.98%112.50%
Robin Murphy33.49%112.50%
Joerg Roedel33.49%112.50%
Total86100.00%8100.00%


static bool arm_smmu_capable(enum iommu_cap cap) { switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: /* * Return true here as the SMMU can always send out coherent * requests. */ return true; case IOMMU_CAP_NOEXEC: return true; default: return false; } }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon2163.64%250.00%
Joerg Roedel721.21%125.00%
Antonios Motakis515.15%125.00%
Total33100.00%4100.00%


static int arm_smmu_match_node(struct device *dev, void *data) { return dev->fwnode == data; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy2195.45%150.00%
Lorenzo Pieralisi14.55%150.00%
Total22100.00%2100.00%


static struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) { struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL, fwnode, arm_smmu_match_node); put_device(dev); return dev ? dev_get_drvdata(dev) : NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy4391.49%150.00%
Lorenzo Pieralisi48.51%150.00%
Total47100.00%2100.00%


static int arm_smmu_add_device(struct device *dev) { struct arm_smmu_device *smmu; struct arm_smmu_master_cfg *cfg; struct iommu_fwspec *fwspec = dev->iommu_fwspec; int i, ret; if (using_legacy_binding) { ret = arm_smmu_register_legacy_master(dev, &smmu); /* * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master() * will allocate/initialise a new one. Thus we need to update fwspec for * later use. */ fwspec = dev->iommu_fwspec; if (ret) goto out_free; } else if (fwspec && fwspec->ops == &arm_smmu_ops) { smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); } else { return -ENODEV; } ret = -EINVAL; for (i = 0; i < fwspec->num_ids; i++) { u16 sid = fwspec->ids[i]; u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT; if (sid & ~smmu->streamid_mask) { dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", sid, smmu->streamid_mask); goto out_free; } if (mask & ~smmu->smr_mask_mask) { dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", mask, smmu->smr_mask_mask); goto out_free; } } ret = -ENOMEM; cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), GFP_KERNEL); if (!cfg) goto out_free; cfg->smmu = smmu; fwspec->iommu_priv = cfg; while (i--) cfg->smendx[i] = INVALID_SMENDX; ret = arm_smmu_master_alloc_smes(dev); if (ret) goto out_cfg_free; iommu_device_link(&smmu->iommu, dev); return 0; out_cfg_free: kfree(cfg); out_free: iommu_fwspec_free(dev); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy21673.47%635.29%
Will Deacon4314.63%423.53%
Joerg Roedel186.12%211.76%
Artem Savkov72.38%15.88%
Vivek Gautam51.70%15.88%
Antonios Motakis31.02%15.88%
Lorenzo Pieralisi10.34%15.88%
Peng Fan10.34%15.88%
Total294100.00%17100.00%


static void arm_smmu_remove_device(struct device *dev) { struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_master_cfg *cfg; struct arm_smmu_device *smmu; if (!fwspec || fwspec->ops != &arm_smmu_ops) return; cfg = fwspec->iommu_priv; smmu = cfg->smmu; iommu_device_unlink(&smmu->iommu, dev); arm_smmu_master_free_smes(fwspec); iommu_group_remove_device(dev); kfree(fwspec->iommu_priv); iommu_fwspec_free(dev); }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy3944.83%350.00%
Joerg Roedel3236.78%116.67%
Will Deacon1112.64%116.67%
Antonios Motakis55.75%116.67%
Total87100.00%6100.00%


static struct iommu_group *arm_smmu_device_group(struct device *dev) { struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_device *smmu = fwspec_smmu(fwspec); struct iommu_group *group = NULL; int i, idx; for_each_cfg_sme(fwspec, i, idx) { if (group && smmu->s2crs[idx].group && group != smmu->s2crs[idx].group) return ERR_PTR(-EINVAL); group = smmu->s2crs[idx].group; } if (group) return iommu_group_ref_get(group); if (dev_is_pci(dev)) group = pci_device_group(dev); else group = generic_device_group(dev); return group; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy9370.99%480.00%
Joerg Roedel3829.01%120.00%
Total131100.00%5100.00%


static int arm_smmu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); if (domain->type != IOMMU_DOMAIN_UNMANAGED) return -EINVAL; switch (attr) { case DOMAIN_ATTR_NESTING: *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); return 0; default: return -ENODEV; } }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon7095.89%266.67%
Joerg Roedel34.11%133.33%
Total73100.00%3100.00%


static int arm_smmu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { int ret = 0; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); if (domain->type != IOMMU_DOMAIN_UNMANAGED) return -EINVAL; mutex_lock(&smmu_domain->init_mutex); switch (attr) { case DOMAIN_ATTR_NESTING: if (smmu_domain->smmu) { ret = -EPERM; goto out_unlock; } if (*(int *)data) smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; else smmu_domain->stage = ARM_SMMU_DOMAIN_S1; break; default: ret = -ENODEV; } out_unlock: mutex_unlock(&smmu_domain->init_mutex); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon11897.52%375.00%
Joerg Roedel32.48%125.00%
Total121100.00%4100.00%


static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) { u32 mask, fwid = 0; if (args->args_count > 0) fwid |= (u16)args->args[0]; if (args->args_count > 1) fwid |= (u16)args->args[1] << SMR_MASK_SHIFT; else if (!of_property_read_u32(args->np, "stream-match-mask", &mask)) fwid |= (u16)mask << SMR_MASK_SHIFT; return iommu_fwspec_add_ids(dev, &fwid, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy9998.02%266.67%
Will Deacon21.98%133.33%
Total101100.00%3100.00%


static void arm_smmu_get_resv_regions(struct device *dev, struct list_head *head) { struct iommu_resv_region *region; int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, prot, IOMMU_RESV_SW_MSI); if (!region) return; list_add_tail(&region->list, head); iommu_dma_get_resv_regions(dev, head); }

Contributors

PersonTokensPropCommitsCommitProp
Eric Auger5684.85%125.00%
Robin Murphy1015.15%375.00%
Total66100.00%4100.00%


static void arm_smmu_put_resv_regions(struct device *dev, struct list_head *head) { struct iommu_resv_region *entry, *next; list_for_each_entry_safe(entry, next, head, list) kfree(entry); }

Contributors

PersonTokensPropCommitsCommitProp
Eric Auger39100.00%1100.00%
Total39100.00%1100.00%

static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, .domain_free = arm_smmu_domain_free, .attach_dev = arm_smmu_attach_dev, .map = arm_smmu_map, .unmap = arm_smmu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = arm_smmu_iova_to_phys, .add_device = arm_smmu_add_device, .remove_device = arm_smmu_remove_device, .device_group = arm_smmu_device_group, .domain_get_attr = arm_smmu_domain_get_attr, .domain_set_attr = arm_smmu_domain_set_attr, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, .put_resv_regions = arm_smmu_put_resv_regions, .pgsize_bitmap = -1UL, /* Restricted during device attach */ };
static void arm_smmu_device_reset(struct arm_smmu_device *smmu) { void __iomem *gr0_base = ARM_SMMU_GR0(smmu); int i; u32 reg, major; /* clear global FSR */ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); /* * Reset stream mapping groups: Initial values mark all SMRn as * invalid and all S2CRn as bypass unless overridden. */ for (i = 0; i < smmu->num_mapping_groups; ++i) arm_smmu_write_sme(smmu, i); if (smmu->model == ARM_MMU500) { /* * Before clearing ARM_MMU500_ACTLR_CPRE, need to * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK * bit is only present in MMU-500r2 onwards. */ reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7); major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK; reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR); if (major >= 2) reg &= ~ARM_MMU500_ACR_CACHE_LOCK; /* * Allow unmatched Stream IDs to allocate bypass * TLB entries for reduced latency. */ reg |= ARM_MMU500_ACR_SMTNMB_TLBEN; writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR); } /* Make sure all context banks are disabled and clear CB_FSR */ for (i = 0; i < smmu->num_context_banks; ++i) { void __iomem *cb_base = ARM_SMMU_CB(smmu, i); arm_smmu_write_context_bank(smmu, i); writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR); /* * Disable MMU-500's not-particularly-beneficial next-page * prefetcher for the sake of errata #841119 and #826419. */ if (smmu->model == ARM_MMU500) { reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR); reg &= ~ARM_MMU500_ACTLR_CPRE; writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR); } } /* Invalidate the TLB, just in case */ writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH); writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH); reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); /* Enable fault reporting */ reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE); /* Disable TLB broadcasting. */ reg |= (sCR0_VMIDPNE | sCR0_PTM); /* Enable client access, handling unmatched streams as appropriate */ reg &= ~sCR0_CLIENTPD; if (disable_bypass) reg |= sCR0_USFCFG; else reg &= ~sCR0_USFCFG; /* Disable forced broadcasting */ reg &= ~sCR0_FB; /* Don't upgrade barriers */ reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT); if (smmu->features & ARM_SMMU_FEAT_VMID16) reg |= sCR0_VMID16EN; if (smmu->features & ARM_SMMU_FEAT_EXIDS) reg |= sCR0_EXIDENABLE; /* Push the button */ arm_smmu_tlb_sync_global(smmu); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon14340.62%214.29%
Robin Murphy6017.05%642.86%
Andreas Herrmann5916.76%214.29%
Peng Fan4512.78%17.14%
Nipun Gupta215.97%17.14%
Aleksey Makarov123.41%17.14%
Tirumalesh Chalamarla123.41%17.14%
Total352100.00%14100.00%


static int arm_smmu_id_size_to_bits(int size) { switch (size) { case 0: return 32; case 1: return 36; case 2: return 40; case 3: return 42; case 4: return 44; case 5: default: return 48; } }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon52100.00%1100.00%
Total52100.00%1100.00%


static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) { unsigned long size; void __iomem *gr0_base = ARM_SMMU_GR0(smmu); u32 id; bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; int i; dev_notice(smmu->dev, "probing hardware configuration...\n"); dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version == ARM_SMMU_V2 ? 2 : 1); /* ID0 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); /* Restrict available stages based on module parameter */ if (force_stage == 1) id &= ~(ID0_S2TS | ID0_NTS); else if (force_stage == 2) id &= ~(ID0_S1TS | ID0_NTS); if (id & ID0_S1TS) { smmu->features |= ARM_SMMU_FEAT_TRANS_S1; dev_notice(smmu->dev, "\tstage 1 translation\n"); } if (id & ID0_S2TS) { smmu->features |= ARM_SMMU_FEAT_TRANS_S2; dev_notice(smmu->dev, "\tstage 2 translation\n"); } if (id & ID0_NTS) { smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; dev_notice(smmu->dev, "\tnested translation\n"); } if (!(smmu->features & (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { dev_err(smmu->dev, "\tno translation support!\n"); return -ENODEV; } if ((id & ID0_S1TS) && ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) { smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; dev_notice(smmu->dev, "\taddress translation ops\n"); } /* * In order for DMA API calls to work properly, we must defer to what * the FW says about coherency, regardless of what the hardware claims. * Fortunately, this also opens up a workaround for systems where the * ID register value has ended up configured incorrectly. */ cttw_reg = !!(id & ID0_CTTW); if (cttw_fw || cttw_reg) dev_notice(smmu->dev, "\t%scoherent table walk\n", cttw_fw ? "" : "non-"); if (cttw_fw != cttw_reg) dev_notice(smmu->dev, "\t(IDR0.CTTW overridden by FW configuration)\n"); /* Max. number of entries we have for stream matching/indexing */ if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) { smmu->features |= ARM_SMMU_FEAT_EXIDS; size = 1 << 16; } else { size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK); } smmu->streamid_mask = size - 1; if (id & ID0_SMS) { smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK; if (size == 0) { dev_err(smmu->dev, "stream-matching supported, but no SMRs present!\n"); return -ENODEV; } /* Zero-initialised to mark as invalid */ smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), GFP_KERNEL); if (!smmu->smrs) return -ENOMEM; dev_notice(smmu->dev, "\tstream matching with %lu register groups", size); } /* s2cr->type == 0 means translation, so initialise explicitly */ smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), GFP_KERNEL); if (!smmu->s2crs) return -ENOMEM; for (i = 0; i < size; i++) smmu->s2crs[i] = s2cr_init_val; smmu->num_mapping_groups = size; mutex_init(&smmu->stream_map_mutex); spin_lock_init(&smmu->global_sync_lock); if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; if (!(id & ID0_PTFS_NO_AARCH32S)) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; } /* ID1 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1); smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; /* Check for size mismatch of SMMU address space from mapped region */ size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); size <<= smmu->pgshift; if (smmu->cb_base != gr0_base + size) dev_warn(smmu->dev, "SMMU address space size (0x%lx) differs from mapped region size (0x%tx)!\n", size * 2, (smmu->cb_base - gr0_base) * 2); smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK; smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK; if (smmu->num_s2_context_banks > smmu->num_context_banks) { dev_err(smmu->dev, "impossible number of S2 context banks!\n"); return -ENODEV; } dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", smmu->num_context_banks, smmu->num_s2_context_banks); /* * Cavium CN88xx erratum #27704. * Ensure ASID and VMID allocation is unique across all SMMUs in * the system. */ if (smmu->model == CAVIUM_SMMUV2) { smmu->cavium_id_base = atomic_add_return(smmu->num_context_banks, &cavium_smmu_context_count); smmu->cavium_id_base -= smmu->num_context_banks; dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n"); } smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks, sizeof(*smmu->cbs), GFP_KERNEL); if (!smmu->cbs) return -ENOMEM; /* ID2 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK); smmu->ipa_size = size; /* The output mask is also applied for bypass */ size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); smmu->pa_size = size; if (id & ID2_VMID16) smmu->features |= ARM_SMMU_FEAT_VMID16; /* * What the page table walker can address actually depends on which * descriptor format is in use, but since a) we don't know that yet, * and b) it can vary per context bank, this will have to do... */ if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) dev_warn(smmu->dev, "failed to set DMA mask for table walker\n"); if (smmu->version < ARM_SMMU_V2) { smmu->va_size = smmu->ipa_size; if (smmu->version == ARM_SMMU_V1_64K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } else { size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; smmu->va_size = arm_smmu_id_size_to_bits(size); if (id & ID2_PTFS_4K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; if (id & ID2_PTFS_16K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; if (id & ID2_PTFS_64K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } /* Now we've corralled the various formats, what'll it do? */ if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; if (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) smmu->pgsize_bitmap |= SZ_16K | SZ_32M; if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) smmu->pgsize_bitmap |= SZ_64K | SZ_512M; if (arm_smmu_ops.pgsize_bitmap == -1UL) arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; else arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", smmu->pgsize_bitmap); if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", smmu->va_size, smmu->ipa_size); if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", smmu->ipa_size, smmu->pa_size); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon58451.77%828.57%
Robin Murphy43638.65%1242.86%
Aleksey Makarov302.66%13.57%
Mitchel Humpherys282.48%13.57%
Lorenzo Pieralisi131.15%13.57%
Tirumalesh Chalamarla121.06%13.57%
Baptiste Reynal90.80%13.57%
Robert Richter90.80%13.57%
Olav Haugan50.44%13.57%
Andreas Herrmann20.18%13.57%
Total1128100.00%28100.00%

struct arm_smmu_match_data { enum arm_smmu_arch_version version; enum arm_smmu_implementation model; }; #define ARM_SMMU_MATCH_DATA(name, ver, imp) \ static struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, { }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); #ifdef CONFIG_ACPI
static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) { int ret = 0; switch (model) { case ACPI_IORT_SMMU_V1: case ACPI_IORT_SMMU_CORELINK_MMU400: smmu->version = ARM_SMMU_V1; smmu->model = GENERIC_SMMU; break; case ACPI_IORT_SMMU_CORELINK_MMU401: smmu->version = ARM_SMMU_V1_64K; smmu->model = GENERIC_SMMU; break; case ACPI_IORT_SMMU_V2: smmu->version = ARM_SMMU_V2; smmu->model = GENERIC_SMMU; break; case ACPI_IORT_SMMU_CORELINK_MMU500: smmu->version = ARM_SMMU_V2; smmu->model = ARM_MMU500; break; case ACPI_IORT_SMMU_CAVIUM_THUNDERX: smmu->version = ARM_SMMU_V2; smmu->model = CAVIUM_SMMUV2; break; default: ret = -ENODEV; } return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Lorenzo Pieralisi7765.81%125.00%
Robin Murphy3328.21%250.00%
Will Deacon75.98%125.00%
Total117100.00%4100.00%


static int arm_smmu_device_acpi_probe(struct platform_device *pdev, struct arm_smmu_device *smmu) { struct device *dev = smmu->dev; struct acpi_iort_node *node = *(struct acpi_iort_node **)dev_get_platdata(dev); struct acpi_iort_smmu *iort_smmu; int ret; /* Retrieve SMMU1/2 specific data */ iort_smmu = (struct acpi_iort_smmu *)node->node_data; ret = acpi_smmu_get_data(iort_smmu->model, smmu); if (ret < 0) return ret; /* Ignore the configuration access interrupt */ smmu->num_global_irqs = 1; if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Lorenzo Pieralisi7671.70%133.33%
Robin Murphy1514.15%133.33%
Will Deacon1514.15%133.33%
Total106100.00%3100.00%

#else
static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev, struct arm_smmu_device *smmu) { return -ENODEV; }

Contributors

PersonTokensPropCommitsCommitProp
Lorenzo Pieralisi21100.00%1100.00%
Total21100.00%1100.00%

#endif
static int arm_smmu_device_dt_probe(struct platform_device *pdev, struct arm_smmu_device *smmu) { const struct arm_smmu_match_data *data; struct device *dev = &pdev->dev; bool legacy_binding; if (of_property_read_u32(dev->of_node, "#global-interrupts", &smmu->num_global_irqs)) { dev_err(dev, "missing #global-interrupts property\n"); return -ENODEV; } data = of_device_get_match_data(dev); smmu->version = data->version; smmu->model = data->model; parse_driver_options(smmu); legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); if (legacy_binding && !using_generic_binding) { if (!using_legacy_binding) pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n"); using_legacy_binding = true; } else if (!legacy_binding && !using_legacy_binding) { using_generic_binding = true; } else { dev_err(dev, "not probing due to mismatched DT properties\n"); return -ENODEV; } if (of_dma_is_coherent(dev->of_node)) smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Lorenzo Pieralisi8146.29%120.00%
Robin Murphy7442.29%360.00%
Will Deacon2011.43%120.00%
Total175100.00%5100.00%


static void arm_smmu_bus_init(void) { /* Oh, for a proper bus abstraction */ if (!iommu_present(&platform_bus_type)) bus_set_iommu(&platform_bus_type, &arm_smmu_ops); #ifdef CONFIG_ARM_AMBA if (!iommu_present(&amba_bustype)) bus_set_iommu(&amba_bustype, &arm_smmu_ops); #endif #ifdef CONFIG_PCI if (!iommu_present(&pci_bus_type)) { pci_request_acs(); bus_set_iommu(&pci_bus_type, &arm_smmu_ops); } #endif }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy7798.72%150.00%
Lorenzo Pieralisi11.28%150.00%
Total78100.00%2100.00%


static int arm_smmu_device_probe(struct platform_device *pdev) { struct resource *res; resource_size_t ioaddr; struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; int num_irqs, i, err; smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { dev_err(dev, "failed to allocate arm_smmu_device\n"); return -ENOMEM; } smmu->dev = dev; if (dev->of_node) err = arm_smmu_device_dt_probe(pdev, smmu); else err = arm_smmu_device_acpi_probe(pdev, smmu); if (err) return err; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ioaddr = res->start; smmu->base = devm_ioremap_resource(dev, res); if (IS_ERR(smmu->base)) return PTR_ERR(smmu->base); smmu->cb_base = smmu->base + resource_size(res) / 2; num_irqs = 0; while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { num_irqs++; if (num_irqs > smmu->num_global_irqs) smmu->num_context_irqs++; } if (!smmu->num_context_irqs) { dev_err(dev, "found %d interrupts but expected at least %d\n", num_irqs, smmu->num_global_irqs + 1); return -ENODEV; } smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs, GFP_KERNEL); if (!smmu->irqs) { dev_err(dev, "failed to allocate %d irqs\n", num_irqs); return -ENOMEM; } for (i = 0; i < num_irqs; ++i) { int irq = platform_get_irq(pdev, i); if (irq < 0) { dev_err(dev, "failed to get irq index %d\n", i); return -ENODEV; } smmu->irqs[i] = irq; } err = arm_smmu_device_cfg_probe(smmu); if (err) return err; if (smmu->version == ARM_SMMU_V2 && smmu->num_context_banks != smmu->num_context_irqs) { dev_err(dev, "found only %d context interrupt(s) but %d required\n", smmu->num_context_irqs, smmu->num_context_banks); return -ENODEV; } for (i = 0; i < smmu->num_global_irqs; ++i) { err = devm_request_irq(smmu->dev, smmu->irqs[i], arm_smmu_global_fault, IRQF_SHARED, "arm-smmu global fault", smmu); if (err) { dev_err(dev, "failed to request global IRQ %d (%u)\n", i, smmu->irqs[i]); return err; } } err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, "smmu.%pa", &ioaddr); if (err) { dev_err(dev, "Failed to register iommu in sysfs\n"); return err; } iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops); iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); err = iommu_device_register(&smmu->iommu); if (err) { dev_err(dev, "Failed to register iommu\n"); return err; } platform_set_drvdata(pdev, smmu); arm_smmu_device_reset(smmu); arm_smmu_test_smr_masks(smmu); /* * For ACPI and generic DT bindings, an SMMU will be probed before * any device which might need it, so we want the bus ops in place * ready to handle default domain setup as soon as any SMMU exists. */ if (!using_legacy_binding) arm_smmu_bus_init(); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon32056.44%210.00%
Joerg Roedel8514.99%15.00%
Lorenzo Pieralisi6110.76%210.00%
Robin Murphy488.47%945.00%
Julia Lawall183.17%15.00%
Olav Haugan142.47%15.00%
Andreas Herrmann91.59%15.00%
Peng Fan50.88%15.00%
Aleksey Makarov50.88%15.00%
Wei Yongjun20.35%15.00%
Total567100.00%20100.00%

/* * With the legacy DT binding in play, though, we have no guarantees about * probe order, but then we're also not doing default domains, so we can * delay setting bus ops until we're sure every possible SMMU is ready, * and that way ensure that no add_device() calls get missed. */
static int arm_smmu_legacy_bus_init(void) { if (using_legacy_binding) arm_smmu_bus_init(); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy1477.78%266.67%
Will Deacon422.22%133.33%
Total18100.00%3100.00%

device_initcall_sync(arm_smmu_legacy_bus_init);
static int arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); if (!smmu) return -ENODEV; if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) dev_err(&pdev->dev, "removing device with active domains!\n"); /* Turn the thing off */ writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Will Deacon6088.24%250.00%
Robin Murphy68.82%125.00%
Andreas Herrmann22.94%125.00%
Total68100.00%4100.00%


static void arm_smmu_device_shutdown(struct platform_device *pdev) { arm_smmu_device_remove(pdev); }

Contributors

PersonTokensPropCommitsCommitProp
Nate Watterson16100.00%1100.00%
Total16100.00%1100.00%


static int __maybe_unused arm_smmu_pm_resume(struct device *dev) { struct arm_smmu_device *smmu = dev_get_drvdata(dev); arm_smmu_device_reset(smmu); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy30100.00%1100.00%
Total30100.00%1100.00%

static SIMPLE_DEV_PM_OPS(arm_smmu_pm_ops, NULL, arm_smmu_pm_resume); static struct platform_driver arm_smmu_driver = { .driver = { .name = "arm-smmu", .of_match_table = of_match_ptr(arm_smmu_of_match), .pm = &arm_smmu_pm_ops, }, .probe = arm_smmu_device_probe, .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; module_platform_driver(arm_smmu_driver); IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", NULL); IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", NULL); IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", NULL); IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", NULL); IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", NULL); IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", NULL); MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); MODULE_LICENSE("GPL v2");

Overall Contributors

PersonTokensPropCommitsCommitProp
Robin Murphy412141.15%4539.13%
Will Deacon408940.83%2723.48%
Lorenzo Pieralisi3523.51%32.61%
Joerg Roedel2732.73%54.35%
Mitchel Humpherys2512.51%32.61%
Aleksey Makarov2022.02%10.87%
Andreas Herrmann1981.98%43.48%
Eric Auger1091.09%10.87%
Tirumalesh Chalamarla1061.06%32.61%
Sunil Goutham700.70%21.74%
Peng Fan630.63%32.61%
Olav Haugan290.29%21.74%
Rob Clark280.28%10.87%
Nipun Gupta210.21%10.87%
Nate Watterson210.21%10.87%
Julia Lawall180.18%10.87%
Baptiste Reynal150.15%10.87%
Antonios Motakis130.13%21.74%
Robert Richter90.09%10.87%
Artem Savkov70.07%10.87%
Arnd Bergmann60.06%10.87%
Vivek Gautam50.05%10.87%
Dan Carpenter40.04%10.87%
Wei Yongjun20.02%10.87%
R Sricharan10.01%10.87%
Bhumika Goyal10.01%10.87%
Fabio Estevam10.01%10.87%
Total10015100.00%115100.00%
Directory: drivers/iommu
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.