| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Maxime Ripard | 3920 | 98.94% | 2 | 25.00% |
| Lu Baolu | 32 | 0.81% | 1 | 12.50% |
| Robin Murphy | 4 | 0.10% | 2 | 25.00% |
| Wei Yongjun | 3 | 0.08% | 1 | 12.50% |
| Joerg Roedel | 2 | 0.05% | 1 | 12.50% |
| Rikard Falkeborn | 1 | 0.03% | 1 | 12.50% |
| Total | 3962 | 8 |
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) // Copyright (C) 2016-2018, Allwinner Technology CO., LTD. // Copyright (C) 2019-2020, Cerno #include <linux/bitfield.h> #include <linux/bug.h> #include <linux/clk.h> #include <linux/device.h> #include <linux/dma-direction.h> #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/ioport.h> #include <linux/log2.h> #include <linux/module.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm.h> #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/sizes.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/types.h> #define IOMMU_RESET_REG 0x010 #define IOMMU_ENABLE_REG 0x020 #define IOMMU_ENABLE_ENABLE BIT(0) #define IOMMU_BYPASS_REG 0x030 #define IOMMU_AUTO_GATING_REG 0x040 #define IOMMU_AUTO_GATING_ENABLE BIT(0) #define IOMMU_WBUF_CTRL_REG 0x044 #define IOMMU_OOO_CTRL_REG 0x048 #define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c #define IOMMU_TTB_REG 0x050 #define IOMMU_TLB_ENABLE_REG 0x060 #define IOMMU_TLB_PREFETCH_REG 0x070 #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m) #define IOMMU_TLB_FLUSH_REG 0x080 #define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17) #define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16) #define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0)) #define IOMMU_TLB_IVLD_ADDR_REG 0x090 #define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094 #define IOMMU_TLB_IVLD_ENABLE_REG 0x098 #define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0) #define IOMMU_PC_IVLD_ADDR_REG 0x0a0 #define IOMMU_PC_IVLD_ENABLE_REG 0x0a8 #define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0) #define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4) #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2))) #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1)) #define IOMMU_DM_AUT_OVWT_REG 0x0d0 #define IOMMU_INT_ENABLE_REG 0x100 #define IOMMU_INT_CLR_REG 0x104 #define IOMMU_INT_STA_REG 0x108 #define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4) #define IOMMU_INT_ERR_ADDR_L1_REG 0x130 #define IOMMU_INT_ERR_ADDR_L2_REG 0x134 #define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4) #define IOMMU_L1PG_INT_REG 0x0180 #define IOMMU_L2PG_INT_REG 0x0184 #define IOMMU_INT_INVALID_L2PG BIT(17) #define IOMMU_INT_INVALID_L1PG BIT(16) #define IOMMU_INT_MASTER_PERMISSION(m) BIT(m) #define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \ IOMMU_INT_MASTER_PERMISSION(1) | \ IOMMU_INT_MASTER_PERMISSION(2) | \ IOMMU_INT_MASTER_PERMISSION(3) | \ IOMMU_INT_MASTER_PERMISSION(4) | \ IOMMU_INT_MASTER_PERMISSION(5)) #define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \ IOMMU_INT_INVALID_L2PG | \ IOMMU_INT_MASTER_MASK) #define PT_ENTRY_SIZE sizeof(u32) #define NUM_DT_ENTRIES 4096 #define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE) #define NUM_PT_ENTRIES 256 #define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE) struct sun50i_iommu { struct iommu_device iommu; /* Lock to modify the IOMMU registers */ spinlock_t iommu_lock; struct device *dev; void __iomem *base; struct reset_control *reset; struct clk *clk; struct iommu_domain *domain; struct iommu_group *group; struct kmem_cache *pt_pool; }; struct sun50i_iommu_domain { struct iommu_domain domain; /* Number of devices attached to the domain */ refcount_t refcnt; /* L1 Page Table */ u32 *dt; dma_addr_t dt_dma; struct sun50i_iommu *iommu; }; static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain) { return container_of(domain, struct sun50i_iommu_domain, domain); } static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev) { return dev_iommu_priv_get(dev); } static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) { return readl(iommu->base + offset); } static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) { writel(value, iommu->base + offset); } /* * The Allwinner H6 IOMMU uses a 2-level page table. * * The first level is the usual Directory Table (DT), that consists of * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page * Table (PT). * * Each PT consits of 256 4-bytes Page Table Entries (PTE), each * pointing to a 4kB page of physical memory. * * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG * register that contains its physical address. */ #define SUN50I_IOVA_DTE_MASK GENMASK(31, 20) #define SUN50I_IOVA_PTE_MASK GENMASK(19, 12) #define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0) static u32 sun50i_iova_get_dte_index(dma_addr_t iova) { return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova); } static u32 sun50i_iova_get_pte_index(dma_addr_t iova) { return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova); } static u32 sun50i_iova_get_page_offset(dma_addr_t iova) { return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova); } /* * Each Directory Table Entry has a Page Table address and a valid * bit: * +---------------------+-----------+-+ * | PT address | Reserved |V| * +---------------------+-----------+-+ * 31:10 - Page Table address * 9:2 - Reserved * 1:0 - 1 if the entry is valid */ #define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10) #define SUN50I_DTE_PT_ATTRS GENMASK(1, 0) #define SUN50I_DTE_PT_VALID 1 static phys_addr_t sun50i_dte_get_pt_address(u32 dte) { return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK; } static bool sun50i_dte_is_pt_valid(u32 dte) { return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID; } static u32 sun50i_mk_dte(dma_addr_t pt_dma) { return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID; } /* * Each PTE has a Page address, an authority index and a valid bit: * * +----------------+-----+-----+-----+---+-----+ * | Page address | Rsv | ACI | Rsv | V | Rsv | * +----------------+-----+-----+-----+---+-----+ * 31:12 - Page address * 11:8 - Reserved * 7:4 - Authority Control Index * 3:2 - Reserved * 1 - 1 if the entry is valid * 0 - Reserved * * The way permissions work is that the IOMMU has 16 "domains" that * can be configured to give each masters either read or write * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain * 0 seems like the default domain, and its permissions in the * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really * useful to enforce any particular permission. * * Each page entry will then have a reference to the domain they are * affected to, so that we can actually enforce them on a per-page * basis. * * In order to make it work with the IOMMU framework, we will be using * 4 different domains, starting at 1: RD_WR, RD, WR and NONE * depending on the permission we want to enforce. Each domain will * have each master setup in the same way, since the IOMMU framework * doesn't seem to restrict page access on a per-device basis. And * then we will use the relevant domain index when generating the page * table entry depending on the permissions we want to be enforced. */ enum sun50i_iommu_aci { SUN50I_IOMMU_ACI_DO_NOT_USE = 0, SUN50I_IOMMU_ACI_NONE, SUN50I_IOMMU_ACI_RD, SUN50I_IOMMU_ACI_WR, SUN50I_IOMMU_ACI_RD_WR, }; #define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12) #define SUN50I_PTE_ACI_MASK GENMASK(7, 4) #define SUN50I_PTE_PAGE_VALID BIT(1) static phys_addr_t sun50i_pte_get_page_address(u32 pte) { return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK; } static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte) { return FIELD_GET(SUN50I_PTE_ACI_MASK, pte); } static bool sun50i_pte_is_page_valid(u32 pte) { return pte & SUN50I_PTE_PAGE_VALID; } static u32 sun50i_mk_pte(phys_addr_t page, int prot) { enum sun50i_iommu_aci aci; u32 flags = 0; if (prot & (IOMMU_READ | IOMMU_WRITE)) aci = SUN50I_IOMMU_ACI_RD_WR; else if (prot & IOMMU_READ) aci = SUN50I_IOMMU_ACI_RD; else if (prot & IOMMU_WRITE) aci = SUN50I_IOMMU_ACI_WR; else aci = SUN50I_IOMMU_ACI_NONE; flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci); page &= SUN50I_PTE_PAGE_ADDRESS_MASK; return page | flags | SUN50I_PTE_PAGE_VALID; } static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain, void *vaddr, unsigned int count) { struct sun50i_iommu *iommu = sun50i_domain->iommu; dma_addr_t dma = virt_to_phys(vaddr); size_t size = count * PT_ENTRY_SIZE; dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE); } static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) { u32 reg; int ret; assert_spin_locked(&iommu->iommu_lock); iommu_write(iommu, IOMMU_TLB_FLUSH_REG, IOMMU_TLB_FLUSH_PTW_CACHE | IOMMU_TLB_FLUSH_MACRO_TLB | IOMMU_TLB_FLUSH_MICRO_TLB(5) | IOMMU_TLB_FLUSH_MICRO_TLB(4) | IOMMU_TLB_FLUSH_MICRO_TLB(3) | IOMMU_TLB_FLUSH_MICRO_TLB(2) | IOMMU_TLB_FLUSH_MICRO_TLB(1) | IOMMU_TLB_FLUSH_MICRO_TLB(0)); ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG, reg, !reg, 1, 2000); if (ret) dev_warn(iommu->dev, "TLB Flush timed out!\n"); return ret; } static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); struct sun50i_iommu *iommu = sun50i_domain->iommu; unsigned long flags; /* * At boot, we'll have a first call into .flush_iotlb_all right after * .probe_device, and since we link our (single) domain to our iommu in * the .attach_device callback, we don't have that pointer set. * * It shouldn't really be any trouble to ignore it though since we flush * all caches as part of the device powerup. */ if (!iommu) return; spin_lock_irqsave(&iommu->iommu_lock, flags); sun50i_iommu_flush_all_tlb(iommu); spin_unlock_irqrestore(&iommu->iommu_lock, flags); } static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { sun50i_iommu_flush_iotlb_all(domain); } static int sun50i_iommu_enable(struct sun50i_iommu *iommu) { struct sun50i_iommu_domain *sun50i_domain; unsigned long flags; int ret; if (!iommu->domain) return 0; sun50i_domain = to_sun50i_domain(iommu->domain); ret = reset_control_deassert(iommu->reset); if (ret) return ret; ret = clk_prepare_enable(iommu->clk); if (ret) goto err_reset_assert; spin_lock_irqsave(&iommu->iommu_lock, flags); iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma); iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) | IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) | IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) | IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) | IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) | IOMMU_TLB_PREFETCH_MASTER_ENABLE(5)); iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5)); iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD), IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) | IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5)); iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR), IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) | IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5)); ret = sun50i_iommu_flush_all_tlb(iommu); if (ret) { spin_unlock_irqrestore(&iommu->iommu_lock, flags); goto err_clk_disable; } iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE); spin_unlock_irqrestore(&iommu->iommu_lock, flags); return 0; err_clk_disable: clk_disable_unprepare(iommu->clk); err_reset_assert: reset_control_assert(iommu->reset); return ret; } static void sun50i_iommu_disable(struct sun50i_iommu *iommu) { unsigned long flags; spin_lock_irqsave(&iommu->iommu_lock, flags); iommu_write(iommu, IOMMU_ENABLE_REG, 0); iommu_write(iommu, IOMMU_TTB_REG, 0); spin_unlock_irqrestore(&iommu->iommu_lock, flags); clk_disable_unprepare(iommu->clk); reset_control_assert(iommu->reset); } static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, gfp_t gfp) { dma_addr_t pt_dma; u32 *page_table; page_table = kmem_cache_zalloc(iommu->pt_pool, gfp); if (!page_table) return ERR_PTR(-ENOMEM); pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(iommu->dev, pt_dma)) { dev_err(iommu->dev, "Couldn't map L2 Page Table\n"); kmem_cache_free(iommu->pt_pool, page_table); return ERR_PTR(-ENOMEM); } /* We rely on the physical address and DMA address being the same */ WARN_ON(pt_dma != virt_to_phys(page_table)); return page_table; } static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, u32 *page_table) { phys_addr_t pt_phys = virt_to_phys(page_table); dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE); kmem_cache_free(iommu->pt_pool, page_table); } static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain, dma_addr_t iova, gfp_t gfp) { struct sun50i_iommu *iommu = sun50i_domain->iommu; u32 *page_table; u32 *dte_addr; u32 old_dte; u32 dte; dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; dte = *dte_addr; if (sun50i_dte_is_pt_valid(dte)) { phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte); return (u32 *)phys_to_virt(pt_phys); } page_table = sun50i_iommu_alloc_page_table(iommu, gfp); if (IS_ERR(page_table)) return page_table; dte = sun50i_mk_dte(virt_to_phys(page_table)); old_dte = cmpxchg(dte_addr, 0, dte); if (old_dte) { phys_addr_t installed_pt_phys = sun50i_dte_get_pt_address(old_dte); u32 *installed_pt = phys_to_virt(installed_pt_phys); u32 *drop_pt = page_table; page_table = installed_pt; dte = old_dte; sun50i_iommu_free_page_table(iommu, drop_pt); } sun50i_table_flush(sun50i_domain, page_table, PT_SIZE); sun50i_table_flush(sun50i_domain, dte_addr, 1); return page_table; } static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); struct sun50i_iommu *iommu = sun50i_domain->iommu; u32 pte_index; u32 *page_table, *pte_addr; int ret = 0; page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp); if (IS_ERR(page_table)) { ret = PTR_ERR(page_table); goto out; } pte_index = sun50i_iova_get_pte_index(iova); pte_addr = &page_table[pte_index]; if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) { phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr); dev_err(iommu->dev, "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n", &iova, &page_phys, &paddr, prot); ret = -EBUSY; goto out; } *pte_addr = sun50i_mk_pte(paddr, prot); sun50i_table_flush(sun50i_domain, pte_addr, 1); out: return ret; } static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size, struct iommu_iotlb_gather *gather) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); phys_addr_t pt_phys; u32 *pte_addr; u32 dte; dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; if (!sun50i_dte_is_pt_valid(dte)) return 0; pt_phys = sun50i_dte_get_pt_address(dte); pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova); if (!sun50i_pte_is_page_valid(*pte_addr)) return 0; memset(pte_addr, 0, sizeof(*pte_addr)); sun50i_table_flush(sun50i_domain, pte_addr, 1); return SZ_4K; } static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); phys_addr_t pt_phys; u32 *page_table; u32 dte, pte; dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; if (!sun50i_dte_is_pt_valid(dte)) return 0; pt_phys = sun50i_dte_get_pt_address(dte); page_table = (u32 *)phys_to_virt(pt_phys); pte = page_table[sun50i_iova_get_pte_index(iova)]; if (!sun50i_pte_is_page_valid(pte)) return 0; return sun50i_pte_get_page_address(pte) + sun50i_iova_get_page_offset(iova); } static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type) { struct sun50i_iommu_domain *sun50i_domain; if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_IDENTITY && type != IOMMU_DOMAIN_UNMANAGED) return NULL; sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL); if (!sun50i_domain) return NULL; sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, get_order(DT_SIZE)); if (!sun50i_domain->dt) goto err_free_domain; refcount_set(&sun50i_domain->refcnt, 1); sun50i_domain->domain.geometry.aperture_start = 0; sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); sun50i_domain->domain.geometry.force_aperture = true; return &sun50i_domain->domain; err_free_domain: kfree(sun50i_domain); return NULL; } static void sun50i_iommu_domain_free(struct iommu_domain *domain) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE)); sun50i_domain->dt = NULL; kfree(sun50i_domain); } static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, struct sun50i_iommu_domain *sun50i_domain) { iommu->domain = &sun50i_domain->domain; sun50i_domain->iommu = iommu; sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt, DT_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) { dev_err(iommu->dev, "Couldn't map L1 Page Table\n"); return -ENOMEM; } return sun50i_iommu_enable(iommu); } static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, struct sun50i_iommu_domain *sun50i_domain) { unsigned int i; for (i = 0; i < NUM_DT_ENTRIES; i++) { phys_addr_t pt_phys; u32 *page_table; u32 *dte_addr; u32 dte; dte_addr = &sun50i_domain->dt[i]; dte = *dte_addr; if (!sun50i_dte_is_pt_valid(dte)) continue; memset(dte_addr, 0, sizeof(*dte_addr)); sun50i_table_flush(sun50i_domain, dte_addr, 1); pt_phys = sun50i_dte_get_pt_address(dte); page_table = phys_to_virt(pt_phys); sun50i_iommu_free_page_table(iommu, page_table); } sun50i_iommu_disable(iommu); dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt), DT_SIZE, DMA_TO_DEVICE); iommu->domain = NULL; } static void sun50i_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); struct sun50i_iommu *iommu = dev_iommu_priv_get(dev); dev_dbg(dev, "Detaching from IOMMU domain\n"); if (iommu->domain != domain) return; if (refcount_dec_and_test(&sun50i_domain->refcnt)) sun50i_iommu_detach_domain(iommu, sun50i_domain); } static int sun50i_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); struct sun50i_iommu *iommu; iommu = sun50i_iommu_from_dev(dev); if (!iommu) return -ENODEV; dev_dbg(dev, "Attaching to IOMMU domain\n"); refcount_inc(&sun50i_domain->refcnt); if (iommu->domain == domain) return 0; if (iommu->domain) sun50i_iommu_detach_device(iommu->domain, dev); sun50i_iommu_attach_domain(iommu, sun50i_domain); return 0; } static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) { struct sun50i_iommu *iommu; iommu = sun50i_iommu_from_dev(dev); if (!iommu) return ERR_PTR(-ENODEV); return &iommu->iommu; } static struct iommu_group *sun50i_iommu_device_group(struct device *dev) { struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev); return iommu_group_ref_get(iommu->group); } static int sun50i_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) { struct platform_device *iommu_pdev = of_find_device_by_node(args->np); unsigned id = args->args[0]; dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev)); return iommu_fwspec_add_ids(dev, &id, 1); } static const struct iommu_ops sun50i_iommu_ops = { .pgsize_bitmap = SZ_4K, .device_group = sun50i_iommu_device_group, .domain_alloc = sun50i_iommu_domain_alloc, .of_xlate = sun50i_iommu_of_xlate, .probe_device = sun50i_iommu_probe_device, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = sun50i_iommu_attach_device, .detach_dev = sun50i_iommu_detach_device, .flush_iotlb_all = sun50i_iommu_flush_iotlb_all, .iotlb_sync = sun50i_iommu_iotlb_sync, .iova_to_phys = sun50i_iommu_iova_to_phys, .map = sun50i_iommu_map, .unmap = sun50i_iommu_unmap, .free = sun50i_iommu_domain_free, } }; static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, unsigned master, phys_addr_t iova, unsigned prot) { dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n", &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd"); if (iommu->domain) report_iommu_fault(iommu->domain, iommu->dev, iova, prot); else dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n"); } static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, unsigned addr_reg, unsigned blame_reg) { phys_addr_t iova; unsigned master; u32 blame; assert_spin_locked(&iommu->iommu_lock); iova = iommu_read(iommu, addr_reg); blame = iommu_read(iommu, blame_reg); master = ilog2(blame & IOMMU_INT_MASTER_MASK); /* * If the address is not in the page table, we can't get what * operation triggered the fault. Assume it's a read * operation. */ sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ); return iova; } static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) { enum sun50i_iommu_aci aci; phys_addr_t iova; unsigned master; unsigned dir; u32 blame; assert_spin_locked(&iommu->iommu_lock); blame = iommu_read(iommu, IOMMU_INT_STA_REG); master = ilog2(blame & IOMMU_INT_MASTER_MASK); iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master)); aci = sun50i_get_pte_aci(iommu_read(iommu, IOMMU_INT_ERR_DATA_REG(master))); switch (aci) { /* * If we are in the read-only domain, then it means we * tried to write. */ case SUN50I_IOMMU_ACI_RD: dir = IOMMU_FAULT_WRITE; break; /* * If we are in the write-only domain, then it means * we tried to read. */ case SUN50I_IOMMU_ACI_WR: /* * If we are in the domain without any permission, we * can't really tell. Let's default to a read * operation. */ case SUN50I_IOMMU_ACI_NONE: /* WTF? */ case SUN50I_IOMMU_ACI_RD_WR: default: dir = IOMMU_FAULT_READ; break; } /* * If the address is not in the page table, we can't get what * operation triggered the fault. Assume it's a read * operation. */ sun50i_iommu_report_fault(iommu, master, iova, dir); return iova; } static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id) { struct sun50i_iommu *iommu = dev_id; u32 status; spin_lock(&iommu->iommu_lock); status = iommu_read(iommu, IOMMU_INT_STA_REG); if (!(status & IOMMU_INT_MASK)) { spin_unlock(&iommu->iommu_lock); return IRQ_NONE; } if (status & IOMMU_INT_INVALID_L2PG) sun50i_iommu_handle_pt_irq(iommu, IOMMU_INT_ERR_ADDR_L2_REG, IOMMU_L2PG_INT_REG); else if (status & IOMMU_INT_INVALID_L1PG) sun50i_iommu_handle_pt_irq(iommu, IOMMU_INT_ERR_ADDR_L1_REG, IOMMU_L1PG_INT_REG); else sun50i_iommu_handle_perm_irq(iommu); iommu_write(iommu, IOMMU_INT_CLR_REG, status); iommu_write(iommu, IOMMU_RESET_REG, ~status); iommu_write(iommu, IOMMU_RESET_REG, status); spin_unlock(&iommu->iommu_lock); return IRQ_HANDLED; } static int sun50i_iommu_probe(struct platform_device *pdev) { struct sun50i_iommu *iommu; int ret, irq; iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); if (!iommu) return -ENOMEM; spin_lock_init(&iommu->iommu_lock); platform_set_drvdata(pdev, iommu); iommu->dev = &pdev->dev; iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), PT_SIZE, PT_SIZE, SLAB_HWCACHE_ALIGN, NULL); if (!iommu->pt_pool) return -ENOMEM; iommu->group = iommu_group_alloc(); if (IS_ERR(iommu->group)) { ret = PTR_ERR(iommu->group); goto err_free_cache; } iommu->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(iommu->base)) { ret = PTR_ERR(iommu->base); goto err_free_group; } irq = platform_get_irq(pdev, 0); if (irq < 0) { ret = irq; goto err_free_group; } iommu->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(iommu->clk)) { dev_err(&pdev->dev, "Couldn't get our clock.\n"); ret = PTR_ERR(iommu->clk); goto err_free_group; } iommu->reset = devm_reset_control_get(&pdev->dev, NULL); if (IS_ERR(iommu->reset)) { dev_err(&pdev->dev, "Couldn't get our reset line.\n"); ret = PTR_ERR(iommu->reset); goto err_free_group; } ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev, NULL, dev_name(&pdev->dev)); if (ret) goto err_free_group; ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev); if (ret) goto err_remove_sysfs; ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0, dev_name(&pdev->dev), iommu); if (ret < 0) goto err_unregister; return 0; err_unregister: iommu_device_unregister(&iommu->iommu); err_remove_sysfs: iommu_device_sysfs_remove(&iommu->iommu); err_free_group: iommu_group_put(iommu->group); err_free_cache: kmem_cache_destroy(iommu->pt_pool); return ret; } static const struct of_device_id sun50i_iommu_dt[] = { { .compatible = "allwinner,sun50i-h6-iommu", }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, sun50i_iommu_dt); static struct platform_driver sun50i_iommu_driver = { .driver = { .name = "sun50i-iommu", .of_match_table = sun50i_iommu_dt, .suppress_bind_attrs = true, } }; builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe); MODULE_DESCRIPTION("Allwinner H6 IOMMU driver"); MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>"); MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>"); MODULE_LICENSE("Dual BSD/GPL");
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1