| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Biju Das | 4489 | 57.70% | 7 | 15.56% |
| Claudiu Beznea | 2482 | 31.90% | 19 | 42.22% |
| John Madieu | 265 | 3.41% | 2 | 4.44% |
| Cosmin Tanislav | 224 | 2.88% | 3 | 6.67% |
| Fabrizio Castro | 206 | 2.65% | 2 | 4.44% |
| Johan Hovold | 47 | 0.60% | 1 | 2.22% |
| Wolfram Sang | 38 | 0.49% | 2 | 4.44% |
| Hien Huynh | 13 | 0.17% | 1 | 2.22% |
| Lad Prabhakar | 5 | 0.06% | 1 | 2.22% |
| Dan Carpenter | 4 | 0.05% | 1 | 2.22% |
| Geert Uytterhoeven | 2 | 0.03% | 1 | 2.22% |
| Uwe Kleine-König | 2 | 0.03% | 2 | 4.44% |
| Colin Ian King | 2 | 0.03% | 2 | 4.44% |
| Kees Cook | 1 | 0.01% | 1 | 2.22% |
| Total | 7780 | 45 |
// SPDX-License-Identifier: GPL-2.0 /* * Renesas RZ/G2L DMA Controller Driver * * Based on imx-dma.c * * Copyright (C) 2021 Renesas Electronics Corp. * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com> */ #include <linux/bitfield.h> #include <linux/cleanup.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/irqchip/irq-renesas-rzv2h.h> #include <linux/irqchip/irq-renesas-rzt2h.h> #include <linux/list.h> #include <linux/lockdep.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_dma.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/slab.h> #include <linux/spinlock.h> #include "../dmaengine.h" #include "../virt-dma.h" enum rz_dmac_prep_type { RZ_DMAC_DESC_MEMCPY, RZ_DMAC_DESC_SLAVE_SG, RZ_DMAC_DESC_CYCLIC, }; struct rz_lmdesc { u32 header; u32 sa; u32 da; u32 tb; u32 chcfg; u32 chitvl; u32 chext; u32 nxla; }; struct rz_dmac_desc { struct virt_dma_desc vd; dma_addr_t src; dma_addr_t dest; size_t len; struct list_head node; enum dma_transfer_direction direction; enum rz_dmac_prep_type type; /* For slave sg */ struct scatterlist *sg; unsigned int sgcount; struct rz_lmdesc *start_lmdesc; }; #define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd) /** * enum rz_dmac_chan_status: RZ DMAC channel status * @RZ_DMAC_CHAN_STATUS_PAUSED: Channel is paused though DMA engine callbacks * @RZ_DMAC_CHAN_STATUS_CYCLIC: Channel is cyclic * @RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL: Channel is paused through driver internal logic */ enum rz_dmac_chan_status { RZ_DMAC_CHAN_STATUS_PAUSED, RZ_DMAC_CHAN_STATUS_CYCLIC, RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL, }; struct rz_dmac_chan { struct virt_dma_chan vc; void __iomem *ch_base; void __iomem *ch_cmn_base; unsigned int index; struct rz_dmac_desc *desc; int descs_allocated; dma_addr_t src_per_address; dma_addr_t dst_per_address; unsigned long status; u32 chcfg; u32 chctrl; int mid_rid; int dmac_ack; struct { u32 nxla; } pm_state; struct list_head ld_free; struct { struct rz_lmdesc *base; struct rz_lmdesc *head; struct rz_lmdesc *tail; dma_addr_t base_dma; } lmdesc; }; #define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan) struct rz_dmac_icu { struct platform_device *pdev; u8 dmac_index; }; struct rz_dmac_info { void (*icu_register_dma_req)(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel, u16 req_no); void (*icu_register_dma_ack)(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel, u16 ack_no); u16 default_dma_ack_no; u16 default_dma_req_no; }; struct rz_dmac { struct dma_device engine; struct rz_dmac_icu icu; const struct rz_dmac_info *info; struct device *dev; struct reset_control *rstc; void __iomem *base; void __iomem *ext_base; unsigned int n_channels; struct rz_dmac_chan *channels; DECLARE_BITMAP(modules, 1024); }; #define to_rz_dmac(d) container_of(d, struct rz_dmac, engine) /* * ----------------------------------------------------------------------------- * Registers */ #define CRTB 0x0020 #define CHSTAT 0x0024 #define CHCTRL 0x0028 #define CHCFG 0x002c #define NXLA 0x0038 #define CRLA 0x003c #define DCTRL 0x0000 #define EACH_CHANNEL_OFFSET 0x0040 #define CHANNEL_0_7_OFFSET 0x0000 #define CHANNEL_0_7_COMMON_BASE 0x0300 #define CHANNEL_8_15_OFFSET 0x0400 #define CHANNEL_8_15_COMMON_BASE 0x0700 #define CHSTAT_ER BIT(4) #define CHSTAT_SUS BIT(3) #define CHSTAT_EN BIT(0) #define CHCTRL_CLRINTMSK BIT(17) #define CHCTRL_CLRSUS BIT(9) #define CHCTRL_SETSUS BIT(8) #define CHCTRL_CLRTC BIT(6) #define CHCTRL_CLREND BIT(5) #define CHCTRL_CLRRQ BIT(4) #define CHCTRL_SWRST BIT(3) #define CHCTRL_STG BIT(2) #define CHCTRL_CLREN BIT(1) #define CHCTRL_SETEN BIT(0) #define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \ CHCTRL_CLRTC | CHCTRL_CLREND | \ CHCTRL_CLRRQ | CHCTRL_SWRST | \ CHCTRL_CLREN) #define CHCFG_DMS BIT(31) #define CHCFG_DEM BIT(24) #define CHCFG_DAD BIT(21) #define CHCFG_SAD BIT(20) #define CHCFG_REQD BIT(3) #define CHCFG_SEL(bits) ((bits) & 0x07) #define CHCFG_MEM_COPY (0x80400008) #define CHCFG_FILL_DDS_MASK GENMASK(19, 16) #define CHCFG_FILL_SDS_MASK GENMASK(15, 12) #define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22) #define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6) #define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5) #define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5) #define MID_RID_MASK GENMASK(9, 0) #define CHCFG_MASK GENMASK(15, 10) #define CHCFG_DS_INVALID 0xFF #define DCTRL_LVINT BIT(1) #define DCTRL_PR BIT(0) #define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR) /* LINK MODE DESCRIPTOR */ #define HEADER_LV BIT(0) #define HEADER_LE BIT(1) #define HEADER_WBD BIT(2) #define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16 #define RZ_DMAC_MAX_CHANNELS 16 #define DMAC_NR_LMDESC 64 /* RZ/V2H ICU related */ #define RZV2H_MAX_DMAC_INDEX 4 /* * ----------------------------------------------------------------------------- * Device access */ static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val, unsigned int offset) { writel(val, dmac->base + offset); } static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val, unsigned int offset) { writel(val, dmac->ext_base + offset); } static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset) { return readl(dmac->ext_base + offset); } static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val, unsigned int offset, int which) { if (which) writel(val, channel->ch_base + offset); else writel(val, channel->ch_cmn_base + offset); } static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel, unsigned int offset, int which) { if (which) return readl(channel->ch_base + offset); else return readl(channel->ch_cmn_base + offset); } /* * ----------------------------------------------------------------------------- * Initialization */ static void rz_lmdesc_setup(struct rz_dmac_chan *channel, struct rz_lmdesc *lmdesc) { u32 nxla; channel->lmdesc.base = lmdesc; channel->lmdesc.head = lmdesc; channel->lmdesc.tail = lmdesc; nxla = channel->lmdesc.base_dma; while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) { lmdesc->header = 0; nxla += sizeof(*lmdesc); lmdesc->nxla = nxla; lmdesc++; } lmdesc->header = 0; lmdesc->nxla = channel->lmdesc.base_dma; } /* * ----------------------------------------------------------------------------- * Descriptors preparation */ static u32 rz_dmac_lmdesc_addr(struct rz_dmac_chan *channel, struct rz_lmdesc *lmdesc) { return channel->lmdesc.base_dma + (sizeof(struct rz_lmdesc) * (lmdesc - channel->lmdesc.base)); } static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) { struct rz_lmdesc *lmdesc = channel->lmdesc.head; while (!(lmdesc->header & HEADER_LV)) { lmdesc->header = 0; lmdesc++; if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) lmdesc = channel->lmdesc.base; } channel->lmdesc.head = lmdesc; } static bool rz_dmac_chan_is_enabled(struct rz_dmac_chan *channel) { u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1); return !!(val & CHSTAT_EN); } static bool rz_dmac_chan_is_paused(struct rz_dmac_chan *channel) { u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1); return !!(val & CHSTAT_SUS); } static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); u32 nxla; u32 chctrl; dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); rz_dmac_lmdesc_recycle(channel); nxla = rz_dmac_lmdesc_addr(channel, channel->lmdesc.head); chctrl = (channel->chctrl | CHCTRL_SETEN); rz_dmac_ch_writel(channel, nxla, NXLA, 1); rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); } static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); } static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) { u32 dmars_offset = (nr / 2) * 4; u32 shift = (nr % 2) * 16; u32 dmars32; dmars32 = rz_dmac_ext_readl(dmac, dmars_offset); dmars32 &= ~(0xffff << shift); dmars32 |= dmars << shift; rz_dmac_ext_writel(dmac, dmars32, dmars_offset); } static void rz_dmac_set_dma_req_no(struct rz_dmac *dmac, unsigned int index, int req_no) { if (dmac->info->icu_register_dma_req) dmac->info->icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, index, req_no); else rz_dmac_set_dmars_register(dmac, index, req_no); } /* * Map MID/RID request number (bits[0:9] of DMA specifier) to the ICU * DMA ACK signal number, per RZ/G3E hardware manual Table 4.6-28. * * Three peripheral groups cover all ACK-capable peripherals: * * PFC external DMA pins (DREQ0..DREQ4): * req_no 0x000-0x004 -> ACK No. 84-88 (ack = req_no + 84) * * SSIU BUSIFs (ssip00..ssip93): * req_no 0x161-0x198 -> ACK No. 28-83 (ack = req_no - 0x145) * * SPDIF (CH0..CH2) + SCU SRC (sr0..sr9) + DVC (cmd0..cmd1): * req_no 0x199-0x1b4 -> ACK No. 0-27 (ack = req_no - 0x199) */ static int rz_dmac_get_ack_no(const struct rz_dmac_info *info, u16 req_no) { if (!info->icu_register_dma_ack) return -EINVAL; switch (req_no) { case 0x000 ... 0x004: /* PFC external DMA pins: ACK No. 84-88 */ return req_no + 84; case 0x161 ... 0x198: /* SSIU BUSIFs: ACK No. 28-83 */ return req_no - 0x145; case 0x199 ... 0x1b4: /* SPDIF + SCU SRC + DVC: ACK No. 0-27 */ return req_no - 0x199; default: return -EINVAL; } } static void rz_dmac_set_dma_ack_no(struct rz_dmac *dmac, unsigned int index, int ack_no) { if (ack_no < 0 || !dmac->info->icu_register_dma_ack) return; dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index, index, ack_no); } static void rz_dmac_reset_dma_ack_no(struct rz_dmac *dmac, int ack_no) { if (ack_no < 0 || !dmac->info->icu_register_dma_ack) return; dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index, dmac->info->default_dma_ack_no, ack_no); } static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); struct rz_lmdesc *lmdesc = channel->lmdesc.tail; struct rz_dmac_desc *d = channel->desc; u32 chcfg = CHCFG_MEM_COPY; d->start_lmdesc = lmdesc; /* prepare descriptor */ lmdesc->sa = d->src; lmdesc->da = d->dest; lmdesc->tb = d->len; lmdesc->chcfg = chcfg; lmdesc->chitvl = 0; lmdesc->chext = 0; lmdesc->header = HEADER_LV | HEADER_LE; rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no); channel->chcfg = chcfg; channel->chctrl = CHCTRL_STG; } static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); struct rz_dmac_desc *d = channel->desc; struct scatterlist *sg, *sgl = d->sg; struct rz_lmdesc *lmdesc; unsigned int i, sg_len = d->sgcount; channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS; if (d->direction == DMA_DEV_TO_MEM) { channel->chcfg |= CHCFG_SAD; channel->chcfg &= ~CHCFG_REQD; } else { channel->chcfg |= CHCFG_DAD | CHCFG_REQD; } lmdesc = channel->lmdesc.tail; d->start_lmdesc = lmdesc; for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) { if (d->direction == DMA_DEV_TO_MEM) { lmdesc->sa = channel->src_per_address; lmdesc->da = sg_dma_address(sg); } else { lmdesc->sa = sg_dma_address(sg); lmdesc->da = channel->dst_per_address; } lmdesc->tb = sg_dma_len(sg); lmdesc->chitvl = 0; lmdesc->chext = 0; if (i == (sg_len - 1)) { lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM); lmdesc->header = HEADER_LV | HEADER_LE; } else { lmdesc->chcfg = channel->chcfg; lmdesc->header = HEADER_LV; } if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) lmdesc = channel->lmdesc.base; } channel->lmdesc.tail = lmdesc; rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); channel->chctrl = 0; } static void rz_dmac_prepare_descs_for_cyclic(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); struct rz_dmac_desc *d = channel->desc; size_t period_len = d->sgcount; struct rz_lmdesc *lmdesc; size_t buf_len = d->len; size_t periods = buf_len / period_len; lockdep_assert_held(&channel->vc.lock); channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DMS; if (d->direction == DMA_DEV_TO_MEM) { channel->chcfg |= CHCFG_SAD; channel->chcfg &= ~CHCFG_REQD; } else { channel->chcfg |= CHCFG_DAD | CHCFG_REQD; } lmdesc = channel->lmdesc.tail; d->start_lmdesc = lmdesc; for (size_t i = 0; i < periods; i++) { if (d->direction == DMA_DEV_TO_MEM) { lmdesc->sa = d->src; lmdesc->da = d->dest + (i * period_len); } else { lmdesc->sa = d->src + (i * period_len); lmdesc->da = d->dest; } lmdesc->tb = period_len; lmdesc->chitvl = 0; lmdesc->chext = 0; lmdesc->chcfg = channel->chcfg; lmdesc->header = HEADER_LV | HEADER_WBD; if (i == periods - 1) lmdesc->nxla = rz_dmac_lmdesc_addr(channel, d->start_lmdesc); if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) lmdesc = channel->lmdesc.base; } channel->lmdesc.tail = lmdesc; rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); } static void rz_dmac_xfer_desc(struct rz_dmac_chan *chan) { struct virt_dma_desc *vd; vd = vchan_next_desc(&chan->vc); if (!vd) { chan->desc = NULL; return; } list_del(&vd->node); chan->desc = to_rz_dmac_desc(vd); switch (chan->desc->type) { case RZ_DMAC_DESC_MEMCPY: rz_dmac_prepare_desc_for_memcpy(chan); break; case RZ_DMAC_DESC_SLAVE_SG: rz_dmac_prepare_descs_for_slave_sg(chan); break; case RZ_DMAC_DESC_CYCLIC: rz_dmac_prepare_descs_for_cyclic(chan); break; } rz_dmac_enable_hw(chan); } /* * ----------------------------------------------------------------------------- * DMA engine operations */ static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) { struct rz_dmac_desc *desc; desc = kzalloc_obj(*desc); if (!desc) break; /* No need to lock. This is called only for the 1st client. */ list_add_tail(&desc->node, &channel->ld_free); channel->descs_allocated++; } if (!channel->descs_allocated) return -ENOMEM; return channel->descs_allocated; } static void rz_dmac_free_chan_resources(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); struct rz_dmac_desc *desc, *_desc; unsigned long flags; int ret; PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) { dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!", dma_chan_name(chan), ret); } spin_lock_irqsave(&channel->vc.lock, flags); rz_lmdesc_setup(channel, channel->lmdesc.base); /* Skip touching HW if RPM resume failed. Let the cleanup do its jobs. */ if (!ret) rz_dmac_disable_hw(channel); if (channel->mid_rid >= 0) { clear_bit(channel->mid_rid, dmac->modules); channel->mid_rid = -EINVAL; } channel->status = 0; rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack); channel->dmac_ack = -EINVAL; spin_unlock_irqrestore(&channel->vc.lock, flags); vchan_free_chan_resources(&channel->vc); spin_lock_irqsave(&channel->vc.lock, flags); list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) { list_del(&desc->node); kfree(desc); channel->descs_allocated--; } INIT_LIST_HEAD(&channel->ld_free); spin_unlock_irqrestore(&channel->vc.lock, flags); } static struct dma_async_tx_descriptor * rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); struct rz_dmac_desc *desc; dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", __func__, channel->index, &src, &dest, len); scoped_guard(spinlock_irqsave, &channel->vc.lock) { desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); if (!desc) return NULL; list_del(&desc->node); } desc->type = RZ_DMAC_DESC_MEMCPY; desc->src = src; desc->dest = dest; desc->len = len; desc->direction = DMA_MEM_TO_MEM; return vchan_tx_prep(&channel->vc, &desc->vd, flags); } static struct dma_async_tx_descriptor * rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac_desc *desc; struct scatterlist *sg; int dma_length = 0; int i = 0; scoped_guard(spinlock_irqsave, &channel->vc.lock) { desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); if (!desc) return NULL; list_del(&desc->node); } for_each_sg(sgl, sg, sg_len, i) dma_length += sg_dma_len(sg); desc->type = RZ_DMAC_DESC_SLAVE_SG; desc->sg = sgl; desc->sgcount = sg_len; desc->len = dma_length; desc->direction = direction; if (direction == DMA_DEV_TO_MEM) desc->src = channel->src_per_address; else desc->dest = channel->dst_per_address; return vchan_tx_prep(&channel->vc, &desc->vd, flags); } static struct dma_async_tx_descriptor * rz_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, unsigned long flags) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac_desc *desc; size_t periods; if (!is_slave_direction(direction)) return NULL; if (!period_len || !buf_len) return NULL; periods = buf_len / period_len; if (!periods || periods > DMAC_NR_LMDESC) return NULL; scoped_guard(spinlock_irqsave, &channel->vc.lock) { if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) return NULL; desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); if (!desc) return NULL; list_del(&desc->node); channel->status |= BIT(RZ_DMAC_CHAN_STATUS_CYCLIC); } desc->type = RZ_DMAC_DESC_CYCLIC; desc->sgcount = period_len; desc->len = buf_len; desc->direction = direction; if (direction == DMA_DEV_TO_MEM) { desc->src = channel->src_per_address; desc->dest = buf_addr; } else { desc->src = buf_addr; desc->dest = channel->dst_per_address; } return vchan_tx_prep(&channel->vc, &desc->vd, flags); } static int rz_dmac_terminate_all(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); unsigned long flags; LIST_HEAD(head); int ret; PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) { dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!", dma_chan_name(chan), ret); } spin_lock_irqsave(&channel->vc.lock, flags); /* Don't return if RPM failed. Let the cleanup do its jobs. */ if (!ret) rz_dmac_disable_hw(channel); rz_lmdesc_setup(channel, channel->lmdesc.base); if (channel->desc) { vchan_terminate_vdesc(&channel->desc->vd); channel->desc = NULL; } vchan_get_all_descriptors(&channel->vc, &head); channel->status = 0; spin_unlock_irqrestore(&channel->vc.lock, flags); vchan_dma_desc_free_list(&channel->vc, &head); return ret; } static void rz_dmac_issue_pending(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); unsigned long flags; int ret; PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) return; spin_lock_irqsave(&channel->vc.lock, flags); /* * Issue the descriptor. If another transfer is already in progress, the * issued descriptor will be handled after the current transfer finishes. */ if (vchan_issue_pending(&channel->vc) && !channel->desc) rz_dmac_xfer_desc(channel); spin_unlock_irqrestore(&channel->vc.lock, flags); } static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds) { u8 i; static const enum dma_slave_buswidth ds_lut[] = { DMA_SLAVE_BUSWIDTH_1_BYTE, DMA_SLAVE_BUSWIDTH_2_BYTES, DMA_SLAVE_BUSWIDTH_4_BYTES, DMA_SLAVE_BUSWIDTH_8_BYTES, DMA_SLAVE_BUSWIDTH_16_BYTES, DMA_SLAVE_BUSWIDTH_32_BYTES, DMA_SLAVE_BUSWIDTH_64_BYTES, DMA_SLAVE_BUSWIDTH_128_BYTES, }; for (i = 0; i < ARRAY_SIZE(ds_lut); i++) { if (ds_lut[i] == ds) return i; } return CHCFG_DS_INVALID; } static int rz_dmac_config(struct dma_chan *chan, struct dma_slave_config *config) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); u32 val; channel->dst_per_address = config->dst_addr; channel->chcfg &= ~CHCFG_FILL_DDS_MASK; if (channel->dst_per_address) { val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); if (val == CHCFG_DS_INVALID) return -EINVAL; channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val); } channel->src_per_address = config->src_addr; channel->chcfg &= ~CHCFG_FILL_SDS_MASK; if (channel->src_per_address) { val = rz_dmac_ds_to_val_mapping(config->src_addr_width); if (val == CHCFG_DS_INVALID) return -EINVAL; channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val); } return 0; } static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd) { struct rz_dmac_chan *channel = to_rz_dmac_chan(vd->tx.chan); struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); struct rz_dmac_desc *desc = to_rz_dmac_desc(vd); guard(spinlock_irqsave)(&vc->lock); list_add_tail(&desc->node, &channel->ld_free); } static void rz_dmac_device_synchronize(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); u32 chstat; int ret; vchan_synchronize(&channel->vc); PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) return; ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN), 100, 100000, false, channel, CHSTAT, 1); if (ret < 0) dev_warn(dmac->dev, "DMA Timeout"); rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no); rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack); } static struct rz_lmdesc * rz_dmac_get_next_lmdesc(struct rz_lmdesc *base, struct rz_lmdesc *lmdesc) { struct rz_lmdesc *next = ++lmdesc; if (next >= base + DMAC_NR_LMDESC) next = base; return next; } static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, struct rz_dmac_desc *desc, u32 crla) { struct rz_lmdesc *lmdesc = desc->start_lmdesc; struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); u32 residue = 0, i = 0; while (lmdesc->nxla != crla) { lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); if (++i >= DMAC_NR_LMDESC) return 0; } /* Calculate residue from next lmdesc to end of virtual desc */ if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) { u32 start_lmdesc_addr = rz_dmac_lmdesc_addr(channel, desc->start_lmdesc); while (lmdesc->nxla != start_lmdesc_addr) { residue += lmdesc->tb; lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); } } else { while (lmdesc->chcfg & CHCFG_DEM) { residue += lmdesc->tb; lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); } } dev_dbg(dmac->dev, "%s: VD residue is %u\n", __func__, residue); return residue; } static int rz_dmac_chan_get_residue(struct device *dev, struct rz_dmac_chan *channel, dma_cookie_t cookie, u32 *residue) { struct rz_dmac_desc *desc = NULL; struct virt_dma_desc *vd; u32 crla, crtb, i; int ret; vd = vchan_find_desc(&channel->vc, cookie); if (vd) { /* Descriptor has been issued but not yet processed. */ desc = to_rz_dmac_desc(vd); *residue = desc->len; return 0; } else if (channel->desc && channel->desc->vd.tx.cookie == cookie) { /* Descriptor is currently processed. */ desc = channel->desc; } if (!desc) { /* Descriptor was not found. May be already completed by now. */ *residue = 0; return 0; } PM_RUNTIME_ACQUIRE_IF_ENABLED(dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) return ret; /* * We need to read two registers. Make sure the hardware does not move * to next lmdesc while reading the current lmdesc. Trying it 3 times * should be enough: initial read, retry, retry for the paranoid. */ for (i = 0; i < 3; i++) { crla = rz_dmac_ch_readl(channel, CRLA, 1); crtb = rz_dmac_ch_readl(channel, CRTB, 1); /* Still the same? */ if (crla == rz_dmac_ch_readl(channel, CRLA, 1)) break; } WARN_ONCE(i >= 3, "residue might not be continuous!"); /* * Calculate number of bytes transferred in processing virtual descriptor. * One virtual descriptor can have many lmdesc. */ *residue = crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, desc, crla); return 0; } static enum dma_status rz_dmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); enum dma_status status; u32 residue; scoped_guard(spinlock_irqsave, &channel->vc.lock) { int ret; status = dma_cookie_status(chan, cookie, txstate); if (status == DMA_COMPLETE || !txstate) return status; ret = rz_dmac_chan_get_residue(dmac->dev, channel, cookie, &residue); if (ret) return DMA_ERROR; if (status == DMA_IN_PROGRESS && rz_dmac_chan_is_paused(channel)) status = DMA_PAUSED; } dma_set_residue(txstate, residue); return status; } static int rz_dmac_device_pause_set(struct rz_dmac_chan *channel, unsigned long set_bitmask) { int ret = 0; u32 val; lockdep_assert_held(&channel->vc.lock); if (!rz_dmac_chan_is_enabled(channel)) return 0; if (rz_dmac_chan_is_paused(channel)) goto set_bit; rz_dmac_ch_writel(channel, CHCTRL_SETSUS, CHCTRL, 1); ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val, (val & CHSTAT_SUS), 1, 1024, false, channel, CHSTAT, 1); set_bit: channel->status |= set_bitmask; return ret; } static int rz_dmac_device_pause(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); int ret; PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) return ret; guard(spinlock_irqsave)(&channel->vc.lock); return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED)); } static int rz_dmac_device_pause_internal(struct rz_dmac_chan *channel) { lockdep_assert_held(&channel->vc.lock); /* Skip channels explicitly paused by consummers or disabled. */ if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED) || !rz_dmac_chan_is_enabled(channel)) return 0; return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL)); } static int rz_dmac_device_resume_set(struct rz_dmac_chan *channel, unsigned long clear_bitmask) { u32 val; int ret; lockdep_assert_held(&channel->vc.lock); /* * We can be: * * 1/ after the channel was paused by a consummer and now it * needs to be resummed * 2/ after the channel was paused internally (as a result of * a system suspend with power loss or not) * 3/ after the channel was paused by a consummer, the system * went through a system suspend (with power loss or not) * and the consummer wants to resume the channel * * To cover all the above cases we set both CLRSUS and SETEN. * * In case 1/ setting SETEN while the channel is still enabled * is harmless for the controller. * * In case 2/ the channel is disabled when calling this function * and setting CLRSUS is harmless for the controller as the * channel is disabled anyway. * * In case 3/ the channel is disabled/enabled if the system * went though a suspend with power loss/or not and setting * CLRSUS/SETEN is harmless for the controller as the channel * is enabled/disabled anyway. */ rz_dmac_ch_writel(channel, CHCTRL_CLRSUS | CHCTRL_SETEN, CHCTRL, 1); ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val, ((val & (CHSTAT_SUS | CHSTAT_EN)) == CHSTAT_EN), 1, 1024, false, channel, CHSTAT, 1); channel->status &= ~clear_bitmask; return ret; } static int rz_dmac_device_resume(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); int ret; PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) return ret; guard(spinlock_irqsave)(&channel->vc.lock); if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED))) return 0; return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED)); } static int rz_dmac_device_resume_internal(struct rz_dmac_chan *channel) { lockdep_assert_held(&channel->vc.lock); if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL))) return 0; return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL)); } /* * ----------------------------------------------------------------------------- * IRQ handling */ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); u32 chstat; chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); if (chstat & CHSTAT_ER) { dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", channel->index, chstat); scoped_guard(spinlock_irqsave, &channel->vc.lock) rz_dmac_disable_hw(channel); return; } /* * No need to lock. This just clears the END interrupt. Writing * zeros to CHCTRL is just ignored by HW. */ rz_dmac_ch_writel(channel, CHCTRL_CLREND, CHCTRL, 1); } static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) { struct rz_dmac_chan *channel = dev_id; if (channel) { rz_dmac_irq_handle_channel(channel); return IRQ_WAKE_THREAD; } /* handle DMAERR irq */ return IRQ_HANDLED; } static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id) { struct rz_dmac_chan *channel = dev_id; struct rz_dmac_desc *desc; guard(spinlock_irqsave)(&channel->vc.lock); desc = channel->desc; if (!desc) return IRQ_HANDLED; if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) { vchan_cyclic_callback(&desc->vd); } else { vchan_cookie_complete(&desc->vd); channel->desc = NULL; rz_dmac_xfer_desc(channel); } return IRQ_HANDLED; } /* * ----------------------------------------------------------------------------- * OF xlate and channel filter */ static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); struct of_phandle_args *dma_spec = arg; u32 ch_cfg; channel->mid_rid = dma_spec->args[0] & MID_RID_MASK; ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10; channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) | CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg); channel->dmac_ack = rz_dmac_get_ack_no(dmac->info, channel->mid_rid); return !test_and_set_bit(channel->mid_rid, dmac->modules); } static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { dma_cap_mask_t mask; if (dma_spec->args_count != 1) return NULL; /* Only slave DMA channels can be allocated via DT */ dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); return __dma_request_channel(&mask, rz_dmac_chan_filter, dma_spec, ofdma->of_node); } /* * ----------------------------------------------------------------------------- * Probe and remove */ static int rz_dmac_chan_probe(struct rz_dmac *dmac, struct rz_dmac_chan *channel, u8 index) { struct platform_device *pdev = to_platform_device(dmac->dev); struct rz_lmdesc *lmdesc; char pdev_irqname[6]; char *irqname; int irq, ret; channel->index = index; channel->mid_rid = -EINVAL; channel->dmac_ack = -EINVAL; /* Set io base address for each channel */ if (index < 8) { channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + EACH_CHANNEL_OFFSET * index; channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE; } else { channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET + EACH_CHANNEL_OFFSET * (index - 8); channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE; } /* Allocate descriptors */ lmdesc = dmam_alloc_coherent(&pdev->dev, sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, &channel->lmdesc.base_dma, GFP_KERNEL); if (!lmdesc) { dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); return -ENOMEM; } rz_lmdesc_setup(channel, lmdesc); channel->vc.desc_free = rz_dmac_virt_desc_free; vchan_init(&channel->vc, &dmac->engine); INIT_LIST_HEAD(&channel->ld_free); /* Initialize register for each channel */ rz_dmac_disable_hw(channel); /* Request the channel interrupt. */ scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); irq = platform_get_irq_byname(pdev, pdev_irqname); if (irq < 0) return irq; irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", dev_name(dmac->dev), index); if (!irqname) return -ENOMEM; ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler, rz_dmac_irq_handler_thread, 0, irqname, channel); if (ret) dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret); return ret; } static void rz_dmac_put_device(void *_dev) { struct device *dev = _dev; put_device(dev); } static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) { struct device_node *np = dev->of_node; struct of_phandle_args args; uint32_t dmac_index; int ret; if (!dmac->info->icu_register_dma_req) return 0; ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); if (ret) return ret; dmac->icu.pdev = of_find_device_by_node(args.np); of_node_put(args.np); if (!dmac->icu.pdev) { dev_err(dev, "ICU device not found.\n"); return -ENODEV; } ret = devm_add_action_or_reset(dev, rz_dmac_put_device, &dmac->icu.pdev->dev); if (ret) return ret; dmac_index = args.args[0]; if (dmac_index > RZV2H_MAX_DMAC_INDEX) { dev_err(dev, "DMAC index %u invalid.\n", dmac_index); return -EINVAL; } dmac->icu.dmac_index = dmac_index; return 0; } static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac) { struct device_node *np = dev->of_node; int ret; ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); if (ret < 0) { dev_err(dev, "unable to read dma-channels property\n"); return ret; } if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) { dev_err(dev, "invalid number of channels %u\n", dmac->n_channels); return -EINVAL; } return rz_dmac_parse_of_icu(dev, dmac); } static int rz_dmac_probe(struct platform_device *pdev) { const char *irqname = "error"; struct dma_device *engine; struct rz_dmac *dmac; int ret; int irq; u8 i; dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); if (!dmac) return -ENOMEM; dmac->info = device_get_match_data(&pdev->dev); dmac->dev = &pdev->dev; platform_set_drvdata(pdev, dmac); ret = rz_dmac_parse_of(&pdev->dev, dmac); if (ret < 0) return ret; dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, sizeof(*dmac->channels), GFP_KERNEL); if (!dmac->channels) return -ENOMEM; /* Request resources */ dmac->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dmac->base)) return PTR_ERR(dmac->base); if (!dmac->info->icu_register_dma_req) { dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(dmac->ext_base)) return PTR_ERR(dmac->ext_base); } /* Initialize the channels. */ INIT_LIST_HEAD(&dmac->engine.channels); dmac->rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev); if (IS_ERR(dmac->rstc)) return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc), "failed to get resets\n"); pm_runtime_irq_safe(&pdev->dev); pm_runtime_enable(&pdev->dev); ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n"); goto err_pm_disable; } ret = reset_control_deassert(dmac->rstc); if (ret) goto err_pm_runtime_put; for (i = 0; i < dmac->n_channels; i++) { ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i); if (ret < 0) goto err; } /* Register interrupt handler for error */ irq = platform_get_irq_byname_optional(pdev, irqname); if (irq > 0) { ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, irqname, NULL); if (ret) { dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", irq, ret); goto err; } } /* Register the DMAC as a DMA provider for DT. */ ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, NULL); if (ret < 0) goto err; /* Register the DMA engine device. */ engine = &dmac->engine; dma_cap_set(DMA_SLAVE, engine->cap_mask); dma_cap_set(DMA_MEMCPY, engine->cap_mask); dma_cap_set(DMA_CYCLIC, engine->cap_mask); engine->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); engine->dev = &pdev->dev; engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; engine->device_free_chan_resources = rz_dmac_free_chan_resources; engine->device_tx_status = rz_dmac_tx_status; engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; engine->device_prep_dma_cyclic = rz_dmac_prep_dma_cyclic; engine->device_config = rz_dmac_config; engine->device_terminate_all = rz_dmac_terminate_all; engine->device_issue_pending = rz_dmac_issue_pending; engine->device_synchronize = rz_dmac_device_synchronize; engine->device_pause = rz_dmac_device_pause; engine->device_resume = rz_dmac_device_resume; engine->copy_align = DMAENGINE_ALIGN_1_BYTE; dma_set_max_seg_size(engine->dev, U32_MAX); ret = dma_async_device_register(engine); if (ret < 0) { dev_err(&pdev->dev, "unable to register\n"); goto dma_register_err; } return 0; dma_register_err: of_dma_controller_free(pdev->dev.of_node); err: reset_control_assert(dmac->rstc); err_pm_runtime_put: pm_runtime_put(&pdev->dev); err_pm_disable: pm_runtime_disable(&pdev->dev); return ret; } static void rz_dmac_remove(struct platform_device *pdev) { struct rz_dmac *dmac = platform_get_drvdata(pdev); dma_async_device_unregister(&dmac->engine); of_dma_controller_free(pdev->dev.of_node); reset_control_assert(dmac->rstc); pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); } static void rz_dmac_suspend_recover(struct rz_dmac *dmac) { int ret; PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) return; for (unsigned int i = 0; i < dmac->n_channels; i++) { struct rz_dmac_chan *channel = &dmac->channels[i]; guard(spinlock_irqsave)(&channel->vc.lock); if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) continue; rz_dmac_device_resume_internal(channel); } } static int rz_dmac_suspend(struct device *dev) { struct rz_dmac *dmac = dev_get_drvdata(dev); int ret = 0; for (unsigned int i = 0; i < dmac->n_channels; i++) { struct rz_dmac_chan *channel = &dmac->channels[i]; guard(spinlock_irqsave)(&channel->vc.lock); if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) continue; ret = rz_dmac_device_pause_internal(channel); if (ret) { dev_err(dev, "Failed to suspend channel %s\n", dma_chan_name(&channel->vc.chan)); break; } channel->pm_state.nxla = rz_dmac_ch_readl(channel, NXLA, 1); } if (ret) goto suspend_recover; ret = reset_control_assert(dmac->rstc); if (ret) goto suspend_recover; ret = pm_runtime_put_sync(dev); if (ret < 0) goto reset_deassert; return 0; reset_deassert: reset_control_deassert(dmac->rstc); suspend_recover: rz_dmac_suspend_recover(dmac); return ret; } static int rz_dmac_resume(struct device *dev) { struct rz_dmac *dmac = dev_get_drvdata(dev); int errors = 0, ret; ret = pm_runtime_resume_and_get(dev); if (ret) return ret; ret = reset_control_deassert(dmac->rstc); if (ret) { /* * Do not put runtime PM here and keep the same state as in * probe. As subsequent suspend/resume cycles may follow, leave * the runtime PM as is, here, to avoid imbalances. */ return ret; } rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); for (unsigned int i = 0; i < dmac->n_channels; i++) { struct rz_dmac_chan *channel = &dmac->channels[i]; guard(spinlock_irqsave)(&channel->vc.lock); rz_dmac_disable_hw(&dmac->channels[i]); if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) continue; rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); rz_dmac_ch_writel(channel, channel->pm_state.nxla, NXLA, 1); rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); rz_dmac_ch_writel(channel, channel->chctrl, CHCTRL, 1); ret = rz_dmac_device_resume_internal(channel); if (ret) { errors = ret; dev_err(dev, "Failed to resume channel %s, ret=%d\n", dma_chan_name(&channel->vc.chan), ret); } } return errors ? : 0; } static DEFINE_SIMPLE_DEV_PM_OPS(rz_dmac_pm_ops, rz_dmac_suspend, rz_dmac_resume); static const struct rz_dmac_info rz_dmac_v2h_info = { .icu_register_dma_req = rzv2h_icu_register_dma_req, .icu_register_dma_ack = rzv2h_icu_register_dma_ack, .default_dma_ack_no = RZV2H_ICU_DMAC_ACK_NO_DEFAULT, .default_dma_req_no = RZV2H_ICU_DMAC_REQ_NO_DEFAULT, }; static const struct rz_dmac_info rz_dmac_t2h_info = { .icu_register_dma_req = rzt2h_icu_register_dma_req, .default_dma_req_no = RZT2H_ICU_DMAC_REQ_NO_DEFAULT, }; static const struct rz_dmac_info rz_dmac_generic_info = { .default_dma_req_no = 0, }; static const struct of_device_id of_rz_dmac_match[] = { { .compatible = "renesas,r9a09g057-dmac", .data = &rz_dmac_v2h_info }, { .compatible = "renesas,r9a09g077-dmac", .data = &rz_dmac_t2h_info }, { .compatible = "renesas,rz-dmac", .data = &rz_dmac_generic_info }, { /* Sentinel */ } }; MODULE_DEVICE_TABLE(of, of_rz_dmac_match); static struct platform_driver rz_dmac_driver = { .driver = { .name = "rz-dmac", .of_match_table = of_rz_dmac_match, .pm = pm_ptr(&rz_dmac_pm_ops), }, .probe = rz_dmac_probe, .remove = rz_dmac_remove, }; module_platform_driver(rz_dmac_driver); MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver"); MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>"); MODULE_LICENSE("GPL v2");
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1