diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2020-08-07 22:41:36 +0300 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2020-08-07 22:41:36 +0300 |
commit | ce615f5c1f73537c8267035d58b3d0c70e19b8da (patch) | |
tree | 54b8e1d34536eed62d898e7d9e81779f6728d286 /drivers/dma/ti/k3-udma.c | |
parent | 81e11336d97e7a4c25a65c302ef2bf9cd9808ed4 (diff) | |
parent | 00043a2689232631f39ebbf6719d545b1d799086 (diff) | |
download | linux-ce615f5c1f73537c8267035d58b3d0c70e19b8da.tar.xz |
Merge tag 'dmaengine-5.9-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine
Pull dmaengine updates from Vinod Koul:
"Core:
- Support out of order dma completion
- Support for repeating transaction
New controllers:
- Support for Actions S700 DMA engine
- Renesas R8A774E1, r8a7742 controller binding
- New driver for Xilinx DPDMA controller
Other:
- Support of out of order dma completion in idxd driver
- W=1 warning cleanup of subsystem
- Updates to ti-k3-dma, dw, idxd drivers"
* tag 'dmaengine-5.9-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (68 commits)
dmaengine: dw: Don't include unneeded header to platform data header
dmaengine: Actions: Add support for S700 DMA engine
dmaengine: Actions: get rid of bit fields from dma descriptor
dt-bindings: dmaengine: convert Actions Semi Owl SoCs bindings to yaml
dmaengine: idxd: add missing invalid flags field to completion
dmaengine: dw: Initialize max_sg_burst capability
dmaengine: dw: Introduce max burst length hw config
dmaengine: dw: Initialize min and max burst DMA device capability
dmaengine: dw: Set DMA device max segment size parameter
dmaengine: dw: Take HC_LLP flag into account for noLLP auto-config
dmaengine: Introduce DMA-device device_caps callback
dmaengine: Introduce max SG burst capability
dmaengine: Introduce min burst length capability
dt-bindings: dma: dw: Add max burst transaction length property
dt-bindings: dma: dw: Convert DW DMAC to DT binding
dmaengine: ti: k3-udma: Query throughput level information from hardware
dmaengine: ti: k3-udma: Use defines for capabilities register parsing
dmaengine: xilinx: dpdma: Fix kerneldoc warning
dmaengine: xilinx: dpdma: add missing kernel doc
dmaengine: xilinx: dpdma: remove comparison of unsigned expression
...
Diffstat (limited to 'drivers/dma/ti/k3-udma.c')
-rw-r--r-- | drivers/dma/ti/k3-udma.c | 309 |
1 files changed, 132 insertions, 177 deletions
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 49d0d3af6311..c14e6cb105cd 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -92,9 +92,6 @@ struct udma_match_data { u32 flags; u32 statictr_z_mask; u32 rchan_oes_offset; - - u8 tpl_levels; - u32 level_start_idx[]; }; struct udma_hwdesc { @@ -121,6 +118,9 @@ struct udma_dev { void __iomem *mmrs[MMR_LAST]; const struct udma_match_data *match_data; + u8 tpl_levels; + u32 tpl_start_idx[3]; + size_t desc_align; /* alignment to use for descriptors */ struct udma_tisci_rm tisci_rm; @@ -282,51 +282,49 @@ static inline void udma_update_bits(void __iomem *base, int reg, } /* TCHANRT */ -static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg) +static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg) { - if (!tchan) + if (!uc->tchan) return 0; - return udma_read(tchan->reg_rt, reg); + return udma_read(uc->tchan->reg_rt, reg); } -static inline void udma_tchanrt_write(struct udma_tchan *tchan, int reg, - u32 val) +static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val) { - if (!tchan) + if (!uc->tchan) return; - udma_write(tchan->reg_rt, reg, val); + udma_write(uc->tchan->reg_rt, reg, val); } -static inline void udma_tchanrt_update_bits(struct udma_tchan *tchan, int reg, +static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg, u32 mask, u32 val) { - if (!tchan) + if (!uc->tchan) return; - udma_update_bits(tchan->reg_rt, reg, mask, val); + udma_update_bits(uc->tchan->reg_rt, reg, mask, val); } /* RCHANRT */ -static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg) +static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg) { - if (!rchan) + if (!uc->rchan) return 0; - return udma_read(rchan->reg_rt, reg); + return udma_read(uc->rchan->reg_rt, reg); } -static inline void udma_rchanrt_write(struct udma_rchan *rchan, int reg, - u32 val) +static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val) { - if (!rchan) + if (!uc->rchan) return; - udma_write(rchan->reg_rt, reg, val); + udma_write(uc->rchan->reg_rt, reg, val); } -static inline void udma_rchanrt_update_bits(struct udma_rchan *rchan, int reg, +static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg, u32 mask, u32 val) { - if (!rchan) + if (!uc->rchan) return; - udma_update_bits(rchan->reg_rt, reg, mask, val); + udma_update_bits(uc->rchan->reg_rt, reg, mask, val); } static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) @@ -366,18 +364,18 @@ static void udma_dump_chan_stdata(struct udma_chan *uc) if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) { dev_dbg(dev, "TCHAN State data:\n"); for (i = 0; i < 32; i++) { - offset = UDMA_TCHAN_RT_STDATA_REG + i * 4; + offset = UDMA_CHAN_RT_STDATA_REG + i * 4; dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i, - udma_tchanrt_read(uc->tchan, offset)); + udma_tchanrt_read(uc, offset)); } } if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) { dev_dbg(dev, "RCHAN State data:\n"); for (i = 0; i < 32; i++) { - offset = UDMA_RCHAN_RT_STDATA_REG + i * 4; + offset = UDMA_CHAN_RT_STDATA_REG + i * 4; dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i, - udma_rchanrt_read(uc->rchan, offset)); + udma_rchanrt_read(uc, offset)); } } } @@ -500,9 +498,9 @@ static bool udma_is_chan_running(struct udma_chan *uc) u32 rrt_ctl = 0; if (uc->tchan) - trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); + trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); if (uc->rchan) - rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG); + rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN) return true; @@ -516,17 +514,15 @@ static bool udma_is_chan_paused(struct udma_chan *uc) switch (uc->config.dir) { case DMA_DEV_TO_MEM: - val = udma_rchanrt_read(uc->rchan, - UDMA_RCHAN_RT_PEER_RT_EN_REG); + val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG); pause_mask = UDMA_PEER_RT_EN_PAUSE; break; case DMA_MEM_TO_DEV: - val = udma_tchanrt_read(uc->tchan, - UDMA_TCHAN_RT_PEER_RT_EN_REG); + val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG); pause_mask = UDMA_PEER_RT_EN_PAUSE; break; case DMA_MEM_TO_MEM: - val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG); + val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG); pause_mask = UDMA_CHAN_RT_CTL_PAUSE; break; default: @@ -539,30 +535,6 @@ static bool udma_is_chan_paused(struct udma_chan *uc) return false; } -static void udma_sync_for_device(struct udma_chan *uc, int idx) -{ - struct udma_desc *d = uc->desc; - - if (uc->cyclic && uc->config.pkt_mode) { - dma_sync_single_for_device(uc->ud->dev, - d->hwdesc[idx].cppi5_desc_paddr, - d->hwdesc[idx].cppi5_desc_size, - DMA_TO_DEVICE); - } else { - int i; - - for (i = 0; i < d->hwdesc_count; i++) { - if (!d->hwdesc[i].cppi5_desc_vaddr) - continue; - - dma_sync_single_for_device(uc->ud->dev, - d->hwdesc[i].cppi5_desc_paddr, - d->hwdesc[i].cppi5_desc_size, - DMA_TO_DEVICE); - } - } -} - static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc) { return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; @@ -593,7 +565,6 @@ static int udma_push_to_ring(struct udma_chan *uc, int idx) paddr = udma_curr_cppi5_desc_paddr(d, idx); wmb(); /* Ensure that writes are not moved over this point */ - udma_sync_for_device(uc, idx); } return k3_ringacc_ring_push(ring, &paddr); @@ -613,7 +584,7 @@ static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr) static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) { struct k3_ring *ring = NULL; - int ret = -ENOENT; + int ret; switch (uc->config.dir) { case DMA_DEV_TO_MEM: @@ -624,34 +595,24 @@ static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) ring = uc->tchan->tc_ring; break; default: - break; + return -ENOENT; } - if (ring && k3_ringacc_ring_get_occ(ring)) { - struct udma_desc *d = NULL; - - ret = k3_ringacc_ring_pop(ring, addr); - if (ret) - return ret; - - /* Teardown completion */ - if (cppi5_desc_is_tdcm(*addr)) - return ret; + ret = k3_ringacc_ring_pop(ring, addr); + if (ret) + return ret; - /* Check for flush descriptor */ - if (udma_desc_is_rx_flush(uc, *addr)) - return -ENOENT; + rmb(); /* Ensure that reads are not moved before this point */ - d = udma_udma_desc_from_paddr(uc, *addr); + /* Teardown completion */ + if (cppi5_desc_is_tdcm(*addr)) + return 0; - if (d) - dma_sync_single_for_cpu(uc->ud->dev, *addr, - d->hwdesc[0].cppi5_desc_size, - DMA_FROM_DEVICE); - rmb(); /* Ensure that reads are not moved before this point */ - } + /* Check for flush descriptor */ + if (udma_desc_is_rx_flush(uc, *addr)) + return -ENOENT; - return ret; + return 0; } static void udma_reset_rings(struct udma_chan *uc) @@ -695,31 +656,31 @@ static void udma_reset_counters(struct udma_chan *uc) u32 val; if (uc->tchan) { - val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val); + val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); + udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); - val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val); + val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); + udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); - val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val); + val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); + udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); - val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val); + val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); + udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); } if (uc->rchan) { - val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG); - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val); + val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); + udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); - val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG); - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val); + val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); + udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); - val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG); - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val); + val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); + udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); - val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG); - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val); + val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); + udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); } uc->bcnt = 0; @@ -729,16 +690,16 @@ static int udma_reset_chan(struct udma_chan *uc, bool hard) { switch (uc->config.dir) { case DMA_DEV_TO_MEM: - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0); - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); + udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); + udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); break; case DMA_MEM_TO_DEV: - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0); + udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); + udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); break; case DMA_MEM_TO_MEM: - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0); + udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); + udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0); break; default: return -EINVAL; @@ -766,7 +727,7 @@ static int udma_reset_chan(struct udma_chan *uc, bool hard) * the rchan. */ if (uc->config.dir == DMA_DEV_TO_MEM) - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, + udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN | UDMA_CHAN_RT_CTL_FTDOWN); @@ -843,11 +804,12 @@ static int udma_start(struct udma_chan *uc) if (uc->config.enable_burst) val |= PDMA_STATIC_TR_XY_BURST; - udma_rchanrt_write(uc->rchan, - UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG, val); + udma_rchanrt_write(uc, + UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG, + val); - udma_rchanrt_write(uc->rchan, - UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG, + udma_rchanrt_write(uc, + UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG, PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt, match_data->statictr_z_mask)); @@ -856,11 +818,11 @@ static int udma_start(struct udma_chan *uc) sizeof(uc->static_tr)); } - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, + udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_EN); /* Enable remote */ - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, + udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_ENABLE); break; @@ -875,8 +837,9 @@ static int udma_start(struct udma_chan *uc) if (uc->config.enable_burst) val |= PDMA_STATIC_TR_XY_BURST; - udma_tchanrt_write(uc->tchan, - UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG, val); + udma_tchanrt_write(uc, + UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG, + val); /* save the current staticTR configuration */ memcpy(&uc->static_tr, &uc->desc->static_tr, @@ -884,17 +847,17 @@ static int udma_start(struct udma_chan *uc) } /* Enable remote */ - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, + udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_ENABLE); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, + udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_EN); break; case DMA_MEM_TO_MEM: - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, + udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_EN); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, + udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_EN); break; @@ -920,20 +883,20 @@ static int udma_stop(struct udma_chan *uc) if (!uc->cyclic && !uc->desc) udma_push_to_ring(uc, -1); - udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, + udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN); break; case DMA_MEM_TO_DEV: - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, + udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_FLUSH); - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, + udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN); break; case DMA_MEM_TO_MEM: - udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, + udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN); break; @@ -973,8 +936,8 @@ static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) uc->config.dir != DMA_MEM_TO_DEV) return true; - peer_bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); - bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); + peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); + bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); /* Transfer is incomplete, store current residue and time stamp */ if (peer_bcnt < bcnt) { @@ -1247,10 +1210,10 @@ static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ } else { \ int start; \ \ - if (tpl >= ud->match_data->tpl_levels) \ - tpl = ud->match_data->tpl_levels - 1; \ + if (tpl >= ud->tpl_levels) \ + tpl = ud->tpl_levels - 1; \ \ - start = ud->match_data->level_start_idx[tpl]; \ + start = ud->tpl_start_idx[tpl]; \ \ id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ start); \ @@ -1299,7 +1262,6 @@ static int udma_get_rchan(struct udma_chan *uc) static int udma_get_chan_pair(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; - const struct udma_match_data *match_data = ud->match_data; int chan_id, end; if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) { @@ -1321,7 +1283,7 @@ static int udma_get_chan_pair(struct udma_chan *uc) /* Can be optimized, but let's have it like this for now */ end = min(ud->tchan_cnt, ud->rchan_cnt); /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */ - chan_id = match_data->level_start_idx[match_data->tpl_levels - 1]; + chan_id = ud->tpl_start_idx[ud->tpl_levels - 1]; for (; chan_id < end; chan_id++) { if (!test_bit(chan_id, ud->tchan_map) && !test_bit(chan_id, ud->rchan_map)) @@ -2195,7 +2157,7 @@ udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, u32 ring_id; unsigned int i; - d = kzalloc(sizeof(*d) + sglen * sizeof(d->hwdesc[0]), GFP_NOWAIT); + d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT); if (!d) return NULL; @@ -2511,7 +2473,7 @@ udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, if (period_len >= SZ_4M) return NULL; - d = kzalloc(sizeof(*d) + periods * sizeof(d->hwdesc[0]), GFP_NOWAIT); + d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT); if (!d) return NULL; @@ -2761,30 +2723,27 @@ static enum dma_status udma_tx_status(struct dma_chan *chan, u32 delay = 0; if (uc->desc->dir == DMA_MEM_TO_DEV) { - bcnt = udma_tchanrt_read(uc->tchan, - UDMA_TCHAN_RT_SBCNT_REG); + bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG); if (uc->config.ep_type != PSIL_EP_NATIVE) { - peer_bcnt = udma_tchanrt_read(uc->tchan, - UDMA_TCHAN_RT_PEER_BCNT_REG); + peer_bcnt = udma_tchanrt_read(uc, + UDMA_CHAN_RT_PEER_BCNT_REG); if (bcnt > peer_bcnt) delay = bcnt - peer_bcnt; } } else if (uc->desc->dir == DMA_DEV_TO_MEM) { - bcnt = udma_rchanrt_read(uc->rchan, - UDMA_RCHAN_RT_BCNT_REG); + bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); if (uc->config.ep_type != PSIL_EP_NATIVE) { - peer_bcnt = udma_rchanrt_read(uc->rchan, - UDMA_RCHAN_RT_PEER_BCNT_REG); + peer_bcnt = udma_rchanrt_read(uc, + UDMA_CHAN_RT_PEER_BCNT_REG); if (peer_bcnt > bcnt) delay = peer_bcnt - bcnt; } } else { - bcnt = udma_tchanrt_read(uc->tchan, - UDMA_TCHAN_RT_BCNT_REG); + bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); } bcnt -= uc->bcnt; @@ -2817,19 +2776,17 @@ static int udma_pause(struct dma_chan *chan) /* pause the channel */ switch (uc->config.dir) { case DMA_DEV_TO_MEM: - udma_rchanrt_update_bits(uc->rchan, - UDMA_RCHAN_RT_PEER_RT_EN_REG, + udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_PAUSE, UDMA_PEER_RT_EN_PAUSE); break; case DMA_MEM_TO_DEV: - udma_tchanrt_update_bits(uc->tchan, - UDMA_TCHAN_RT_PEER_RT_EN_REG, + udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_PAUSE, UDMA_PEER_RT_EN_PAUSE); break; case DMA_MEM_TO_MEM: - udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, + udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_PAUSE, UDMA_CHAN_RT_CTL_PAUSE); break; @@ -2847,18 +2804,16 @@ static int udma_resume(struct dma_chan *chan) /* resume the channel */ switch (uc->config.dir) { case DMA_DEV_TO_MEM: - udma_rchanrt_update_bits(uc->rchan, - UDMA_RCHAN_RT_PEER_RT_EN_REG, + udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_PAUSE, 0); break; case DMA_MEM_TO_DEV: - udma_tchanrt_update_bits(uc->tchan, - UDMA_TCHAN_RT_PEER_RT_EN_REG, + udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_PAUSE, 0); break; case DMA_MEM_TO_MEM: - udma_tchanrt_update_bits(uc->tchan, UDMA_TCHAN_RT_CTL_REG, + udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_PAUSE, 0); break; default: @@ -3147,11 +3102,6 @@ static struct udma_match_data am654_main_data = { .enable_memcpy_support = true, .statictr_z_mask = GENMASK(11, 0), .rchan_oes_offset = 0x2000, - .tpl_levels = 2, - .level_start_idx = { - [0] = 8, /* Normal channels */ - [1] = 0, /* High Throughput channels */ - }, }; static struct udma_match_data am654_mcu_data = { @@ -3159,11 +3109,6 @@ static struct udma_match_data am654_mcu_data = { .enable_memcpy_support = false, .statictr_z_mask = GENMASK(11, 0), .rchan_oes_offset = 0x2000, - .tpl_levels = 2, - .level_start_idx = { - [0] = 2, /* Normal channels */ - [1] = 0, /* High Throughput channels */ - }, }; static struct udma_match_data j721e_main_data = { @@ -3172,12 +3117,6 @@ static struct udma_match_data j721e_main_data = { .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, .statictr_z_mask = GENMASK(23, 0), .rchan_oes_offset = 0x400, - .tpl_levels = 3, - .level_start_idx = { - [0] = 16, /* Normal channels */ - [1] = 4, /* High Throughput channels */ - [2] = 0, /* Ultra High Throughput channels */ - }, }; static struct udma_match_data j721e_mcu_data = { @@ -3186,11 +3125,6 @@ static struct udma_match_data j721e_mcu_data = { .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, .statictr_z_mask = GENMASK(23, 0), .rchan_oes_offset = 0x400, - .tpl_levels = 2, - .level_start_idx = { - [0] = 2, /* Normal channels */ - [1] = 0, /* High Throughput channels */ - }, }; static const struct of_device_id udma_of_match[] = { @@ -3239,15 +3173,36 @@ static int udma_setup_resources(struct udma_dev *ud) "ti,sci-rm-range-rchan", "ti,sci-rm-range-rflow" }; - cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28); - cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); + cap2 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(2)); + cap3 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(3)); - ud->rflow_cnt = cap3 & 0x3fff; - ud->tchan_cnt = cap2 & 0x1ff; - ud->echan_cnt = (cap2 >> 9) & 0x1ff; - ud->rchan_cnt = (cap2 >> 18) & 0x1ff; + ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); + ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); + ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2); + ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); ch_count = ud->tchan_cnt + ud->rchan_cnt; + /* Set up the throughput level start indexes */ + if (of_device_is_compatible(dev->of_node, + "ti,am654-navss-main-udmap")) { + ud->tpl_levels = 2; + ud->tpl_start_idx[0] = 8; + } else if (of_device_is_compatible(dev->of_node, + "ti,am654-navss-mcu-udmap")) { + ud->tpl_levels = 2; + ud->tpl_start_idx[0] = 2; + } else if (UDMA_CAP3_UCHAN_CNT(cap3)) { + ud->tpl_levels = 3; + ud->tpl_start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); + ud->tpl_start_idx[0] = ud->tpl_start_idx[1] + + UDMA_CAP3_HCHAN_CNT(cap3); + } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { + ud->tpl_levels = 2; + ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); + } else { + ud->tpl_levels = 1; + } + ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), sizeof(unsigned long), GFP_KERNEL); ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), |