Gitiles
Code Review Sign In
git01.mediatek.com / filogic / uboot / 329112330d0ff051ef398314b2ba35f8dd6b3b54 / . / drivers / dma / ti / k3-udma.c
blob: 3013c4741d0dd594578668ce5140d3a6224c938a [file] [log] [blame]
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1// SPDX-License-Identifier: GPL-2.0+
2/*
Nishanth Menoneaa39c62023-11-01 15:56:03 -0500[diff] [blame]3 * Copyright (C) 2018 Texas Instruments Incorporated - https://www.ti.com
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
5 */
6#define pr_fmt(fmt) "udma: " fmt
7
Simon Glass63334482019-11-14 12:57:39 -0700[diff] [blame]8#include <cpu_func.h>
Simon Glass0f2af882020-05-10 11:40:05 -0600[diff] [blame]9#include <log.h>
Simon Glass274e0b02020-05-10 11:39:56 -0600[diff] [blame]10#include <asm/cache.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]11#include <asm/io.h>
12#include <asm/bitops.h>
13#include <malloc.h>
Matthias Schifferfe2ee882024-04-26 10:02:27 +0200[diff] [blame]14#include <net.h>
Simon Glass4dcacfc2020-05-10 11:40:13 -0600[diff] [blame]15#include <linux/bitops.h>
Masahiro Yamada6373a172020-02-14 16:40:19 +0900[diff] [blame]16#include <linux/dma-mapping.h>
Dhruva Golee6b42392022-09-20 10:56:02 +0530[diff] [blame]17#include <linux/sizes.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]18#include <dm.h>
Simon Glass9bc15642020-02-03 07:36:16 -0700[diff] [blame]19#include <dm/device_compat.h>
Simon Glassd66c5f72020-02-03 07:36:15 -0700[diff] [blame]20#include <dm/devres.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]21#include <dm/read.h>
22#include <dm/of_access.h>
23#include <dma.h>
24#include <dma-uclass.h>
25#include <linux/delay.h>
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]26#include <linux/bitmap.h>
Simon Glassd66c5f72020-02-03 07:36:15 -0700[diff] [blame]27#include <linux/err.h>
Simon Glassbdd5f812023-09-14 18:21:46 -0600[diff] [blame]28#include <linux/printk.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]29#include <linux/soc/ti/k3-navss-ringacc.h>
30#include <linux/soc/ti/cppi5.h>
31#include <linux/soc/ti/ti-udma.h>
32#include <linux/soc/ti/ti_sci_protocol.h>
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]33#include <linux/soc/ti/cppi5.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]34
35#include "k3-udma-hwdef.h"
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]36#include "k3-psil-priv.h"
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]37
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]38#define K3_UDMA_MAX_RFLOWS 1024
39
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]40struct udma_chan;
41
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]42enum k3_dma_type {
43 DMA_TYPE_UDMA = 0,
44 DMA_TYPE_BCDMA,
45 DMA_TYPE_PKTDMA,
46};
47
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]48enum udma_mmr {
49 MMR_GCFG = 0,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]50 MMR_BCHANRT,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]51 MMR_RCHANRT,
52 MMR_TCHANRT,
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]53 MMR_RCHAN,
54 MMR_TCHAN,
55 MMR_RFLOW,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]56 MMR_LAST,
57};
58
59static const char * const mmr_names[] = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]60 [MMR_GCFG] = "gcfg",
61 [MMR_BCHANRT] = "bchanrt",
62 [MMR_RCHANRT] = "rchanrt",
63 [MMR_TCHANRT] = "tchanrt",
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]64 [MMR_RCHAN] = "rchan",
65 [MMR_TCHAN] = "tchan",
66 [MMR_RFLOW] = "rflow",
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]67};
68
69struct udma_tchan {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]70 void __iomem *reg_chan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]71 void __iomem *reg_rt;
72
73 int id;
74 struct k3_nav_ring *t_ring; /* Transmit ring */
75 struct k3_nav_ring *tc_ring; /* Transmit Completion ring */
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]76 int tflow_id; /* applicable only for PKTDMA */
77
78};
79
80#define udma_bchan udma_tchan
81
82struct udma_rflow {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]83 void __iomem *reg_rflow;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]84 int id;
85 struct k3_nav_ring *fd_ring; /* Free Descriptor ring */
86 struct k3_nav_ring *r_ring; /* Receive ring */
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]87};
88
89struct udma_rchan {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]90 void __iomem *reg_chan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]91 void __iomem *reg_rt;
92
93 int id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]94};
95
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]96struct udma_oes_offsets {
97 /* K3 UDMA Output Event Offset */
98 u32 udma_rchan;
99
100 /* BCDMA Output Event Offsets */
101 u32 bcdma_bchan_data;
102 u32 bcdma_bchan_ring;
103 u32 bcdma_tchan_data;
104 u32 bcdma_tchan_ring;
105 u32 bcdma_rchan_data;
106 u32 bcdma_rchan_ring;
107
108 /* PKTDMA Output Event Offsets */
109 u32 pktdma_tchan_flow;
110 u32 pktdma_rchan_flow;
111};
112
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]113#define UDMA_FLAG_PDMA_ACC32 BIT(0)
114#define UDMA_FLAG_PDMA_BURST BIT(1)
115#define UDMA_FLAG_TDTYPE BIT(2)
116
117struct udma_match_data {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]118 enum k3_dma_type type;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]119 u32 psil_base;
120 bool enable_memcpy_support;
121 u32 flags;
122 u32 statictr_z_mask;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]123 struct udma_oes_offsets oes;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]124
125 u8 tpl_levels;
126 u32 level_start_idx[];
127};
128
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]129enum udma_rm_range {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]130 RM_RANGE_BCHAN = 0,
131 RM_RANGE_TCHAN,
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]132 RM_RANGE_RCHAN,
133 RM_RANGE_RFLOW,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]134 RM_RANGE_TFLOW,
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]135 RM_RANGE_LAST,
136};
137
138struct udma_tisci_rm {
139 const struct ti_sci_handle *tisci;
140 const struct ti_sci_rm_udmap_ops *tisci_udmap_ops;
141 u32 tisci_dev_id;
142
143 /* tisci information for PSI-L thread pairing/unpairing */
144 const struct ti_sci_rm_psil_ops *tisci_psil_ops;
145 u32 tisci_navss_dev_id;
146
147 struct ti_sci_resource *rm_ranges[RM_RANGE_LAST];
148};
149
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]150struct udma_dev {
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]151 struct udevice *dev;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]152 void __iomem *mmrs[MMR_LAST];
153
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]154 struct udma_tisci_rm tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]155 struct k3_nav_ringacc *ringacc;
156
157 u32 features;
158
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]159 int bchan_cnt;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]160 int tchan_cnt;
161 int echan_cnt;
162 int rchan_cnt;
163 int rflow_cnt;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]164 int tflow_cnt;
165 unsigned long *bchan_map;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]166 unsigned long *tchan_map;
167 unsigned long *rchan_map;
168 unsigned long *rflow_map;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]169 unsigned long *rflow_map_reserved;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]170 unsigned long *tflow_map;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]171
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]172 struct udma_bchan *bchans;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]173 struct udma_tchan *tchans;
174 struct udma_rchan *rchans;
175 struct udma_rflow *rflows;
176
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]177 struct udma_match_data *match_data;
178
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]179 struct udma_chan *channels;
180 u32 psil_base;
181
182 u32 ch_count;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]183};
184
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]185struct udma_chan_config {
186 u32 psd_size; /* size of Protocol Specific Data */
187 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
188 u32 hdesc_size; /* Size of a packet descriptor in packet mode */
189 int remote_thread_id;
190 u32 atype;
191 u32 src_thread;
192 u32 dst_thread;
193 enum psil_endpoint_type ep_type;
194 enum udma_tp_level channel_tpl; /* Channel Throughput Level */
195
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]196 /* PKTDMA mapped channel */
197 int mapped_channel_id;
198 /* PKTDMA default tflow or rflow for mapped channel */
199 int default_flow_id;
200
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]201 enum dma_direction dir;
202
203 unsigned int pkt_mode:1; /* TR or packet */
204 unsigned int needs_epib:1; /* EPIB is needed for the communication or not */
205 unsigned int enable_acc32:1;
206 unsigned int enable_burst:1;
207 unsigned int notdpkt:1; /* Suppress sending TDC packet */
208};
209
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]210struct udma_chan {
211 struct udma_dev *ud;
212 char name[20];
213
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]214 struct udma_bchan *bchan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]215 struct udma_tchan *tchan;
216 struct udma_rchan *rchan;
217 struct udma_rflow *rflow;
218
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]219 struct ti_udma_drv_chan_cfg_data cfg_data;
220
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]221 u32 bcnt; /* number of bytes completed since the start of the channel */
222
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]223 struct udma_chan_config config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]224
225 u32 id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]226
227 struct cppi5_host_desc_t *desc_tx;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]228 bool in_use;
229 void *desc_rx;
230 u32 num_rx_bufs;
231 u32 desc_rx_cur;
232
233};
234
235#define UDMA_CH_1000(ch) (ch * 0x1000)
236#define UDMA_CH_100(ch) (ch * 0x100)
237#define UDMA_CH_40(ch) (ch * 0x40)
238
239#ifdef PKTBUFSRX
240#define UDMA_RX_DESC_NUM PKTBUFSRX
241#else
242#define UDMA_RX_DESC_NUM 4
243#endif
244
245/* Generic register access functions */
246static inline u32 udma_read(void __iomem *base, int reg)
247{
248 u32 v;
249
250 v = __raw_readl(base + reg);
251 pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, base + reg);
252 return v;
253}
254
255static inline void udma_write(void __iomem *base, int reg, u32 val)
256{
257 pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", val, base + reg);
258 __raw_writel(val, base + reg);
259}
260
261static inline void udma_update_bits(void __iomem *base, int reg,
262 u32 mask, u32 val)
263{
264 u32 tmp, orig;
265
266 orig = udma_read(base, reg);
267 tmp = orig & ~mask;
268 tmp |= (val & mask);
269
270 if (tmp != orig)
271 udma_write(base, reg, tmp);
272}
273
274/* TCHANRT */
275static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg)
276{
277 if (!tchan)
278 return 0;
279 return udma_read(tchan->reg_rt, reg);
280}
281
282static inline void udma_tchanrt_write(struct udma_tchan *tchan,
283 int reg, u32 val)
284{
285 if (!tchan)
286 return;
287 udma_write(tchan->reg_rt, reg, val);
288}
289
290/* RCHANRT */
291static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg)
292{
293 if (!rchan)
294 return 0;
295 return udma_read(rchan->reg_rt, reg);
296}
297
298static inline void udma_rchanrt_write(struct udma_rchan *rchan,
299 int reg, u32 val)
300{
301 if (!rchan)
302 return;
303 udma_write(rchan->reg_rt, reg, val);
304}
305
306static inline int udma_navss_psil_pair(struct udma_dev *ud, u32 src_thread,
307 u32 dst_thread)
308{
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]309 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
310
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]311 dst_thread |= UDMA_PSIL_DST_THREAD_ID_OFFSET;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]312
313 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
314 tisci_rm->tisci_navss_dev_id,
315 src_thread, dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]316}
317
318static inline int udma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
319 u32 dst_thread)
320{
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]321 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
322
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]323 dst_thread |= UDMA_PSIL_DST_THREAD_ID_OFFSET;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]324
325 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
326 tisci_rm->tisci_navss_dev_id,
327 src_thread, dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]328}
329
330static inline char *udma_get_dir_text(enum dma_direction dir)
331{
332 switch (dir) {
333 case DMA_DEV_TO_MEM:
334 return "DEV_TO_MEM";
335 case DMA_MEM_TO_DEV:
336 return "MEM_TO_DEV";
337 case DMA_MEM_TO_MEM:
338 return "MEM_TO_MEM";
339 case DMA_DEV_TO_DEV:
340 return "DEV_TO_DEV";
341 default:
342 break;
343 }
344
345 return "invalid";
346}
347
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]348#include "k3-udma-u-boot.c"
349
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]350static void udma_reset_uchan(struct udma_chan *uc)
351{
352 memset(&uc->config, 0, sizeof(uc->config));
353 uc->config.remote_thread_id = -1;
354 uc->config.mapped_channel_id = -1;
355 uc->config.default_flow_id = -1;
356}
357
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]358static inline bool udma_is_chan_running(struct udma_chan *uc)
359{
360 u32 trt_ctl = 0;
361 u32 rrt_ctl = 0;
362
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]363 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]364 case DMA_DEV_TO_MEM:
365 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
366 pr_debug("%s: rrt_ctl: 0x%08x (peer: 0x%08x)\n",
367 __func__, rrt_ctl,
368 udma_rchanrt_read(uc->rchan,
369 UDMA_RCHAN_RT_PEER_RT_EN_REG));
370 break;
371 case DMA_MEM_TO_DEV:
372 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
373 pr_debug("%s: trt_ctl: 0x%08x (peer: 0x%08x)\n",
374 __func__, trt_ctl,
375 udma_tchanrt_read(uc->tchan,
376 UDMA_TCHAN_RT_PEER_RT_EN_REG));
377 break;
378 case DMA_MEM_TO_MEM:
379 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
380 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
381 break;
382 default:
383 break;
384 }
385
386 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
387 return true;
388
389 return false;
390}
391
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]392static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
393{
394 struct k3_nav_ring *ring = NULL;
395 int ret = -ENOENT;
396
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]397 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]398 case DMA_DEV_TO_MEM:
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]399 ring = uc->rflow->r_ring;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]400 break;
401 case DMA_MEM_TO_DEV:
402 ring = uc->tchan->tc_ring;
403 break;
404 case DMA_MEM_TO_MEM:
405 ring = uc->tchan->tc_ring;
406 break;
407 default:
408 break;
409 }
410
411 if (ring && k3_nav_ringacc_ring_get_occ(ring))
412 ret = k3_nav_ringacc_ring_pop(ring, addr);
413
414 return ret;
415}
416
417static void udma_reset_rings(struct udma_chan *uc)
418{
419 struct k3_nav_ring *ring1 = NULL;
420 struct k3_nav_ring *ring2 = NULL;
421
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]422 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]423 case DMA_DEV_TO_MEM:
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]424 ring1 = uc->rflow->fd_ring;
425 ring2 = uc->rflow->r_ring;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]426 break;
427 case DMA_MEM_TO_DEV:
428 ring1 = uc->tchan->t_ring;
429 ring2 = uc->tchan->tc_ring;
430 break;
431 case DMA_MEM_TO_MEM:
432 ring1 = uc->tchan->t_ring;
433 ring2 = uc->tchan->tc_ring;
434 break;
435 default:
436 break;
437 }
438
439 if (ring1)
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]440 k3_nav_ringacc_ring_reset_dma(ring1, k3_nav_ringacc_ring_get_occ(ring1));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]441 if (ring2)
442 k3_nav_ringacc_ring_reset(ring2);
443}
444
445static void udma_reset_counters(struct udma_chan *uc)
446{
447 u32 val;
448
449 if (uc->tchan) {
450 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG);
451 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val);
452
453 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG);
454 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val);
455
456 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG);
457 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val);
458
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]459 if (!uc->bchan) {
460 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG);
461 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val);
462 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]463 }
464
465 if (uc->rchan) {
466 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG);
467 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val);
468
469 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG);
470 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val);
471
472 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG);
473 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val);
474
475 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG);
476 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val);
477 }
478
479 uc->bcnt = 0;
480}
481
482static inline int udma_stop_hard(struct udma_chan *uc)
483{
484 pr_debug("%s: ENTER (chan%d)\n", __func__, uc->id);
485
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]486 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]487 case DMA_DEV_TO_MEM:
488 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0);
489 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
490 break;
491 case DMA_MEM_TO_DEV:
492 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
493 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0);
494 break;
495 case DMA_MEM_TO_MEM:
496 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
497 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
498 break;
499 default:
500 return -EINVAL;
501 }
502
503 return 0;
504}
505
506static int udma_start(struct udma_chan *uc)
507{
508 /* Channel is already running, no need to proceed further */
509 if (udma_is_chan_running(uc))
510 goto out;
511
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]512 pr_debug("%s: chan:%d dir:%s\n",
513 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]514
515 /* Make sure that we clear the teardown bit, if it is set */
516 udma_stop_hard(uc);
517
518 /* Reset all counters */
519 udma_reset_counters(uc);
520
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]521 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]522 case DMA_DEV_TO_MEM:
523 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG,
524 UDMA_CHAN_RT_CTL_EN);
525
526 /* Enable remote */
527 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG,
528 UDMA_PEER_RT_EN_ENABLE);
529
530 pr_debug("%s(rx): RT_CTL:0x%08x PEER RT_ENABLE:0x%08x\n",
531 __func__,
532 udma_rchanrt_read(uc->rchan,
533 UDMA_RCHAN_RT_CTL_REG),
534 udma_rchanrt_read(uc->rchan,
535 UDMA_RCHAN_RT_PEER_RT_EN_REG));
536 break;
537 case DMA_MEM_TO_DEV:
538 /* Enable remote */
539 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG,
540 UDMA_PEER_RT_EN_ENABLE);
541
542 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
543 UDMA_CHAN_RT_CTL_EN);
544
545 pr_debug("%s(tx): RT_CTL:0x%08x PEER RT_ENABLE:0x%08x\n",
546 __func__,
Vignesh Raghavendrac2237992019-12-09 10:25:36 +0530[diff] [blame]547 udma_tchanrt_read(uc->tchan,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]548 UDMA_TCHAN_RT_CTL_REG),
Vignesh Raghavendrac2237992019-12-09 10:25:36 +0530[diff] [blame]549 udma_tchanrt_read(uc->tchan,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]550 UDMA_TCHAN_RT_PEER_RT_EN_REG));
551 break;
552 case DMA_MEM_TO_MEM:
553 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG,
554 UDMA_CHAN_RT_CTL_EN);
555 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
556 UDMA_CHAN_RT_CTL_EN);
557
558 break;
559 default:
560 return -EINVAL;
561 }
562
563 pr_debug("%s: DONE chan:%d\n", __func__, uc->id);
564out:
565 return 0;
566}
567
568static inline void udma_stop_mem2dev(struct udma_chan *uc, bool sync)
569{
570 int i = 0;
571 u32 val;
572
573 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
574 UDMA_CHAN_RT_CTL_EN |
575 UDMA_CHAN_RT_CTL_TDOWN);
576
577 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
578
579 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
580 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
581 udelay(1);
582 if (i > 1000) {
583 printf(" %s TIMEOUT !\n", __func__);
584 break;
585 }
586 i++;
587 }
588
589 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG);
590 if (val & UDMA_PEER_RT_EN_ENABLE)
591 printf("%s: peer not stopped TIMEOUT !\n", __func__);
592}
593
594static inline void udma_stop_dev2mem(struct udma_chan *uc, bool sync)
595{
596 int i = 0;
597 u32 val;
598
599 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG,
600 UDMA_PEER_RT_EN_ENABLE |
601 UDMA_PEER_RT_EN_TEARDOWN);
602
603 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
604
605 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
606 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
607 udelay(1);
608 if (i > 1000) {
609 printf("%s TIMEOUT !\n", __func__);
610 break;
611 }
612 i++;
613 }
614
615 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG);
616 if (val & UDMA_PEER_RT_EN_ENABLE)
617 printf("%s: peer not stopped TIMEOUT !\n", __func__);
618}
619
620static inline int udma_stop(struct udma_chan *uc)
621{
622 pr_debug("%s: chan:%d dir:%s\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]623 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]624
625 udma_reset_counters(uc);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]626 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]627 case DMA_DEV_TO_MEM:
628 udma_stop_dev2mem(uc, true);
629 break;
630 case DMA_MEM_TO_DEV:
631 udma_stop_mem2dev(uc, true);
632 break;
633 case DMA_MEM_TO_MEM:
634 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
635 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
636 break;
637 default:
638 return -EINVAL;
639 }
640
641 return 0;
642}
643
644static void udma_poll_completion(struct udma_chan *uc, dma_addr_t *paddr)
645{
646 int i = 1;
647
648 while (udma_pop_from_ring(uc, paddr)) {
649 udelay(1);
650 if (!(i % 1000000))
651 printf(".");
652 i++;
653 }
654}
655
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]656static struct udma_rflow *__udma_reserve_rflow(struct udma_dev *ud, int id)
657{
658 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
659
660 if (id >= 0) {
661 if (test_bit(id, ud->rflow_map)) {
662 dev_err(ud->dev, "rflow%d is in use\n", id);
663 return ERR_PTR(-ENOENT);
664 }
665 } else {
666 bitmap_or(tmp, ud->rflow_map, ud->rflow_map_reserved,
667 ud->rflow_cnt);
668
669 id = find_next_zero_bit(tmp, ud->rflow_cnt, ud->rchan_cnt);
670 if (id >= ud->rflow_cnt)
671 return ERR_PTR(-ENOENT);
672 }
673
674 __set_bit(id, ud->rflow_map);
675 return &ud->rflows[id];
676}
677
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]678#define UDMA_RESERVE_RESOURCE(res) \
679static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \
680 int id) \
681{ \
682 if (id >= 0) { \
683 if (test_bit(id, ud->res##_map)) { \
684 dev_err(ud->dev, "res##%d is in use\n", id); \
685 return ERR_PTR(-ENOENT); \
686 } \
687 } else { \
688 id = find_first_zero_bit(ud->res##_map, ud->res##_cnt); \
689 if (id == ud->res##_cnt) { \
690 return ERR_PTR(-ENOENT); \
691 } \
692 } \
693 \
694 __set_bit(id, ud->res##_map); \
695 return &ud->res##s[id]; \
696}
697
698UDMA_RESERVE_RESOURCE(tchan);
699UDMA_RESERVE_RESOURCE(rchan);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]700
701static int udma_get_tchan(struct udma_chan *uc)
702{
703 struct udma_dev *ud = uc->ud;
704
705 if (uc->tchan) {
706 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
707 uc->id, uc->tchan->id);
708 return 0;
709 }
710
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]711 uc->tchan = __udma_reserve_tchan(ud, uc->config.mapped_channel_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]712 if (IS_ERR(uc->tchan))
713 return PTR_ERR(uc->tchan);
714
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]715 if (ud->tflow_cnt) {
716 int tflow_id;
717
718 /* Only PKTDMA have support for tx flows */
719 if (uc->config.default_flow_id >= 0)
720 tflow_id = uc->config.default_flow_id;
721 else
722 tflow_id = uc->tchan->id;
723
724 if (test_bit(tflow_id, ud->tflow_map)) {
725 dev_err(ud->dev, "tflow%d is in use\n", tflow_id);
726 __clear_bit(uc->tchan->id, ud->tchan_map);
727 uc->tchan = NULL;
728 return -ENOENT;
729 }
730
731 uc->tchan->tflow_id = tflow_id;
732 __set_bit(tflow_id, ud->tflow_map);
733 } else {
734 uc->tchan->tflow_id = -1;
735 }
736
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]737 pr_debug("chan%d: got tchan%d\n", uc->id, uc->tchan->id);
738
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]739 return 0;
740}
741
742static int udma_get_rchan(struct udma_chan *uc)
743{
744 struct udma_dev *ud = uc->ud;
745
746 if (uc->rchan) {
747 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
748 uc->id, uc->rchan->id);
749 return 0;
750 }
751
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]752 uc->rchan = __udma_reserve_rchan(ud, uc->config.mapped_channel_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]753 if (IS_ERR(uc->rchan))
754 return PTR_ERR(uc->rchan);
755
756 pr_debug("chan%d: got rchan%d\n", uc->id, uc->rchan->id);
757
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]758 return 0;
759}
760
761static int udma_get_chan_pair(struct udma_chan *uc)
762{
763 struct udma_dev *ud = uc->ud;
764 int chan_id, end;
765
766 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
767 dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
768 uc->id, uc->tchan->id);
769 return 0;
770 }
771
772 if (uc->tchan) {
773 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
774 uc->id, uc->tchan->id);
775 return -EBUSY;
776 } else if (uc->rchan) {
777 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
778 uc->id, uc->rchan->id);
779 return -EBUSY;
780 }
781
782 /* Can be optimized, but let's have it like this for now */
783 end = min(ud->tchan_cnt, ud->rchan_cnt);
784 for (chan_id = 0; chan_id < end; chan_id++) {
785 if (!test_bit(chan_id, ud->tchan_map) &&
786 !test_bit(chan_id, ud->rchan_map))
787 break;
788 }
789
790 if (chan_id == end)
791 return -ENOENT;
792
793 __set_bit(chan_id, ud->tchan_map);
794 __set_bit(chan_id, ud->rchan_map);
795 uc->tchan = &ud->tchans[chan_id];
796 uc->rchan = &ud->rchans[chan_id];
797
798 pr_debug("chan%d: got t/rchan%d pair\n", uc->id, chan_id);
799
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]800 return 0;
801}
802
803static int udma_get_rflow(struct udma_chan *uc, int flow_id)
804{
805 struct udma_dev *ud = uc->ud;
806
807 if (uc->rflow) {
808 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
809 uc->id, uc->rflow->id);
810 return 0;
811 }
812
813 if (!uc->rchan)
814 dev_warn(ud->dev, "chan%d: does not have rchan??\n", uc->id);
815
816 uc->rflow = __udma_reserve_rflow(ud, flow_id);
817 if (IS_ERR(uc->rflow))
818 return PTR_ERR(uc->rflow);
819
820 pr_debug("chan%d: got rflow%d\n", uc->id, uc->rflow->id);
821 return 0;
822}
823
824static void udma_put_rchan(struct udma_chan *uc)
825{
826 struct udma_dev *ud = uc->ud;
827
828 if (uc->rchan) {
829 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
830 uc->rchan->id);
831 __clear_bit(uc->rchan->id, ud->rchan_map);
832 uc->rchan = NULL;
833 }
834}
835
836static void udma_put_tchan(struct udma_chan *uc)
837{
838 struct udma_dev *ud = uc->ud;
839
840 if (uc->tchan) {
841 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
842 uc->tchan->id);
843 __clear_bit(uc->tchan->id, ud->tchan_map);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]844 if (uc->tchan->tflow_id >= 0)
845 __clear_bit(uc->tchan->tflow_id, ud->tflow_map);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]846 uc->tchan = NULL;
847 }
848}
849
850static void udma_put_rflow(struct udma_chan *uc)
851{
852 struct udma_dev *ud = uc->ud;
853
854 if (uc->rflow) {
855 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
856 uc->rflow->id);
857 __clear_bit(uc->rflow->id, ud->rflow_map);
858 uc->rflow = NULL;
859 }
860}
861
862static void udma_free_tx_resources(struct udma_chan *uc)
863{
864 if (!uc->tchan)
865 return;
866
867 k3_nav_ringacc_ring_free(uc->tchan->t_ring);
868 k3_nav_ringacc_ring_free(uc->tchan->tc_ring);
869 uc->tchan->t_ring = NULL;
870 uc->tchan->tc_ring = NULL;
871
872 udma_put_tchan(uc);
873}
874
875static int udma_alloc_tx_resources(struct udma_chan *uc)
876{
877 struct k3_nav_ring_cfg ring_cfg;
878 struct udma_dev *ud = uc->ud;
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]879 struct udma_tchan *tchan;
880 int ring_idx, ret;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]881
882 ret = udma_get_tchan(uc);
883 if (ret)
884 return ret;
885
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]886 tchan = uc->tchan;
Udit Kumarf084e402024-02-21 19:53:44 +0530[diff] [blame]887 if (tchan->tflow_id > 0)
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]888 ring_idx = tchan->tflow_id;
889 else
Udit Kumarf084e402024-02-21 19:53:44 +0530[diff] [blame]890 ring_idx = tchan->id;
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]891
892 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1,
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]893 &uc->tchan->t_ring,
894 &uc->tchan->tc_ring);
895 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]896 ret = -EBUSY;
897 goto err_tx_ring;
898 }
899
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]900 memset(&ring_cfg, 0, sizeof(ring_cfg));
901 ring_cfg.size = 16;
902 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
Vignesh Raghavendra0fe24d32019-12-09 10:25:37 +0530[diff] [blame]903 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]904
905 ret = k3_nav_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg);
906 ret |= k3_nav_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg);
907
908 if (ret)
909 goto err_ringcfg;
910
911 return 0;
912
913err_ringcfg:
914 k3_nav_ringacc_ring_free(uc->tchan->tc_ring);
915 uc->tchan->tc_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]916 k3_nav_ringacc_ring_free(uc->tchan->t_ring);
917 uc->tchan->t_ring = NULL;
918err_tx_ring:
919 udma_put_tchan(uc);
920
921 return ret;
922}
923
924static void udma_free_rx_resources(struct udma_chan *uc)
925{
926 if (!uc->rchan)
927 return;
928
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]929 if (uc->rflow) {
930 k3_nav_ringacc_ring_free(uc->rflow->fd_ring);
931 k3_nav_ringacc_ring_free(uc->rflow->r_ring);
932 uc->rflow->fd_ring = NULL;
933 uc->rflow->r_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]934
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]935 udma_put_rflow(uc);
936 }
937
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]938 udma_put_rchan(uc);
939}
940
941static int udma_alloc_rx_resources(struct udma_chan *uc)
942{
943 struct k3_nav_ring_cfg ring_cfg;
944 struct udma_dev *ud = uc->ud;
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]945 struct udma_rflow *rflow;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]946 int fd_ring_id;
947 int ret;
948
949 ret = udma_get_rchan(uc);
950 if (ret)
951 return ret;
952
953 /* For MEM_TO_MEM we don't need rflow or rings */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]954 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]955 return 0;
956
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]957 if (uc->config.default_flow_id >= 0)
958 ret = udma_get_rflow(uc, uc->config.default_flow_id);
959 else
960 ret = udma_get_rflow(uc, uc->rchan->id);
961
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]962 if (ret) {
963 ret = -EBUSY;
964 goto err_rflow;
965 }
966
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]967 rflow = uc->rflow;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]968 if (ud->tflow_cnt) {
969 fd_ring_id = ud->tflow_cnt + rflow->id;
970 } else {
971 fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt +
972 uc->rchan->id;
973 }
974
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]975 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
976 &rflow->fd_ring, &rflow->r_ring);
977 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]978 ret = -EBUSY;
979 goto err_rx_ring;
980 }
981
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]982 memset(&ring_cfg, 0, sizeof(ring_cfg));
983 ring_cfg.size = 16;
984 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
Vignesh Raghavendra0fe24d32019-12-09 10:25:37 +0530[diff] [blame]985 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]986
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]987 ret = k3_nav_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
988 ret |= k3_nav_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]989 if (ret)
990 goto err_ringcfg;
991
992 return 0;
993
994err_ringcfg:
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]995 k3_nav_ringacc_ring_free(rflow->r_ring);
996 rflow->r_ring = NULL;
997 k3_nav_ringacc_ring_free(rflow->fd_ring);
998 rflow->fd_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]999err_rx_ring:
1000 udma_put_rflow(uc);
1001err_rflow:
1002 udma_put_rchan(uc);
1003
1004 return ret;
1005}
1006
1007static int udma_alloc_tchan_sci_req(struct udma_chan *uc)
1008{
1009 struct udma_dev *ud = uc->ud;
1010 int tc_ring = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1011 struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1012 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1013 u32 mode;
1014 int ret;
1015
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1016 if (uc->config.pkt_mode)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1017 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1018 else
1019 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1020
1021 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
1022 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
1023 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1024 req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1025 req.index = uc->tchan->id;
1026 req.tx_chan_type = mode;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1027 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1028 req.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1029 else
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1030 req.tx_fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1031 uc->config.psd_size,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1032 0) >> 2;
1033 req.txcq_qnum = tc_ring;
1034
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1035 ret = tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1036 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1037 dev_err(ud->dev, "tisci tx alloc failed %d\n", ret);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1038 return ret;
1039 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1040
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1041 /*
1042 * Above TI SCI call handles firewall configuration, cfg
1043 * register configuration still has to be done locally in
1044 * absence of RM services.
1045 */
1046 if (IS_ENABLED(CONFIG_K3_DM_FW))
1047 udma_alloc_tchan_raw(uc);
1048
1049 return 0;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1050}
1051
1052static int udma_alloc_rchan_sci_req(struct udma_chan *uc)
1053{
1054 struct udma_dev *ud = uc->ud;
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]1055 int fd_ring = k3_nav_ringacc_get_ring_id(uc->rflow->fd_ring);
1056 int rx_ring = k3_nav_ringacc_get_ring_id(uc->rflow->r_ring);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1057 int tc_ring = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1058 struct ti_sci_msg_rm_udmap_rx_ch_cfg req = { 0 };
1059 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1060 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1061 u32 mode;
1062 int ret;
1063
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1064 if (uc->config.pkt_mode)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1065 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1066 else
1067 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1068
1069 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
1070 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1071 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1072 req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1073 req.index = uc->rchan->id;
1074 req.rx_chan_type = mode;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1075 if (uc->config.dir == DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1076 req.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1077 req.rxcq_qnum = tc_ring;
1078 } else {
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1079 req.rx_fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1080 uc->config.psd_size,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1081 0) >> 2;
1082 req.rxcq_qnum = rx_ring;
1083 }
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1084 if (ud->match_data->type == DMA_TYPE_UDMA &&
1085 uc->rflow->id != uc->rchan->id &&
1086 uc->config.dir != DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1087 req.flowid_start = uc->rflow->id;
1088 req.flowid_cnt = 1;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1089 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
1090 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1091 }
1092
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1093 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1094 if (ret) {
1095 dev_err(ud->dev, "tisci rx %u cfg failed %d\n",
1096 uc->rchan->id, ret);
1097 return ret;
1098 }
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1099 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1100 return ret;
1101
1102 flow_req.valid_params =
1103 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
1104 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
1105 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
1106 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
1107 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1108 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
1109 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
1110 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
1111 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
1112 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1113 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1114 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1115 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID |
1116 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PS_LOCATION_VALID;
1117
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1118 flow_req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1119 flow_req.flow_index = uc->rflow->id;
1120
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1121 if (uc->config.needs_epib)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1122 flow_req.rx_einfo_present = 1;
1123 else
1124 flow_req.rx_einfo_present = 0;
1125
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1126 if (uc->config.psd_size)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1127 flow_req.rx_psinfo_present = 1;
1128 else
1129 flow_req.rx_psinfo_present = 0;
1130
1131 flow_req.rx_error_handling = 0;
1132 flow_req.rx_desc_type = 0;
1133 flow_req.rx_dest_qnum = rx_ring;
1134 flow_req.rx_src_tag_hi_sel = 2;
1135 flow_req.rx_src_tag_lo_sel = 4;
1136 flow_req.rx_dest_tag_hi_sel = 5;
1137 flow_req.rx_dest_tag_lo_sel = 4;
1138 flow_req.rx_fdq0_sz0_qnum = fd_ring;
1139 flow_req.rx_fdq1_qnum = fd_ring;
1140 flow_req.rx_fdq2_qnum = fd_ring;
1141 flow_req.rx_fdq3_qnum = fd_ring;
1142 flow_req.rx_ps_location = 0;
1143
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1144 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci,
1145 &flow_req);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1146 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1147 dev_err(ud->dev, "tisci rx %u flow %u cfg failed %d\n",
1148 uc->rchan->id, uc->rflow->id, ret);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1149 return ret;
1150 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1151
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1152 /*
1153 * Above TI SCI call handles firewall configuration, cfg
1154 * register configuration still has to be done locally in
1155 * absence of RM services.
1156 */
1157 if (IS_ENABLED(CONFIG_K3_DM_FW))
1158 udma_alloc_rchan_raw(uc);
1159
1160 return 0;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1161}
1162
1163static int udma_alloc_chan_resources(struct udma_chan *uc)
1164{
1165 struct udma_dev *ud = uc->ud;
1166 int ret;
1167
1168 pr_debug("%s: chan:%d as %s\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1169 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1170
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1171 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1172 case DMA_MEM_TO_MEM:
1173 /* Non synchronized - mem to mem type of transfer */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1174 uc->config.pkt_mode = false;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1175 ret = udma_get_chan_pair(uc);
1176 if (ret)
1177 return ret;
1178
1179 ret = udma_alloc_tx_resources(uc);
1180 if (ret)
1181 goto err_free_res;
1182
1183 ret = udma_alloc_rx_resources(uc);
1184 if (ret)
1185 goto err_free_res;
1186
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1187 uc->config.src_thread = ud->psil_base + uc->tchan->id;
1188 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1189 break;
1190 case DMA_MEM_TO_DEV:
1191 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
1192 ret = udma_alloc_tx_resources(uc);
1193 if (ret)
1194 goto err_free_res;
1195
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1196 uc->config.src_thread = ud->psil_base + uc->tchan->id;
1197 uc->config.dst_thread = uc->config.remote_thread_id;
1198 uc->config.dst_thread |= 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1199
1200 break;
1201 case DMA_DEV_TO_MEM:
1202 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
1203 ret = udma_alloc_rx_resources(uc);
1204 if (ret)
1205 goto err_free_res;
1206
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1207 uc->config.src_thread = uc->config.remote_thread_id;
1208 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1209
1210 break;
1211 default:
1212 /* Can not happen */
1213 pr_debug("%s: chan:%d invalid direction (%u)\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1214 __func__, uc->id, uc->config.dir);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1215 return -EINVAL;
1216 }
1217
1218 /* We have channel indexes and rings */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1219 if (uc->config.dir == DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1220 ret = udma_alloc_tchan_sci_req(uc);
1221 if (ret)
1222 goto err_free_res;
1223
1224 ret = udma_alloc_rchan_sci_req(uc);
1225 if (ret)
1226 goto err_free_res;
1227 } else {
1228 /* Slave transfer */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1229 if (uc->config.dir == DMA_MEM_TO_DEV) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1230 ret = udma_alloc_tchan_sci_req(uc);
1231 if (ret)
1232 goto err_free_res;
1233 } else {
1234 ret = udma_alloc_rchan_sci_req(uc);
1235 if (ret)
1236 goto err_free_res;
1237 }
1238 }
1239
Peter Ujfalusid15f8652019-04-25 12:08:15 +0530[diff] [blame]1240 if (udma_is_chan_running(uc)) {
1241 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
1242 udma_stop(uc);
1243 if (udma_is_chan_running(uc)) {
1244 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
1245 goto err_free_res;
1246 }
1247 }
1248
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1249 /* PSI-L pairing */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1250 ret = udma_navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1251 if (ret) {
1252 dev_err(ud->dev, "k3_nav_psil_request_link fail\n");
1253 goto err_free_res;
1254 }
1255
1256 return 0;
1257
1258err_free_res:
1259 udma_free_tx_resources(uc);
1260 udma_free_rx_resources(uc);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1261 uc->config.remote_thread_id = -1;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1262 return ret;
1263}
1264
1265static void udma_free_chan_resources(struct udma_chan *uc)
1266{
Vignesh Raghavendrabe7bdcc2020-09-17 20:11:22 +0530[diff] [blame]1267 /* Hard reset UDMA channel */
1268 udma_stop_hard(uc);
1269 udma_reset_counters(uc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1270
1271 /* Release PSI-L pairing */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1272 udma_navss_psil_unpair(uc->ud, uc->config.src_thread, uc->config.dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1273
1274 /* Reset the rings for a new start */
1275 udma_reset_rings(uc);
1276 udma_free_tx_resources(uc);
1277 udma_free_rx_resources(uc);
1278
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1279 uc->config.remote_thread_id = -1;
1280 uc->config.dir = DMA_MEM_TO_MEM;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1281}
1282
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1283static const char * const range_names[] = {
1284 [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan",
1285 [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan",
1286 [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan",
1287 [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow",
1288 [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow",
1289};
1290
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1291static int udma_get_mmrs(struct udevice *dev)
1292{
1293 struct udma_dev *ud = dev_get_priv(dev);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1294 u32 cap2, cap3, cap4;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1295 int i;
1296
Matthias Schiffer47331932023-09-27 15:33:34 +0200[diff] [blame]1297 ud->mmrs[MMR_GCFG] = dev_read_addr_name_ptr(dev, mmr_names[MMR_GCFG]);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1298 if (!ud->mmrs[MMR_GCFG])
1299 return -EINVAL;
1300
1301 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);
1302 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
1303
1304 switch (ud->match_data->type) {
1305 case DMA_TYPE_UDMA:
1306 ud->rflow_cnt = cap3 & 0x3fff;
1307 ud->tchan_cnt = cap2 & 0x1ff;
1308 ud->echan_cnt = (cap2 >> 9) & 0x1ff;
1309 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1310 break;
1311 case DMA_TYPE_BCDMA:
1312 ud->bchan_cnt = cap2 & 0x1ff;
1313 ud->tchan_cnt = (cap2 >> 9) & 0x1ff;
1314 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1315 break;
1316 case DMA_TYPE_PKTDMA:
1317 cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30);
1318 ud->tchan_cnt = cap2 & 0x1ff;
1319 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1320 ud->rflow_cnt = cap3 & 0x3fff;
1321 ud->tflow_cnt = cap4 & 0x3fff;
1322 break;
1323 default:
1324 return -EINVAL;
1325 }
1326
1327 for (i = 1; i < MMR_LAST; i++) {
1328 if (i == MMR_BCHANRT && ud->bchan_cnt == 0)
1329 continue;
1330 if (i == MMR_TCHANRT && ud->tchan_cnt == 0)
1331 continue;
1332 if (i == MMR_RCHANRT && ud->rchan_cnt == 0)
1333 continue;
Prasanth Babu Mantena32911232024-12-18 18:30:45 +0530[diff] [blame^]1334 if (i == MMR_RFLOW && ud->match_data->type == DMA_TYPE_BCDMA)
1335 continue;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1336
Matthias Schiffer47331932023-09-27 15:33:34 +0200[diff] [blame]1337 ud->mmrs[i] = dev_read_addr_name_ptr(dev, mmr_names[i]);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1338 if (!ud->mmrs[i])
1339 return -EINVAL;
1340 }
1341
1342 return 0;
1343}
1344
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1345static int udma_setup_resources(struct udma_dev *ud)
1346{
1347 struct udevice *dev = ud->dev;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1348 int i;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1349 struct ti_sci_resource_desc *rm_desc;
1350 struct ti_sci_resource *rm_res;
1351 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1352
1353 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1354 sizeof(unsigned long), GFP_KERNEL);
1355 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1356 GFP_KERNEL);
1357 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1358 sizeof(unsigned long), GFP_KERNEL);
1359 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1360 GFP_KERNEL);
1361 ud->rflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
1362 sizeof(unsigned long), GFP_KERNEL);
1363 ud->rflow_map_reserved = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
1364 sizeof(unsigned long),
1365 GFP_KERNEL);
1366 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
1367 GFP_KERNEL);
1368
1369 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_map ||
1370 !ud->rflow_map_reserved || !ud->tchans || !ud->rchans ||
1371 !ud->rflows)
1372 return -ENOMEM;
1373
1374 /*
1375 * RX flows with the same Ids as RX channels are reserved to be used
1376 * as default flows if remote HW can't generate flow_ids. Those
1377 * RX flows can be requested only explicitly by id.
1378 */
1379 bitmap_set(ud->rflow_map_reserved, 0, ud->rchan_cnt);
1380
1381 /* Get resource ranges from tisci */
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1382 for (i = 0; i < RM_RANGE_LAST; i++) {
1383 if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW)
1384 continue;
1385
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1386 tisci_rm->rm_ranges[i] =
1387 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1388 tisci_rm->tisci_dev_id,
1389 (char *)range_names[i]);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1390 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1391
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1392 /* tchan ranges */
1393 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1394 if (IS_ERR(rm_res)) {
1395 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1396 } else {
1397 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1398 for (i = 0; i < rm_res->sets; i++) {
1399 rm_desc = &rm_res->desc[i];
1400 bitmap_clear(ud->tchan_map, rm_desc->start,
1401 rm_desc->num);
1402 }
1403 }
1404
1405 /* rchan and matching default flow ranges */
1406 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1407 if (IS_ERR(rm_res)) {
1408 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1409 bitmap_zero(ud->rflow_map, ud->rchan_cnt);
1410 } else {
1411 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1412 bitmap_fill(ud->rflow_map, ud->rchan_cnt);
1413 for (i = 0; i < rm_res->sets; i++) {
1414 rm_desc = &rm_res->desc[i];
1415 bitmap_clear(ud->rchan_map, rm_desc->start,
1416 rm_desc->num);
1417 bitmap_clear(ud->rflow_map, rm_desc->start,
1418 rm_desc->num);
1419 }
1420 }
1421
1422 /* GP rflow ranges */
1423 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
1424 if (IS_ERR(rm_res)) {
1425 bitmap_clear(ud->rflow_map, ud->rchan_cnt,
1426 ud->rflow_cnt - ud->rchan_cnt);
1427 } else {
1428 bitmap_set(ud->rflow_map, ud->rchan_cnt,
1429 ud->rflow_cnt - ud->rchan_cnt);
1430 for (i = 0; i < rm_res->sets; i++) {
1431 rm_desc = &rm_res->desc[i];
1432 bitmap_clear(ud->rflow_map, rm_desc->start,
1433 rm_desc->num);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1434 }
1435 }
1436
1437 return 0;
1438}
1439
1440static int bcdma_setup_resources(struct udma_dev *ud)
1441{
1442 int i;
1443 struct udevice *dev = ud->dev;
1444 struct ti_sci_resource_desc *rm_desc;
1445 struct ti_sci_resource *rm_res;
1446 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1447
1448 ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt),
1449 sizeof(unsigned long), GFP_KERNEL);
1450 ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans),
1451 GFP_KERNEL);
1452 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1453 sizeof(unsigned long), GFP_KERNEL);
1454 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1455 GFP_KERNEL);
1456 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1457 sizeof(unsigned long), GFP_KERNEL);
1458 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1459 GFP_KERNEL);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1460 ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows),
1461 GFP_KERNEL);
1462
1463 if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map ||
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1464 !ud->bchans || !ud->tchans || !ud->rchans ||
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1465 !ud->rflows)
1466 return -ENOMEM;
1467
1468 /* Get resource ranges from tisci */
1469 for (i = 0; i < RM_RANGE_LAST; i++) {
1470 if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW)
1471 continue;
1472
1473 tisci_rm->rm_ranges[i] =
1474 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1475 tisci_rm->tisci_dev_id,
1476 (char *)range_names[i]);
1477 }
1478
1479 /* bchan ranges */
1480 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
1481 if (IS_ERR(rm_res)) {
1482 bitmap_zero(ud->bchan_map, ud->bchan_cnt);
1483 } else {
1484 bitmap_fill(ud->bchan_map, ud->bchan_cnt);
1485 for (i = 0; i < rm_res->sets; i++) {
1486 rm_desc = &rm_res->desc[i];
1487 bitmap_clear(ud->bchan_map, rm_desc->start,
1488 rm_desc->num);
1489 dev_dbg(dev, "ti-sci-res: bchan: %d:%d\n",
1490 rm_desc->start, rm_desc->num);
1491 }
1492 }
1493
1494 /* tchan ranges */
1495 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1496 if (IS_ERR(rm_res)) {
1497 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1498 } else {
1499 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1500 for (i = 0; i < rm_res->sets; i++) {
1501 rm_desc = &rm_res->desc[i];
1502 bitmap_clear(ud->tchan_map, rm_desc->start,
1503 rm_desc->num);
1504 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
1505 rm_desc->start, rm_desc->num);
1506 }
1507 }
1508
1509 /* rchan ranges */
1510 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1511 if (IS_ERR(rm_res)) {
1512 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1513 } else {
1514 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1515 for (i = 0; i < rm_res->sets; i++) {
1516 rm_desc = &rm_res->desc[i];
1517 bitmap_clear(ud->rchan_map, rm_desc->start,
1518 rm_desc->num);
1519 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
1520 rm_desc->start, rm_desc->num);
1521 }
1522 }
1523
1524 return 0;
1525}
1526
1527static int pktdma_setup_resources(struct udma_dev *ud)
1528{
1529 int i;
1530 struct udevice *dev = ud->dev;
1531 struct ti_sci_resource *rm_res;
1532 struct ti_sci_resource_desc *rm_desc;
1533 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1534
1535 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1536 sizeof(unsigned long), GFP_KERNEL);
1537 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1538 GFP_KERNEL);
1539 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1540 sizeof(unsigned long), GFP_KERNEL);
1541 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1542 GFP_KERNEL);
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1543 ud->rflow_map = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
1544 sizeof(unsigned long),
1545 GFP_KERNEL);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1546 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
1547 GFP_KERNEL);
1548 ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt),
1549 sizeof(unsigned long), GFP_KERNEL);
1550
1551 if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans ||
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1552 !ud->rchans || !ud->rflows || !ud->rflow_map)
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1553 return -ENOMEM;
1554
1555 /* Get resource ranges from tisci */
1556 for (i = 0; i < RM_RANGE_LAST; i++) {
1557 if (i == RM_RANGE_BCHAN)
1558 continue;
1559
1560 tisci_rm->rm_ranges[i] =
1561 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1562 tisci_rm->tisci_dev_id,
1563 (char *)range_names[i]);
1564 }
1565
1566 /* tchan ranges */
1567 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1568 if (IS_ERR(rm_res)) {
1569 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1570 } else {
1571 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1572 for (i = 0; i < rm_res->sets; i++) {
1573 rm_desc = &rm_res->desc[i];
1574 bitmap_clear(ud->tchan_map, rm_desc->start,
1575 rm_desc->num);
1576 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
1577 rm_desc->start, rm_desc->num);
1578 }
1579 }
1580
1581 /* rchan ranges */
1582 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1583 if (IS_ERR(rm_res)) {
1584 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1585 } else {
1586 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1587 for (i = 0; i < rm_res->sets; i++) {
1588 rm_desc = &rm_res->desc[i];
1589 bitmap_clear(ud->rchan_map, rm_desc->start,
1590 rm_desc->num);
1591 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
1592 rm_desc->start, rm_desc->num);
1593 }
1594 }
1595
1596 /* rflow ranges */
1597 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
1598 if (IS_ERR(rm_res)) {
1599 /* all rflows are assigned exclusively to Linux */
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1600 bitmap_zero(ud->rflow_map, ud->rflow_cnt);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1601 } else {
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1602 bitmap_fill(ud->rflow_map, ud->rflow_cnt);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1603 for (i = 0; i < rm_res->sets; i++) {
1604 rm_desc = &rm_res->desc[i];
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1605 bitmap_clear(ud->rflow_map, rm_desc->start,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1606 rm_desc->num);
1607 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n",
1608 rm_desc->start, rm_desc->num);
1609 }
1610 }
1611
1612 /* tflow ranges */
1613 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
1614 if (IS_ERR(rm_res)) {
1615 /* all tflows are assigned exclusively to Linux */
1616 bitmap_zero(ud->tflow_map, ud->tflow_cnt);
1617 } else {
1618 bitmap_fill(ud->tflow_map, ud->tflow_cnt);
1619 for (i = 0; i < rm_res->sets; i++) {
1620 rm_desc = &rm_res->desc[i];
1621 bitmap_clear(ud->tflow_map, rm_desc->start,
1622 rm_desc->num);
1623 dev_dbg(dev, "ti-sci-res: tflow: %d:%d\n",
1624 rm_desc->start, rm_desc->num);
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1625 }
1626 }
1627
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1628 return 0;
1629}
1630
1631static int setup_resources(struct udma_dev *ud)
1632{
1633 struct udevice *dev = ud->dev;
1634 int ch_count, ret;
1635
1636 switch (ud->match_data->type) {
1637 case DMA_TYPE_UDMA:
1638 ret = udma_setup_resources(ud);
1639 break;
1640 case DMA_TYPE_BCDMA:
1641 ret = bcdma_setup_resources(ud);
1642 break;
1643 case DMA_TYPE_PKTDMA:
1644 ret = pktdma_setup_resources(ud);
1645 break;
1646 default:
1647 return -EINVAL;
1648 }
1649
1650 if (ret)
1651 return ret;
1652
1653 ch_count = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt;
1654 if (ud->bchan_cnt)
1655 ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt);
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1656 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
1657 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
1658 if (!ch_count)
1659 return -ENODEV;
1660
1661 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
1662 GFP_KERNEL);
1663 if (!ud->channels)
1664 return -ENOMEM;
1665
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1666 switch (ud->match_data->type) {
1667 case DMA_TYPE_UDMA:
1668 dev_dbg(dev,
1669 "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
1670 ch_count,
1671 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1672 ud->tchan_cnt),
1673 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1674 ud->rchan_cnt),
1675 ud->rflow_cnt - bitmap_weight(ud->rflow_map,
1676 ud->rflow_cnt));
1677 break;
1678 case DMA_TYPE_BCDMA:
1679 dev_dbg(dev,
1680 "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n",
1681 ch_count,
1682 ud->bchan_cnt - bitmap_weight(ud->bchan_map,
1683 ud->bchan_cnt),
1684 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1685 ud->tchan_cnt),
1686 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1687 ud->rchan_cnt));
1688 break;
1689 case DMA_TYPE_PKTDMA:
1690 dev_dbg(dev,
1691 "Channels: %d (tchan: %u, rchan: %u)\n",
1692 ch_count,
1693 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1694 ud->tchan_cnt),
1695 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1696 ud->rchan_cnt));
1697 break;
1698 default:
1699 break;
1700 }
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1701
1702 return ch_count;
1703}
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1704
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]1705static int udma_push_to_ring(struct k3_nav_ring *ring, void *elem)
1706{
1707 u64 addr = 0;
1708
1709 memcpy(&addr, &elem, sizeof(elem));
1710 return k3_nav_ringacc_ring_push(ring, &addr);
1711}
1712
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1713static int *udma_prep_dma_memcpy(struct udma_chan *uc, dma_addr_t dest,
1714 dma_addr_t src, size_t len)
1715{
1716 u32 tc_ring_id = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1717 struct cppi5_tr_type15_t *tr_req;
1718 int num_tr;
1719 size_t tr_size = sizeof(struct cppi5_tr_type15_t);
1720 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
1721 unsigned long dummy;
1722 void *tr_desc;
1723 size_t desc_size;
1724
1725 if (len < SZ_64K) {
1726 num_tr = 1;
1727 tr0_cnt0 = len;
1728 tr0_cnt1 = 1;
1729 } else {
1730 unsigned long align_to = __ffs(src | dest);
1731
1732 if (align_to > 3)
1733 align_to = 3;
1734 /*
1735 * Keep simple: tr0: SZ_64K-alignment blocks,
1736 * tr1: the remaining
1737 */
1738 num_tr = 2;
1739 tr0_cnt0 = (SZ_64K - BIT(align_to));
1740 if (len / tr0_cnt0 >= SZ_64K) {
1741 dev_err(uc->ud->dev, "size %zu is not supported\n",
1742 len);
1743 return NULL;
1744 }
1745
1746 tr0_cnt1 = len / tr0_cnt0;
1747 tr1_cnt0 = len % tr0_cnt0;
1748 }
1749
1750 desc_size = cppi5_trdesc_calc_size(num_tr, tr_size);
1751 tr_desc = dma_alloc_coherent(desc_size, &dummy);
1752 if (!tr_desc)
1753 return NULL;
1754 memset(tr_desc, 0, desc_size);
1755
1756 cppi5_trdesc_init(tr_desc, num_tr, tr_size, 0, 0);
1757 cppi5_desc_set_pktids(tr_desc, uc->id, 0x3fff);
1758 cppi5_desc_set_retpolicy(tr_desc, 0, tc_ring_id);
1759
1760 tr_req = tr_desc + tr_size;
1761
1762 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
1763 CPPI5_TR_EVENT_SIZE_COMPLETION, 1);
1764 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT);
1765
1766 tr_req[0].addr = src;
1767 tr_req[0].icnt0 = tr0_cnt0;
1768 tr_req[0].icnt1 = tr0_cnt1;
1769 tr_req[0].icnt2 = 1;
1770 tr_req[0].icnt3 = 1;
1771 tr_req[0].dim1 = tr0_cnt0;
1772
1773 tr_req[0].daddr = dest;
1774 tr_req[0].dicnt0 = tr0_cnt0;
1775 tr_req[0].dicnt1 = tr0_cnt1;
1776 tr_req[0].dicnt2 = 1;
1777 tr_req[0].dicnt3 = 1;
1778 tr_req[0].ddim1 = tr0_cnt0;
1779
1780 if (num_tr == 2) {
1781 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
1782 CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
1783 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT);
1784
1785 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
1786 tr_req[1].icnt0 = tr1_cnt0;
1787 tr_req[1].icnt1 = 1;
1788 tr_req[1].icnt2 = 1;
1789 tr_req[1].icnt3 = 1;
1790
1791 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
1792 tr_req[1].dicnt0 = tr1_cnt0;
1793 tr_req[1].dicnt1 = 1;
1794 tr_req[1].dicnt2 = 1;
1795 tr_req[1].dicnt3 = 1;
1796 }
1797
1798 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, CPPI5_TR_CSF_EOP);
1799
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]1800 flush_dcache_range((unsigned long)tr_desc,
1801 ALIGN((unsigned long)tr_desc + desc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]1802 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1803
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]1804 udma_push_to_ring(uc->tchan->t_ring, tr_desc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1805
1806 return 0;
1807}
1808
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1809#define TISCI_BCDMA_BCHAN_VALID_PARAMS ( \
1810 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1811 TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID)
1812
1813#define TISCI_BCDMA_TCHAN_VALID_PARAMS ( \
1814 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1815 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID)
1816
1817#define TISCI_BCDMA_RCHAN_VALID_PARAMS ( \
1818 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID)
1819
1820#define TISCI_UDMA_TCHAN_VALID_PARAMS ( \
1821 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1822 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \
1823 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \
1824 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
1825 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \
1826 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
1827 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
1828 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1829
1830#define TISCI_UDMA_RCHAN_VALID_PARAMS ( \
1831 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1832 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
1833 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
1834 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
1835 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \
1836 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \
1837 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \
1838 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \
1839 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1840
1841static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc)
1842{
1843 struct udma_dev *ud = uc->ud;
1844 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1845 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1846 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1847 struct udma_bchan *bchan = uc->bchan;
1848 int ret = 0;
1849
1850 req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS;
1851 req_tx.nav_id = tisci_rm->tisci_dev_id;
1852 req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN;
1853 req_tx.index = bchan->id;
1854
1855 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1856 if (ret)
1857 dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret);
1858
1859 return ret;
1860}
1861
1862static struct udma_bchan *__bcdma_reserve_bchan(struct udma_dev *ud, int id)
1863{
1864 if (id >= 0) {
1865 if (test_bit(id, ud->bchan_map)) {
1866 dev_err(ud->dev, "bchan%d is in use\n", id);
1867 return ERR_PTR(-ENOENT);
1868 }
1869 } else {
1870 id = find_next_zero_bit(ud->bchan_map, ud->bchan_cnt, 0);
1871 if (id == ud->bchan_cnt)
1872 return ERR_PTR(-ENOENT);
1873 }
1874 __set_bit(id, ud->bchan_map);
1875 return &ud->bchans[id];
1876}
1877
1878static int bcdma_get_bchan(struct udma_chan *uc)
1879{
1880 struct udma_dev *ud = uc->ud;
1881
1882 if (uc->bchan) {
1883 dev_err(ud->dev, "chan%d: already have bchan%d allocated\n",
1884 uc->id, uc->bchan->id);
1885 return 0;
1886 }
1887
1888 uc->bchan = __bcdma_reserve_bchan(ud, -1);
1889 if (IS_ERR(uc->bchan))
1890 return PTR_ERR(uc->bchan);
1891
1892 uc->tchan = uc->bchan;
1893
1894 return 0;
1895}
1896
1897static void bcdma_put_bchan(struct udma_chan *uc)
1898{
1899 struct udma_dev *ud = uc->ud;
1900
1901 if (uc->bchan) {
1902 dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id,
1903 uc->bchan->id);
1904 __clear_bit(uc->bchan->id, ud->bchan_map);
1905 uc->bchan = NULL;
1906 uc->tchan = NULL;
1907 }
1908}
1909
1910static void bcdma_free_bchan_resources(struct udma_chan *uc)
1911{
1912 if (!uc->bchan)
1913 return;
1914
1915 k3_nav_ringacc_ring_free(uc->bchan->tc_ring);
1916 k3_nav_ringacc_ring_free(uc->bchan->t_ring);
1917 uc->bchan->tc_ring = NULL;
1918 uc->bchan->t_ring = NULL;
1919
1920 bcdma_put_bchan(uc);
1921}
1922
1923static int bcdma_alloc_bchan_resources(struct udma_chan *uc)
1924{
1925 struct k3_nav_ring_cfg ring_cfg;
1926 struct udma_dev *ud = uc->ud;
1927 int ret;
1928
1929 ret = bcdma_get_bchan(uc);
1930 if (ret)
1931 return ret;
1932
1933 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1,
1934 &uc->bchan->t_ring,
1935 &uc->bchan->tc_ring);
1936 if (ret) {
1937 ret = -EBUSY;
1938 goto err_ring;
1939 }
1940
1941 memset(&ring_cfg, 0, sizeof(ring_cfg));
1942 ring_cfg.size = 16;
1943 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
1944 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
1945
1946 ret = k3_nav_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg);
1947 if (ret)
1948 goto err_ringcfg;
1949
1950 return 0;
1951
1952err_ringcfg:
1953 k3_nav_ringacc_ring_free(uc->bchan->tc_ring);
1954 uc->bchan->tc_ring = NULL;
1955 k3_nav_ringacc_ring_free(uc->bchan->t_ring);
1956 uc->bchan->t_ring = NULL;
1957err_ring:
1958 bcdma_put_bchan(uc);
1959
1960 return ret;
1961}
1962
1963static int bcdma_tisci_tx_channel_config(struct udma_chan *uc)
1964{
1965 struct udma_dev *ud = uc->ud;
1966 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1967 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1968 struct udma_tchan *tchan = uc->tchan;
1969 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1970 int ret = 0;
1971
1972 req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS;
1973 req_tx.nav_id = tisci_rm->tisci_dev_id;
1974 req_tx.index = tchan->id;
1975 req_tx.tx_supr_tdpkt = uc->config.notdpkt;
1976 if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
1977 ud->match_data->flags & UDMA_FLAG_TDTYPE) {
1978 /* wait for peer to complete the teardown for PDMAs */
1979 req_tx.valid_params |=
1980 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
1981 req_tx.tx_tdtype = 1;
1982 }
1983
1984 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1985 if (ret)
1986 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1987
Kishon Vijay Abraham I8878dad2024-08-26 15:55:10 +0530[diff] [blame]1988 if (IS_ENABLED(CONFIG_K3_DM_FW))
1989 udma_alloc_tchan_raw(uc);
1990
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1991 return ret;
1992}
1993
1994#define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config
1995
1996static int pktdma_tisci_rx_channel_config(struct udma_chan *uc)
1997{
1998 struct udma_dev *ud = uc->ud;
1999 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2000 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2001 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2002 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
2003 int ret = 0;
2004
2005 req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2006 req_rx.nav_id = tisci_rm->tisci_dev_id;
2007 req_rx.index = uc->rchan->id;
2008
2009 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2010 if (ret) {
2011 dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret);
2012 return ret;
2013 }
2014
2015 flow_req.valid_params =
2016 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2017 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2018 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID;
2019
2020 flow_req.nav_id = tisci_rm->tisci_dev_id;
2021 flow_req.flow_index = uc->rflow->id;
2022
2023 if (uc->config.needs_epib)
2024 flow_req.rx_einfo_present = 1;
2025 else
2026 flow_req.rx_einfo_present = 0;
2027 if (uc->config.psd_size)
2028 flow_req.rx_psinfo_present = 1;
2029 else
2030 flow_req.rx_psinfo_present = 0;
Vignesh Raghavendra87fa0d62023-03-08 09:42:57 +0530[diff] [blame]2031 flow_req.rx_error_handling = 0;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2032
2033 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2034
2035 if (ret)
2036 dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id,
2037 ret);
2038
Kishon Vijay Abraham I8878dad2024-08-26 15:55:10 +0530[diff] [blame]2039 if (IS_ENABLED(CONFIG_K3_DM_FW))
2040 udma_alloc_rchan_raw(uc);
2041
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2042 return ret;
2043}
2044
2045static int bcdma_alloc_chan_resources(struct udma_chan *uc)
2046{
2047 int ret;
2048
2049 uc->config.pkt_mode = false;
2050
2051 switch (uc->config.dir) {
2052 case DMA_MEM_TO_MEM:
2053 /* Non synchronized - mem to mem type of transfer */
2054 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2055 uc->id);
2056
2057 ret = bcdma_alloc_bchan_resources(uc);
2058 if (ret)
2059 return ret;
2060
2061 ret = bcdma_tisci_m2m_channel_config(uc);
2062 break;
2063 default:
2064 /* Can not happen */
2065 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2066 __func__, uc->id, uc->config.dir);
2067 return -EINVAL;
2068 }
2069
2070 /* check if the channel configuration was successful */
2071 if (ret)
2072 goto err_res_free;
2073
2074 if (udma_is_chan_running(uc)) {
2075 dev_warn(uc->ud->dev, "chan%d: is running!\n", uc->id);
2076 udma_stop(uc);
2077 if (udma_is_chan_running(uc)) {
2078 dev_err(uc->ud->dev, "chan%d: won't stop!\n", uc->id);
2079 goto err_res_free;
2080 }
2081 }
2082
2083 udma_reset_rings(uc);
2084
2085 return 0;
2086
2087err_res_free:
2088 bcdma_free_bchan_resources(uc);
2089 udma_free_tx_resources(uc);
2090 udma_free_rx_resources(uc);
2091
2092 udma_reset_uchan(uc);
2093
2094 return ret;
2095}
2096
2097static int pktdma_alloc_chan_resources(struct udma_chan *uc)
2098{
2099 struct udma_dev *ud = uc->ud;
2100 int ret;
2101
2102 switch (uc->config.dir) {
2103 case DMA_MEM_TO_DEV:
2104 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2105 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2106 uc->id);
2107
2108 ret = udma_alloc_tx_resources(uc);
2109 if (ret) {
2110 uc->config.remote_thread_id = -1;
2111 return ret;
2112 }
2113
2114 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2115 uc->config.dst_thread = uc->config.remote_thread_id;
2116 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2117
2118 ret = pktdma_tisci_tx_channel_config(uc);
2119 break;
2120 case DMA_DEV_TO_MEM:
2121 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2122 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2123 uc->id);
2124
2125 ret = udma_alloc_rx_resources(uc);
2126 if (ret) {
2127 uc->config.remote_thread_id = -1;
2128 return ret;
2129 }
2130
2131 uc->config.src_thread = uc->config.remote_thread_id;
2132 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2133 K3_PSIL_DST_THREAD_ID_OFFSET;
2134
2135 ret = pktdma_tisci_rx_channel_config(uc);
2136 break;
2137 default:
2138 /* Can not happen */
2139 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2140 __func__, uc->id, uc->config.dir);
2141 return -EINVAL;
2142 }
2143
2144 /* check if the channel configuration was successful */
2145 if (ret)
2146 goto err_res_free;
2147
2148 /* PSI-L pairing */
2149 ret = udma_navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2150 if (ret) {
2151 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2152 uc->config.src_thread, uc->config.dst_thread);
2153 goto err_res_free;
2154 }
2155
2156 if (udma_is_chan_running(uc)) {
2157 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2158 udma_stop(uc);
2159 if (udma_is_chan_running(uc)) {
2160 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2161 goto err_res_free;
2162 }
2163 }
2164
2165 udma_reset_rings(uc);
2166
2167 if (uc->tchan)
2168 dev_dbg(ud->dev,
2169 "chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n",
2170 uc->id, uc->tchan->id, uc->tchan->tflow_id,
2171 uc->config.remote_thread_id);
2172 else if (uc->rchan)
2173 dev_dbg(ud->dev,
2174 "chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n",
2175 uc->id, uc->rchan->id, uc->rflow->id,
2176 uc->config.remote_thread_id);
2177 return 0;
2178
2179err_res_free:
2180 udma_free_tx_resources(uc);
2181 udma_free_rx_resources(uc);
2182
2183 udma_reset_uchan(uc);
2184
2185 return ret;
2186}
2187
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2188static int udma_transfer(struct udevice *dev, int direction,
Andrew Davisd2da2842022-10-07 12:11:13 -0500[diff] [blame]2189 dma_addr_t dst, dma_addr_t src, size_t len)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2190{
2191 struct udma_dev *ud = dev_get_priv(dev);
2192 /* Channel0 is reserved for memcpy */
2193 struct udma_chan *uc = &ud->channels[0];
2194 dma_addr_t paddr = 0;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2195
Andrew Davisd2da2842022-10-07 12:11:13 -0500[diff] [blame]2196 udma_prep_dma_memcpy(uc, dst, src, len);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2197 udma_start(uc);
2198 udma_poll_completion(uc, &paddr);
2199 udma_stop(uc);
2200
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2201 return 0;
2202}
2203
2204static int udma_request(struct dma *dma)
2205{
2206 struct udma_dev *ud = dev_get_priv(dma->dev);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2207 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2208 struct udma_chan *uc;
2209 unsigned long dummy;
2210 int ret;
2211
2212 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2213 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2214 return -EINVAL;
2215 }
2216
2217 uc = &ud->channels[dma->id];
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2218 ucc = &uc->config;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2219 switch (ud->match_data->type) {
2220 case DMA_TYPE_UDMA:
2221 ret = udma_alloc_chan_resources(uc);
2222 break;
2223 case DMA_TYPE_BCDMA:
2224 ret = bcdma_alloc_chan_resources(uc);
2225 break;
2226 case DMA_TYPE_PKTDMA:
2227 ret = pktdma_alloc_chan_resources(uc);
2228 break;
2229 default:
2230 return -EINVAL;
2231 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2232 if (ret) {
2233 dev_err(dma->dev, "alloc dma res failed %d\n", ret);
2234 return -EINVAL;
2235 }
2236
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2237 if (uc->config.dir == DMA_MEM_TO_DEV) {
2238 uc->desc_tx = dma_alloc_coherent(ucc->hdesc_size, &dummy);
2239 memset(uc->desc_tx, 0, ucc->hdesc_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2240 } else {
2241 uc->desc_rx = dma_alloc_coherent(
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2242 ucc->hdesc_size * UDMA_RX_DESC_NUM, &dummy);
2243 memset(uc->desc_rx, 0, ucc->hdesc_size * UDMA_RX_DESC_NUM);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2244 }
2245
2246 uc->in_use = true;
2247 uc->desc_rx_cur = 0;
2248 uc->num_rx_bufs = 0;
2249
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2250 if (uc->config.dir == DMA_DEV_TO_MEM) {
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2251 uc->cfg_data.flow_id_base = uc->rflow->id;
2252 uc->cfg_data.flow_id_cnt = 1;
2253 }
2254
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2255 return 0;
2256}
2257
Simon Glass75c0ad62020-02-03 07:35:55 -0700[diff] [blame]2258static int udma_rfree(struct dma *dma)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2259{
2260 struct udma_dev *ud = dev_get_priv(dma->dev);
2261 struct udma_chan *uc;
2262
2263 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2264 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2265 return -EINVAL;
2266 }
2267 uc = &ud->channels[dma->id];
2268
2269 if (udma_is_chan_running(uc))
2270 udma_stop(uc);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2271
2272 udma_navss_psil_unpair(ud, uc->config.src_thread,
2273 uc->config.dst_thread);
2274
2275 bcdma_free_bchan_resources(uc);
2276 udma_free_tx_resources(uc);
2277 udma_free_rx_resources(uc);
2278 udma_reset_uchan(uc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2279
2280 uc->in_use = false;
2281
2282 return 0;
2283}
2284
2285static int udma_enable(struct dma *dma)
2286{
2287 struct udma_dev *ud = dev_get_priv(dma->dev);
2288 struct udma_chan *uc;
2289 int ret;
2290
2291 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2292 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2293 return -EINVAL;
2294 }
2295 uc = &ud->channels[dma->id];
2296
2297 ret = udma_start(uc);
2298
2299 return ret;
2300}
2301
2302static int udma_disable(struct dma *dma)
2303{
2304 struct udma_dev *ud = dev_get_priv(dma->dev);
2305 struct udma_chan *uc;
2306 int ret = 0;
2307
2308 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2309 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2310 return -EINVAL;
2311 }
2312 uc = &ud->channels[dma->id];
2313
2314 if (udma_is_chan_running(uc))
2315 ret = udma_stop(uc);
2316 else
2317 dev_err(dma->dev, "%s not running\n", __func__);
2318
2319 return ret;
2320}
2321
2322static int udma_send(struct dma *dma, void *src, size_t len, void *metadata)
2323{
2324 struct udma_dev *ud = dev_get_priv(dma->dev);
2325 struct cppi5_host_desc_t *desc_tx;
2326 dma_addr_t dma_src = (dma_addr_t)src;
2327 struct ti_udma_drv_packet_data packet_data = { 0 };
2328 dma_addr_t paddr;
2329 struct udma_chan *uc;
2330 u32 tc_ring_id;
2331 int ret;
2332
Keerthya3c8bb12019-04-24 16:33:54 +0530[diff] [blame]2333 if (metadata)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2334 packet_data = *((struct ti_udma_drv_packet_data *)metadata);
2335
2336 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2337 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2338 return -EINVAL;
2339 }
2340 uc = &ud->channels[dma->id];
2341
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2342 if (uc->config.dir != DMA_MEM_TO_DEV)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2343 return -EINVAL;
2344
2345 tc_ring_id = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
2346
2347 desc_tx = uc->desc_tx;
2348
2349 cppi5_hdesc_reset_hbdesc(desc_tx);
2350
2351 cppi5_hdesc_init(desc_tx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2352 uc->config.needs_epib ? CPPI5_INFO0_HDESC_EPIB_PRESENT : 0,
2353 uc->config.psd_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2354 cppi5_hdesc_set_pktlen(desc_tx, len);
2355 cppi5_hdesc_attach_buf(desc_tx, dma_src, len, dma_src, len);
2356 cppi5_desc_set_pktids(&desc_tx->hdr, uc->id, 0x3fff);
2357 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, tc_ring_id);
2358 /* pass below information from caller */
2359 cppi5_hdesc_set_pkttype(desc_tx, packet_data.pkt_type);
2360 cppi5_desc_set_tags_ids(&desc_tx->hdr, 0, packet_data.dest_tag);
2361
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2362 flush_dcache_range((unsigned long)dma_src,
2363 ALIGN((unsigned long)dma_src + len,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2364 ARCH_DMA_MINALIGN));
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2365 flush_dcache_range((unsigned long)desc_tx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2366 ALIGN((unsigned long)desc_tx + uc->config.hdesc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2367 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2368
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]2369 ret = udma_push_to_ring(uc->tchan->t_ring, uc->desc_tx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2370 if (ret) {
2371 dev_err(dma->dev, "TX dma push fail ch_id %lu %d\n",
2372 dma->id, ret);
2373 return ret;
2374 }
2375
2376 udma_poll_completion(uc, &paddr);
2377
2378 return 0;
2379}
2380
2381static int udma_receive(struct dma *dma, void **dst, void *metadata)
2382{
2383 struct udma_dev *ud = dev_get_priv(dma->dev);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2384 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2385 struct cppi5_host_desc_t *desc_rx;
2386 dma_addr_t buf_dma;
2387 struct udma_chan *uc;
2388 u32 buf_dma_len, pkt_len;
2389 u32 port_id = 0;
2390 int ret;
2391
2392 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2393 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2394 return -EINVAL;
2395 }
2396 uc = &ud->channels[dma->id];
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2397 ucc = &uc->config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2398
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2399 if (uc->config.dir != DMA_DEV_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2400 return -EINVAL;
2401 if (!uc->num_rx_bufs)
2402 return -EINVAL;
2403
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]2404 ret = k3_nav_ringacc_ring_pop(uc->rflow->r_ring, &desc_rx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2405 if (ret && ret != -ENODATA) {
2406 dev_err(dma->dev, "rx dma fail ch_id:%lu %d\n", dma->id, ret);
2407 return ret;
2408 } else if (ret == -ENODATA) {
2409 return 0;
2410 }
2411
2412 /* invalidate cache data */
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2413 invalidate_dcache_range((ulong)desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2414 (ulong)(desc_rx + ucc->hdesc_size));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2415
2416 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
2417 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
2418
2419 /* invalidate cache data */
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2420 invalidate_dcache_range((ulong)buf_dma,
2421 (ulong)(buf_dma + buf_dma_len));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2422
2423 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
2424
2425 *dst = (void *)buf_dma;
2426 uc->num_rx_bufs--;
2427
2428 return pkt_len;
2429}
2430
2431static int udma_of_xlate(struct dma *dma, struct ofnode_phandle_args *args)
2432{
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2433 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2434 struct udma_dev *ud = dev_get_priv(dma->dev);
2435 struct udma_chan *uc = &ud->channels[0];
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2436 struct psil_endpoint_config *ep_config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2437 u32 val;
2438
2439 for (val = 0; val < ud->ch_count; val++) {
2440 uc = &ud->channels[val];
2441 if (!uc->in_use)
2442 break;
2443 }
2444
2445 if (val == ud->ch_count)
2446 return -EBUSY;
2447
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2448 ucc = &uc->config;
2449 ucc->remote_thread_id = args->args[0];
2450 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)
2451 ucc->dir = DMA_MEM_TO_DEV;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2452 else
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2453 ucc->dir = DMA_DEV_TO_MEM;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2454
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2455 ep_config = psil_get_ep_config(ucc->remote_thread_id);
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2456 if (IS_ERR(ep_config)) {
2457 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2458 uc->config.remote_thread_id);
2459 ucc->dir = DMA_MEM_TO_MEM;
2460 ucc->remote_thread_id = -1;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2461 return false;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2462 }
2463
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2464 ucc->pkt_mode = ep_config->pkt_mode;
2465 ucc->channel_tpl = ep_config->channel_tpl;
2466 ucc->notdpkt = ep_config->notdpkt;
2467 ucc->ep_type = ep_config->ep_type;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2468
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2469 if (ud->match_data->type == DMA_TYPE_PKTDMA &&
2470 ep_config->mapped_channel_id >= 0) {
2471 ucc->mapped_channel_id = ep_config->mapped_channel_id;
2472 ucc->default_flow_id = ep_config->default_flow_id;
2473 } else {
2474 ucc->mapped_channel_id = -1;
2475 ucc->default_flow_id = -1;
2476 }
2477
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2478 ucc->needs_epib = ep_config->needs_epib;
2479 ucc->psd_size = ep_config->psd_size;
2480 ucc->metadata_size = (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + ucc->psd_size;
2481
2482 ucc->hdesc_size = cppi5_hdesc_calc_size(ucc->needs_epib,
2483 ucc->psd_size, 0);
2484 ucc->hdesc_size = ALIGN(ucc->hdesc_size, ARCH_DMA_MINALIGN);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2485
2486 dma->id = uc->id;
2487 pr_debug("Allocated dma chn:%lu epib:%d psdata:%u meta:%u thread_id:%x\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2488 dma->id, ucc->needs_epib,
2489 ucc->psd_size, ucc->metadata_size,
2490 ucc->remote_thread_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2491
2492 return 0;
2493}
2494
2495int udma_prepare_rcv_buf(struct dma *dma, void *dst, size_t size)
2496{
2497 struct udma_dev *ud = dev_get_priv(dma->dev);
2498 struct cppi5_host_desc_t *desc_rx;
2499 dma_addr_t dma_dst;
2500 struct udma_chan *uc;
2501 u32 desc_num;
2502
2503 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2504 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2505 return -EINVAL;
2506 }
2507 uc = &ud->channels[dma->id];
2508
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2509 if (uc->config.dir != DMA_DEV_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2510 return -EINVAL;
2511
2512 if (uc->num_rx_bufs >= UDMA_RX_DESC_NUM)
2513 return -EINVAL;
2514
2515 desc_num = uc->desc_rx_cur % UDMA_RX_DESC_NUM;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2516 desc_rx = uc->desc_rx + (desc_num * uc->config.hdesc_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2517 dma_dst = (dma_addr_t)dst;
2518
2519 cppi5_hdesc_reset_hbdesc(desc_rx);
2520
2521 cppi5_hdesc_init(desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2522 uc->config.needs_epib ? CPPI5_INFO0_HDESC_EPIB_PRESENT : 0,
2523 uc->config.psd_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2524 cppi5_hdesc_set_pktlen(desc_rx, size);
2525 cppi5_hdesc_attach_buf(desc_rx, dma_dst, size, dma_dst, size);
2526
Matthias Schiffer65aef702024-04-26 10:02:28 +0200[diff] [blame]2527 invalidate_dcache_range((unsigned long)dma_dst,
2528 (unsigned long)(dma_dst + size));
2529
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2530 flush_dcache_range((unsigned long)desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2531 ALIGN((unsigned long)desc_rx + uc->config.hdesc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2532 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2533
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]2534 udma_push_to_ring(uc->rflow->fd_ring, desc_rx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2535
2536 uc->num_rx_bufs++;
2537 uc->desc_rx_cur++;
2538
2539 return 0;
2540}
2541
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2542static int udma_get_cfg(struct dma *dma, u32 id, void **data)
2543{
2544 struct udma_dev *ud = dev_get_priv(dma->dev);
2545 struct udma_chan *uc;
2546
2547 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2548 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2549 return -EINVAL;
2550 }
2551
2552 switch (id) {
2553 case TI_UDMA_CHAN_PRIV_INFO:
2554 uc = &ud->channels[dma->id];
2555 *data = &uc->cfg_data;
2556 return 0;
2557 }
2558
2559 return -EINVAL;
2560}
2561
Santhosh Kumar K976edc62024-10-09 20:27:02 +0530[diff] [blame]2562static int udma_probe(struct udevice *dev)
2563{
2564 struct dma_dev_priv *uc_priv = dev_get_uclass_priv(dev);
2565 struct udma_dev *ud = dev_get_priv(dev);
2566 int i, ret;
2567 struct udevice *tmp;
2568 struct udevice *tisci_dev = NULL;
2569 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Santhosh Kumar K16e3d042024-10-09 20:27:03 +0530[diff] [blame]2570 struct udma_chan *uc;
Santhosh Kumar K976edc62024-10-09 20:27:02 +0530[diff] [blame]2571 ofnode navss_ofnode = ofnode_get_parent(dev_ofnode(dev));
2572
2573 ud->match_data = (void *)dev_get_driver_data(dev);
2574 ret = udma_get_mmrs(dev);
2575 if (ret)
2576 return ret;
2577
2578 ud->psil_base = ud->match_data->psil_base;
2579
2580 ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, dev,
2581 "ti,sci", &tisci_dev);
2582 if (ret) {
2583 debug("Failed to get TISCI phandle (%d)\n", ret);
2584 tisci_rm->tisci = NULL;
2585 return -EINVAL;
2586 }
2587 tisci_rm->tisci = (struct ti_sci_handle *)
2588 (ti_sci_get_handle_from_sysfw(tisci_dev));
2589
2590 tisci_rm->tisci_dev_id = -1;
2591 ret = dev_read_u32(dev, "ti,sci-dev-id", &tisci_rm->tisci_dev_id);
2592 if (ret) {
2593 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
2594 return ret;
2595 }
2596
2597 tisci_rm->tisci_navss_dev_id = -1;
2598 ret = ofnode_read_u32(navss_ofnode, "ti,sci-dev-id",
2599 &tisci_rm->tisci_navss_dev_id);
2600 if (ret) {
2601 dev_err(dev, "navss sci-dev-id read failure %d\n", ret);
2602 return ret;
2603 }
2604
2605 tisci_rm->tisci_udmap_ops = &tisci_rm->tisci->ops.rm_udmap_ops;
2606 tisci_rm->tisci_psil_ops = &tisci_rm->tisci->ops.rm_psil_ops;
2607
2608 if (ud->match_data->type == DMA_TYPE_UDMA) {
2609 ret = uclass_get_device_by_phandle(UCLASS_MISC, dev,
2610 "ti,ringacc", &tmp);
2611 ud->ringacc = dev_get_priv(tmp);
2612 } else {
2613 struct k3_ringacc_init_data ring_init_data;
2614
2615 ring_init_data.tisci = ud->tisci_rm.tisci;
2616 ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id;
2617 if (ud->match_data->type == DMA_TYPE_BCDMA) {
2618 ring_init_data.num_rings = ud->bchan_cnt +
2619 ud->tchan_cnt +
2620 ud->rchan_cnt;
2621 } else {
2622 ring_init_data.num_rings = ud->rflow_cnt +
2623 ud->tflow_cnt;
2624 }
2625
2626 ud->ringacc = k3_ringacc_dmarings_init(dev, &ring_init_data);
2627 }
2628 if (IS_ERR(ud->ringacc))
2629 return PTR_ERR(ud->ringacc);
2630
2631 ud->dev = dev;
2632 ret = setup_resources(ud);
2633 if (ret < 0)
2634 return ret;
2635
2636 ud->ch_count = ret;
2637
2638 for (i = 0; i < ud->bchan_cnt; i++) {
2639 struct udma_bchan *bchan = &ud->bchans[i];
2640
2641 bchan->id = i;
2642 bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000;
2643 }
2644
2645 for (i = 0; i < ud->tchan_cnt; i++) {
2646 struct udma_tchan *tchan = &ud->tchans[i];
2647
2648 tchan->id = i;
2649 tchan->reg_chan = ud->mmrs[MMR_TCHAN] + UDMA_CH_100(i);
2650 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + UDMA_CH_1000(i);
2651 }
2652
2653 for (i = 0; i < ud->rchan_cnt; i++) {
2654 struct udma_rchan *rchan = &ud->rchans[i];
2655
2656 rchan->id = i;
2657 rchan->reg_chan = ud->mmrs[MMR_RCHAN] + UDMA_CH_100(i);
2658 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + UDMA_CH_1000(i);
2659 }
2660
2661 for (i = 0; i < ud->rflow_cnt; i++) {
2662 struct udma_rflow *rflow = &ud->rflows[i];
2663
2664 rflow->id = i;
2665 rflow->reg_rflow = ud->mmrs[MMR_RFLOW] + UDMA_CH_40(i);
2666 }
2667
2668 for (i = 0; i < ud->ch_count; i++) {
2669 struct udma_chan *uc = &ud->channels[i];
2670
2671 uc->ud = ud;
2672 uc->id = i;
2673 uc->config.remote_thread_id = -1;
2674 uc->bchan = NULL;
2675 uc->tchan = NULL;
2676 uc->rchan = NULL;
2677 uc->config.mapped_channel_id = -1;
2678 uc->config.default_flow_id = -1;
2679 uc->config.dir = DMA_MEM_TO_MEM;
2680 sprintf(uc->name, "UDMA chan%d\n", i);
2681 if (!i)
2682 uc->in_use = true;
2683 }
2684
2685 pr_debug("%s(rev: 0x%08x) CAP0-3: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
2686 dev->name,
2687 udma_read(ud->mmrs[MMR_GCFG], 0),
2688 udma_read(ud->mmrs[MMR_GCFG], 0x20),
2689 udma_read(ud->mmrs[MMR_GCFG], 0x24),
2690 udma_read(ud->mmrs[MMR_GCFG], 0x28),
2691 udma_read(ud->mmrs[MMR_GCFG], 0x2c));
2692
2693 uc_priv->supported = DMA_SUPPORTS_MEM_TO_MEM | DMA_SUPPORTS_MEM_TO_DEV;
2694
Santhosh Kumar K16e3d042024-10-09 20:27:03 +0530[diff] [blame]2695 uc = &ud->channels[0];
2696 ret = 0;
2697 switch (ud->match_data->type) {
2698 case DMA_TYPE_UDMA:
2699 ret = udma_alloc_chan_resources(uc);
2700 break;
2701 case DMA_TYPE_BCDMA:
2702 ret = bcdma_alloc_chan_resources(uc);
2703 break;
2704 default:
2705 break; /* Do nothing in any other case */
2706 };
2707
2708 if (ret)
2709 dev_err(dev, " Channel 0 allocation failure %d\n", ret);
2710
2711 return ret;
2712}
2713
2714static int udma_remove(struct udevice *dev)
2715{
2716 struct udma_dev *ud = dev_get_priv(dev);
2717 struct udma_chan *uc = &ud->channels[0];
2718
2719 switch (ud->match_data->type) {
2720 case DMA_TYPE_UDMA:
2721 udma_free_chan_resources(uc);
2722 break;
2723 case DMA_TYPE_BCDMA:
2724 bcdma_free_bchan_resources(uc);
2725 break;
2726 default:
2727 break;
2728 };
2729
Santhosh Kumar K976edc62024-10-09 20:27:02 +0530[diff] [blame]2730 return 0;
2731}
2732
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2733static const struct dma_ops udma_ops = {
2734 .transfer = udma_transfer,
2735 .of_xlate = udma_of_xlate,
2736 .request = udma_request,
Simon Glass75c0ad62020-02-03 07:35:55 -0700[diff] [blame]2737 .rfree = udma_rfree,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2738 .enable = udma_enable,
2739 .disable = udma_disable,
2740 .send = udma_send,
2741 .receive = udma_receive,
2742 .prepare_rcv_buf = udma_prepare_rcv_buf,
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2743 .get_cfg = udma_get_cfg,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2744};
2745
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2746static struct udma_match_data am654_main_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2747 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2748 .psil_base = 0x1000,
2749 .enable_memcpy_support = true,
2750 .statictr_z_mask = GENMASK(11, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2751 .oes = {
2752 .udma_rchan = 0x200,
2753 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2754 .tpl_levels = 2,
2755 .level_start_idx = {
2756 [0] = 8, /* Normal channels */
2757 [1] = 0, /* High Throughput channels */
2758 },
2759};
2760
2761static struct udma_match_data am654_mcu_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2762 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2763 .psil_base = 0x6000,
2764 .enable_memcpy_support = true,
2765 .statictr_z_mask = GENMASK(11, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2766 .oes = {
2767 .udma_rchan = 0x200,
2768 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2769 .tpl_levels = 2,
2770 .level_start_idx = {
2771 [0] = 2, /* Normal channels */
2772 [1] = 0, /* High Throughput channels */
2773 },
2774};
2775
2776static struct udma_match_data j721e_main_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2777 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2778 .psil_base = 0x1000,
2779 .enable_memcpy_support = true,
2780 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2781 .statictr_z_mask = GENMASK(23, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2782 .oes = {
2783 .udma_rchan = 0x400,
2784 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2785 .tpl_levels = 3,
2786 .level_start_idx = {
2787 [0] = 16, /* Normal channels */
2788 [1] = 4, /* High Throughput channels */
2789 [2] = 0, /* Ultra High Throughput channels */
2790 },
2791};
2792
2793static struct udma_match_data j721e_mcu_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2794 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2795 .psil_base = 0x6000,
2796 .enable_memcpy_support = true,
2797 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2798 .statictr_z_mask = GENMASK(23, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2799 .oes = {
2800 .udma_rchan = 0x400,
2801 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2802 .tpl_levels = 2,
2803 .level_start_idx = {
2804 [0] = 2, /* Normal channels */
2805 [1] = 0, /* High Throughput channels */
2806 },
2807};
2808
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2809static struct udma_match_data am64_bcdma_data = {
2810 .type = DMA_TYPE_BCDMA,
2811 .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */
2812 .enable_memcpy_support = true, /* Supported via bchan */
2813 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2814 .statictr_z_mask = GENMASK(23, 0),
2815 .oes = {
2816 .bcdma_bchan_data = 0x2200,
2817 .bcdma_bchan_ring = 0x2400,
2818 .bcdma_tchan_data = 0x2800,
2819 .bcdma_tchan_ring = 0x2a00,
2820 .bcdma_rchan_data = 0x2e00,
2821 .bcdma_rchan_ring = 0x3000,
2822 },
2823 /* No throughput levels */
2824};
2825
2826static struct udma_match_data am64_pktdma_data = {
2827 .type = DMA_TYPE_PKTDMA,
2828 .psil_base = 0x1000,
2829 .enable_memcpy_support = false,
2830 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2831 .statictr_z_mask = GENMASK(23, 0),
2832 .oes = {
2833 .pktdma_tchan_flow = 0x1200,
2834 .pktdma_rchan_flow = 0x1600,
2835 },
2836 /* No throughput levels */
2837};
2838
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2839static const struct udevice_id udma_ids[] = {
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2840 {
2841 .compatible = "ti,am654-navss-main-udmap",
2842 .data = (ulong)&am654_main_data,
2843 },
2844 {
2845 .compatible = "ti,am654-navss-mcu-udmap",
2846 .data = (ulong)&am654_mcu_data,
2847 }, {
2848 .compatible = "ti,j721e-navss-main-udmap",
2849 .data = (ulong)&j721e_main_data,
2850 }, {
2851 .compatible = "ti,j721e-navss-mcu-udmap",
2852 .data = (ulong)&j721e_mcu_data,
2853 },
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2854 {
2855 .compatible = "ti,am64-dmss-bcdma",
2856 .data = (ulong)&am64_bcdma_data,
2857 },
2858 {
2859 .compatible = "ti,am64-dmss-pktdma",
2860 .data = (ulong)&am64_pktdma_data,
2861 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2862 { /* Sentinel */ },
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2863};
2864
2865U_BOOT_DRIVER(ti_edma3) = {
2866 .name = "ti-udma",
2867 .id = UCLASS_DMA,
2868 .of_match = udma_ids,
2869 .ops = &udma_ops,
2870 .probe = udma_probe,
Santhosh Kumar K16e3d042024-10-09 20:27:03 +0530[diff] [blame]2871 .remove = udma_remove,
Simon Glass8a2b47f2020-12-03 16:55:17 -0700[diff] [blame]2872 .priv_auto = sizeof(struct udma_dev),
Santhosh Kumar K16e3d042024-10-09 20:27:03 +0530[diff] [blame]2873 .flags = DM_FLAG_OS_PREPARE,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2874};