Gitiles
Code Review Sign In
git01.mediatek.com / filogic / uboot / 976edc6fd6c6e701f4972f92798a0560d5d217ea / . / drivers / dma / ti / k3-udma.c
blob: 0543f5f4c827d0ba5960cbae93d87673efe20ce5 [file] [log] [blame]
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1// SPDX-License-Identifier: GPL-2.0+
2/*
Nishanth Menoneaa39c62023-11-01 15:56:03 -0500[diff] [blame]3 * Copyright (C) 2018 Texas Instruments Incorporated - https://www.ti.com
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
5 */
6#define pr_fmt(fmt) "udma: " fmt
7
Simon Glass63334482019-11-14 12:57:39 -0700[diff] [blame]8#include <cpu_func.h>
Simon Glass0f2af882020-05-10 11:40:05 -0600[diff] [blame]9#include <log.h>
Simon Glass274e0b02020-05-10 11:39:56 -0600[diff] [blame]10#include <asm/cache.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]11#include <asm/io.h>
12#include <asm/bitops.h>
13#include <malloc.h>
Matthias Schifferfe2ee882024-04-26 10:02:27 +0200[diff] [blame]14#include <net.h>
Simon Glass4dcacfc2020-05-10 11:40:13 -0600[diff] [blame]15#include <linux/bitops.h>
Masahiro Yamada6373a172020-02-14 16:40:19 +0900[diff] [blame]16#include <linux/dma-mapping.h>
Dhruva Golee6b42392022-09-20 10:56:02 +0530[diff] [blame]17#include <linux/sizes.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]18#include <dm.h>
Simon Glass9bc15642020-02-03 07:36:16 -0700[diff] [blame]19#include <dm/device_compat.h>
Simon Glassd66c5f72020-02-03 07:36:15 -0700[diff] [blame]20#include <dm/devres.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]21#include <dm/read.h>
22#include <dm/of_access.h>
23#include <dma.h>
24#include <dma-uclass.h>
25#include <linux/delay.h>
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]26#include <linux/bitmap.h>
Simon Glassd66c5f72020-02-03 07:36:15 -0700[diff] [blame]27#include <linux/err.h>
Simon Glassbdd5f812023-09-14 18:21:46 -0600[diff] [blame]28#include <linux/printk.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]29#include <linux/soc/ti/k3-navss-ringacc.h>
30#include <linux/soc/ti/cppi5.h>
31#include <linux/soc/ti/ti-udma.h>
32#include <linux/soc/ti/ti_sci_protocol.h>
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]33#include <linux/soc/ti/cppi5.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]34
35#include "k3-udma-hwdef.h"
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]36#include "k3-psil-priv.h"
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]37
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]38#define K3_UDMA_MAX_RFLOWS 1024
39
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]40struct udma_chan;
41
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]42enum k3_dma_type {
43 DMA_TYPE_UDMA = 0,
44 DMA_TYPE_BCDMA,
45 DMA_TYPE_PKTDMA,
46};
47
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]48enum udma_mmr {
49 MMR_GCFG = 0,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]50 MMR_BCHANRT,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]51 MMR_RCHANRT,
52 MMR_TCHANRT,
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]53 MMR_RCHAN,
54 MMR_TCHAN,
55 MMR_RFLOW,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]56 MMR_LAST,
57};
58
59static const char * const mmr_names[] = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]60 [MMR_GCFG] = "gcfg",
61 [MMR_BCHANRT] = "bchanrt",
62 [MMR_RCHANRT] = "rchanrt",
63 [MMR_TCHANRT] = "tchanrt",
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]64 [MMR_RCHAN] = "rchan",
65 [MMR_TCHAN] = "tchan",
66 [MMR_RFLOW] = "rflow",
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]67};
68
69struct udma_tchan {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]70 void __iomem *reg_chan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]71 void __iomem *reg_rt;
72
73 int id;
74 struct k3_nav_ring *t_ring; /* Transmit ring */
75 struct k3_nav_ring *tc_ring; /* Transmit Completion ring */
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]76 int tflow_id; /* applicable only for PKTDMA */
77
78};
79
80#define udma_bchan udma_tchan
81
82struct udma_rflow {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]83 void __iomem *reg_rflow;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]84 int id;
85 struct k3_nav_ring *fd_ring; /* Free Descriptor ring */
86 struct k3_nav_ring *r_ring; /* Receive ring */
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]87};
88
89struct udma_rchan {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]90 void __iomem *reg_chan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]91 void __iomem *reg_rt;
92
93 int id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]94};
95
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]96struct udma_oes_offsets {
97 /* K3 UDMA Output Event Offset */
98 u32 udma_rchan;
99
100 /* BCDMA Output Event Offsets */
101 u32 bcdma_bchan_data;
102 u32 bcdma_bchan_ring;
103 u32 bcdma_tchan_data;
104 u32 bcdma_tchan_ring;
105 u32 bcdma_rchan_data;
106 u32 bcdma_rchan_ring;
107
108 /* PKTDMA Output Event Offsets */
109 u32 pktdma_tchan_flow;
110 u32 pktdma_rchan_flow;
111};
112
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]113#define UDMA_FLAG_PDMA_ACC32 BIT(0)
114#define UDMA_FLAG_PDMA_BURST BIT(1)
115#define UDMA_FLAG_TDTYPE BIT(2)
116
117struct udma_match_data {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]118 enum k3_dma_type type;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]119 u32 psil_base;
120 bool enable_memcpy_support;
121 u32 flags;
122 u32 statictr_z_mask;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]123 struct udma_oes_offsets oes;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]124
125 u8 tpl_levels;
126 u32 level_start_idx[];
127};
128
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]129enum udma_rm_range {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]130 RM_RANGE_BCHAN = 0,
131 RM_RANGE_TCHAN,
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]132 RM_RANGE_RCHAN,
133 RM_RANGE_RFLOW,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]134 RM_RANGE_TFLOW,
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]135 RM_RANGE_LAST,
136};
137
138struct udma_tisci_rm {
139 const struct ti_sci_handle *tisci;
140 const struct ti_sci_rm_udmap_ops *tisci_udmap_ops;
141 u32 tisci_dev_id;
142
143 /* tisci information for PSI-L thread pairing/unpairing */
144 const struct ti_sci_rm_psil_ops *tisci_psil_ops;
145 u32 tisci_navss_dev_id;
146
147 struct ti_sci_resource *rm_ranges[RM_RANGE_LAST];
148};
149
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]150struct udma_dev {
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]151 struct udevice *dev;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]152 void __iomem *mmrs[MMR_LAST];
153
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]154 struct udma_tisci_rm tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]155 struct k3_nav_ringacc *ringacc;
156
157 u32 features;
158
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]159 int bchan_cnt;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]160 int tchan_cnt;
161 int echan_cnt;
162 int rchan_cnt;
163 int rflow_cnt;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]164 int tflow_cnt;
165 unsigned long *bchan_map;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]166 unsigned long *tchan_map;
167 unsigned long *rchan_map;
168 unsigned long *rflow_map;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]169 unsigned long *rflow_map_reserved;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]170 unsigned long *tflow_map;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]171
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]172 struct udma_bchan *bchans;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]173 struct udma_tchan *tchans;
174 struct udma_rchan *rchans;
175 struct udma_rflow *rflows;
176
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]177 struct udma_match_data *match_data;
178
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]179 struct udma_chan *channels;
180 u32 psil_base;
181
182 u32 ch_count;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]183};
184
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]185struct udma_chan_config {
186 u32 psd_size; /* size of Protocol Specific Data */
187 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
188 u32 hdesc_size; /* Size of a packet descriptor in packet mode */
189 int remote_thread_id;
190 u32 atype;
191 u32 src_thread;
192 u32 dst_thread;
193 enum psil_endpoint_type ep_type;
194 enum udma_tp_level channel_tpl; /* Channel Throughput Level */
195
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]196 /* PKTDMA mapped channel */
197 int mapped_channel_id;
198 /* PKTDMA default tflow or rflow for mapped channel */
199 int default_flow_id;
200
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]201 enum dma_direction dir;
202
203 unsigned int pkt_mode:1; /* TR or packet */
204 unsigned int needs_epib:1; /* EPIB is needed for the communication or not */
205 unsigned int enable_acc32:1;
206 unsigned int enable_burst:1;
207 unsigned int notdpkt:1; /* Suppress sending TDC packet */
208};
209
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]210struct udma_chan {
211 struct udma_dev *ud;
212 char name[20];
213
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]214 struct udma_bchan *bchan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]215 struct udma_tchan *tchan;
216 struct udma_rchan *rchan;
217 struct udma_rflow *rflow;
218
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]219 struct ti_udma_drv_chan_cfg_data cfg_data;
220
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]221 u32 bcnt; /* number of bytes completed since the start of the channel */
222
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]223 struct udma_chan_config config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]224
225 u32 id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]226
227 struct cppi5_host_desc_t *desc_tx;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]228 bool in_use;
229 void *desc_rx;
230 u32 num_rx_bufs;
231 u32 desc_rx_cur;
232
233};
234
235#define UDMA_CH_1000(ch) (ch * 0x1000)
236#define UDMA_CH_100(ch) (ch * 0x100)
237#define UDMA_CH_40(ch) (ch * 0x40)
238
239#ifdef PKTBUFSRX
240#define UDMA_RX_DESC_NUM PKTBUFSRX
241#else
242#define UDMA_RX_DESC_NUM 4
243#endif
244
245/* Generic register access functions */
246static inline u32 udma_read(void __iomem *base, int reg)
247{
248 u32 v;
249
250 v = __raw_readl(base + reg);
251 pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, base + reg);
252 return v;
253}
254
255static inline void udma_write(void __iomem *base, int reg, u32 val)
256{
257 pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", val, base + reg);
258 __raw_writel(val, base + reg);
259}
260
261static inline void udma_update_bits(void __iomem *base, int reg,
262 u32 mask, u32 val)
263{
264 u32 tmp, orig;
265
266 orig = udma_read(base, reg);
267 tmp = orig & ~mask;
268 tmp |= (val & mask);
269
270 if (tmp != orig)
271 udma_write(base, reg, tmp);
272}
273
274/* TCHANRT */
275static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg)
276{
277 if (!tchan)
278 return 0;
279 return udma_read(tchan->reg_rt, reg);
280}
281
282static inline void udma_tchanrt_write(struct udma_tchan *tchan,
283 int reg, u32 val)
284{
285 if (!tchan)
286 return;
287 udma_write(tchan->reg_rt, reg, val);
288}
289
290/* RCHANRT */
291static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg)
292{
293 if (!rchan)
294 return 0;
295 return udma_read(rchan->reg_rt, reg);
296}
297
298static inline void udma_rchanrt_write(struct udma_rchan *rchan,
299 int reg, u32 val)
300{
301 if (!rchan)
302 return;
303 udma_write(rchan->reg_rt, reg, val);
304}
305
306static inline int udma_navss_psil_pair(struct udma_dev *ud, u32 src_thread,
307 u32 dst_thread)
308{
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]309 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
310
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]311 dst_thread |= UDMA_PSIL_DST_THREAD_ID_OFFSET;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]312
313 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
314 tisci_rm->tisci_navss_dev_id,
315 src_thread, dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]316}
317
318static inline int udma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
319 u32 dst_thread)
320{
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]321 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
322
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]323 dst_thread |= UDMA_PSIL_DST_THREAD_ID_OFFSET;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]324
325 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
326 tisci_rm->tisci_navss_dev_id,
327 src_thread, dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]328}
329
330static inline char *udma_get_dir_text(enum dma_direction dir)
331{
332 switch (dir) {
333 case DMA_DEV_TO_MEM:
334 return "DEV_TO_MEM";
335 case DMA_MEM_TO_DEV:
336 return "MEM_TO_DEV";
337 case DMA_MEM_TO_MEM:
338 return "MEM_TO_MEM";
339 case DMA_DEV_TO_DEV:
340 return "DEV_TO_DEV";
341 default:
342 break;
343 }
344
345 return "invalid";
346}
347
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]348#include "k3-udma-u-boot.c"
349
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]350static void udma_reset_uchan(struct udma_chan *uc)
351{
352 memset(&uc->config, 0, sizeof(uc->config));
353 uc->config.remote_thread_id = -1;
354 uc->config.mapped_channel_id = -1;
355 uc->config.default_flow_id = -1;
356}
357
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]358static inline bool udma_is_chan_running(struct udma_chan *uc)
359{
360 u32 trt_ctl = 0;
361 u32 rrt_ctl = 0;
362
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]363 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]364 case DMA_DEV_TO_MEM:
365 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
366 pr_debug("%s: rrt_ctl: 0x%08x (peer: 0x%08x)\n",
367 __func__, rrt_ctl,
368 udma_rchanrt_read(uc->rchan,
369 UDMA_RCHAN_RT_PEER_RT_EN_REG));
370 break;
371 case DMA_MEM_TO_DEV:
372 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
373 pr_debug("%s: trt_ctl: 0x%08x (peer: 0x%08x)\n",
374 __func__, trt_ctl,
375 udma_tchanrt_read(uc->tchan,
376 UDMA_TCHAN_RT_PEER_RT_EN_REG));
377 break;
378 case DMA_MEM_TO_MEM:
379 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
380 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
381 break;
382 default:
383 break;
384 }
385
386 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
387 return true;
388
389 return false;
390}
391
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]392static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
393{
394 struct k3_nav_ring *ring = NULL;
395 int ret = -ENOENT;
396
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]397 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]398 case DMA_DEV_TO_MEM:
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]399 ring = uc->rflow->r_ring;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]400 break;
401 case DMA_MEM_TO_DEV:
402 ring = uc->tchan->tc_ring;
403 break;
404 case DMA_MEM_TO_MEM:
405 ring = uc->tchan->tc_ring;
406 break;
407 default:
408 break;
409 }
410
411 if (ring && k3_nav_ringacc_ring_get_occ(ring))
412 ret = k3_nav_ringacc_ring_pop(ring, addr);
413
414 return ret;
415}
416
417static void udma_reset_rings(struct udma_chan *uc)
418{
419 struct k3_nav_ring *ring1 = NULL;
420 struct k3_nav_ring *ring2 = NULL;
421
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]422 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]423 case DMA_DEV_TO_MEM:
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]424 ring1 = uc->rflow->fd_ring;
425 ring2 = uc->rflow->r_ring;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]426 break;
427 case DMA_MEM_TO_DEV:
428 ring1 = uc->tchan->t_ring;
429 ring2 = uc->tchan->tc_ring;
430 break;
431 case DMA_MEM_TO_MEM:
432 ring1 = uc->tchan->t_ring;
433 ring2 = uc->tchan->tc_ring;
434 break;
435 default:
436 break;
437 }
438
439 if (ring1)
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]440 k3_nav_ringacc_ring_reset_dma(ring1, k3_nav_ringacc_ring_get_occ(ring1));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]441 if (ring2)
442 k3_nav_ringacc_ring_reset(ring2);
443}
444
445static void udma_reset_counters(struct udma_chan *uc)
446{
447 u32 val;
448
449 if (uc->tchan) {
450 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG);
451 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val);
452
453 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG);
454 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val);
455
456 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG);
457 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val);
458
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]459 if (!uc->bchan) {
460 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG);
461 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val);
462 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]463 }
464
465 if (uc->rchan) {
466 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG);
467 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val);
468
469 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG);
470 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val);
471
472 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG);
473 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val);
474
475 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG);
476 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val);
477 }
478
479 uc->bcnt = 0;
480}
481
482static inline int udma_stop_hard(struct udma_chan *uc)
483{
484 pr_debug("%s: ENTER (chan%d)\n", __func__, uc->id);
485
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]486 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]487 case DMA_DEV_TO_MEM:
488 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0);
489 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
490 break;
491 case DMA_MEM_TO_DEV:
492 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
493 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0);
494 break;
495 case DMA_MEM_TO_MEM:
496 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
497 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
498 break;
499 default:
500 return -EINVAL;
501 }
502
503 return 0;
504}
505
506static int udma_start(struct udma_chan *uc)
507{
508 /* Channel is already running, no need to proceed further */
509 if (udma_is_chan_running(uc))
510 goto out;
511
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]512 pr_debug("%s: chan:%d dir:%s\n",
513 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]514
515 /* Make sure that we clear the teardown bit, if it is set */
516 udma_stop_hard(uc);
517
518 /* Reset all counters */
519 udma_reset_counters(uc);
520
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]521 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]522 case DMA_DEV_TO_MEM:
523 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG,
524 UDMA_CHAN_RT_CTL_EN);
525
526 /* Enable remote */
527 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG,
528 UDMA_PEER_RT_EN_ENABLE);
529
530 pr_debug("%s(rx): RT_CTL:0x%08x PEER RT_ENABLE:0x%08x\n",
531 __func__,
532 udma_rchanrt_read(uc->rchan,
533 UDMA_RCHAN_RT_CTL_REG),
534 udma_rchanrt_read(uc->rchan,
535 UDMA_RCHAN_RT_PEER_RT_EN_REG));
536 break;
537 case DMA_MEM_TO_DEV:
538 /* Enable remote */
539 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG,
540 UDMA_PEER_RT_EN_ENABLE);
541
542 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
543 UDMA_CHAN_RT_CTL_EN);
544
545 pr_debug("%s(tx): RT_CTL:0x%08x PEER RT_ENABLE:0x%08x\n",
546 __func__,
Vignesh Raghavendrac2237992019-12-09 10:25:36 +0530[diff] [blame]547 udma_tchanrt_read(uc->tchan,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]548 UDMA_TCHAN_RT_CTL_REG),
Vignesh Raghavendrac2237992019-12-09 10:25:36 +0530[diff] [blame]549 udma_tchanrt_read(uc->tchan,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]550 UDMA_TCHAN_RT_PEER_RT_EN_REG));
551 break;
552 case DMA_MEM_TO_MEM:
553 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG,
554 UDMA_CHAN_RT_CTL_EN);
555 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
556 UDMA_CHAN_RT_CTL_EN);
557
558 break;
559 default:
560 return -EINVAL;
561 }
562
563 pr_debug("%s: DONE chan:%d\n", __func__, uc->id);
564out:
565 return 0;
566}
567
568static inline void udma_stop_mem2dev(struct udma_chan *uc, bool sync)
569{
570 int i = 0;
571 u32 val;
572
573 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
574 UDMA_CHAN_RT_CTL_EN |
575 UDMA_CHAN_RT_CTL_TDOWN);
576
577 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
578
579 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
580 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
581 udelay(1);
582 if (i > 1000) {
583 printf(" %s TIMEOUT !\n", __func__);
584 break;
585 }
586 i++;
587 }
588
589 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG);
590 if (val & UDMA_PEER_RT_EN_ENABLE)
591 printf("%s: peer not stopped TIMEOUT !\n", __func__);
592}
593
594static inline void udma_stop_dev2mem(struct udma_chan *uc, bool sync)
595{
596 int i = 0;
597 u32 val;
598
599 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG,
600 UDMA_PEER_RT_EN_ENABLE |
601 UDMA_PEER_RT_EN_TEARDOWN);
602
603 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
604
605 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
606 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
607 udelay(1);
608 if (i > 1000) {
609 printf("%s TIMEOUT !\n", __func__);
610 break;
611 }
612 i++;
613 }
614
615 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG);
616 if (val & UDMA_PEER_RT_EN_ENABLE)
617 printf("%s: peer not stopped TIMEOUT !\n", __func__);
618}
619
620static inline int udma_stop(struct udma_chan *uc)
621{
622 pr_debug("%s: chan:%d dir:%s\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]623 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]624
625 udma_reset_counters(uc);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]626 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]627 case DMA_DEV_TO_MEM:
628 udma_stop_dev2mem(uc, true);
629 break;
630 case DMA_MEM_TO_DEV:
631 udma_stop_mem2dev(uc, true);
632 break;
633 case DMA_MEM_TO_MEM:
634 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
635 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
636 break;
637 default:
638 return -EINVAL;
639 }
640
641 return 0;
642}
643
644static void udma_poll_completion(struct udma_chan *uc, dma_addr_t *paddr)
645{
646 int i = 1;
647
648 while (udma_pop_from_ring(uc, paddr)) {
649 udelay(1);
650 if (!(i % 1000000))
651 printf(".");
652 i++;
653 }
654}
655
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]656static struct udma_rflow *__udma_reserve_rflow(struct udma_dev *ud, int id)
657{
658 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
659
660 if (id >= 0) {
661 if (test_bit(id, ud->rflow_map)) {
662 dev_err(ud->dev, "rflow%d is in use\n", id);
663 return ERR_PTR(-ENOENT);
664 }
665 } else {
666 bitmap_or(tmp, ud->rflow_map, ud->rflow_map_reserved,
667 ud->rflow_cnt);
668
669 id = find_next_zero_bit(tmp, ud->rflow_cnt, ud->rchan_cnt);
670 if (id >= ud->rflow_cnt)
671 return ERR_PTR(-ENOENT);
672 }
673
674 __set_bit(id, ud->rflow_map);
675 return &ud->rflows[id];
676}
677
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]678#define UDMA_RESERVE_RESOURCE(res) \
679static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \
680 int id) \
681{ \
682 if (id >= 0) { \
683 if (test_bit(id, ud->res##_map)) { \
684 dev_err(ud->dev, "res##%d is in use\n", id); \
685 return ERR_PTR(-ENOENT); \
686 } \
687 } else { \
688 id = find_first_zero_bit(ud->res##_map, ud->res##_cnt); \
689 if (id == ud->res##_cnt) { \
690 return ERR_PTR(-ENOENT); \
691 } \
692 } \
693 \
694 __set_bit(id, ud->res##_map); \
695 return &ud->res##s[id]; \
696}
697
698UDMA_RESERVE_RESOURCE(tchan);
699UDMA_RESERVE_RESOURCE(rchan);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]700
701static int udma_get_tchan(struct udma_chan *uc)
702{
703 struct udma_dev *ud = uc->ud;
704
705 if (uc->tchan) {
706 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
707 uc->id, uc->tchan->id);
708 return 0;
709 }
710
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]711 uc->tchan = __udma_reserve_tchan(ud, uc->config.mapped_channel_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]712 if (IS_ERR(uc->tchan))
713 return PTR_ERR(uc->tchan);
714
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]715 if (ud->tflow_cnt) {
716 int tflow_id;
717
718 /* Only PKTDMA have support for tx flows */
719 if (uc->config.default_flow_id >= 0)
720 tflow_id = uc->config.default_flow_id;
721 else
722 tflow_id = uc->tchan->id;
723
724 if (test_bit(tflow_id, ud->tflow_map)) {
725 dev_err(ud->dev, "tflow%d is in use\n", tflow_id);
726 __clear_bit(uc->tchan->id, ud->tchan_map);
727 uc->tchan = NULL;
728 return -ENOENT;
729 }
730
731 uc->tchan->tflow_id = tflow_id;
732 __set_bit(tflow_id, ud->tflow_map);
733 } else {
734 uc->tchan->tflow_id = -1;
735 }
736
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]737 pr_debug("chan%d: got tchan%d\n", uc->id, uc->tchan->id);
738
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]739 return 0;
740}
741
742static int udma_get_rchan(struct udma_chan *uc)
743{
744 struct udma_dev *ud = uc->ud;
745
746 if (uc->rchan) {
747 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
748 uc->id, uc->rchan->id);
749 return 0;
750 }
751
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]752 uc->rchan = __udma_reserve_rchan(ud, uc->config.mapped_channel_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]753 if (IS_ERR(uc->rchan))
754 return PTR_ERR(uc->rchan);
755
756 pr_debug("chan%d: got rchan%d\n", uc->id, uc->rchan->id);
757
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]758 return 0;
759}
760
761static int udma_get_chan_pair(struct udma_chan *uc)
762{
763 struct udma_dev *ud = uc->ud;
764 int chan_id, end;
765
766 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
767 dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
768 uc->id, uc->tchan->id);
769 return 0;
770 }
771
772 if (uc->tchan) {
773 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
774 uc->id, uc->tchan->id);
775 return -EBUSY;
776 } else if (uc->rchan) {
777 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
778 uc->id, uc->rchan->id);
779 return -EBUSY;
780 }
781
782 /* Can be optimized, but let's have it like this for now */
783 end = min(ud->tchan_cnt, ud->rchan_cnt);
784 for (chan_id = 0; chan_id < end; chan_id++) {
785 if (!test_bit(chan_id, ud->tchan_map) &&
786 !test_bit(chan_id, ud->rchan_map))
787 break;
788 }
789
790 if (chan_id == end)
791 return -ENOENT;
792
793 __set_bit(chan_id, ud->tchan_map);
794 __set_bit(chan_id, ud->rchan_map);
795 uc->tchan = &ud->tchans[chan_id];
796 uc->rchan = &ud->rchans[chan_id];
797
798 pr_debug("chan%d: got t/rchan%d pair\n", uc->id, chan_id);
799
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]800 return 0;
801}
802
803static int udma_get_rflow(struct udma_chan *uc, int flow_id)
804{
805 struct udma_dev *ud = uc->ud;
806
807 if (uc->rflow) {
808 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
809 uc->id, uc->rflow->id);
810 return 0;
811 }
812
813 if (!uc->rchan)
814 dev_warn(ud->dev, "chan%d: does not have rchan??\n", uc->id);
815
816 uc->rflow = __udma_reserve_rflow(ud, flow_id);
817 if (IS_ERR(uc->rflow))
818 return PTR_ERR(uc->rflow);
819
820 pr_debug("chan%d: got rflow%d\n", uc->id, uc->rflow->id);
821 return 0;
822}
823
824static void udma_put_rchan(struct udma_chan *uc)
825{
826 struct udma_dev *ud = uc->ud;
827
828 if (uc->rchan) {
829 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
830 uc->rchan->id);
831 __clear_bit(uc->rchan->id, ud->rchan_map);
832 uc->rchan = NULL;
833 }
834}
835
836static void udma_put_tchan(struct udma_chan *uc)
837{
838 struct udma_dev *ud = uc->ud;
839
840 if (uc->tchan) {
841 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
842 uc->tchan->id);
843 __clear_bit(uc->tchan->id, ud->tchan_map);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]844 if (uc->tchan->tflow_id >= 0)
845 __clear_bit(uc->tchan->tflow_id, ud->tflow_map);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]846 uc->tchan = NULL;
847 }
848}
849
850static void udma_put_rflow(struct udma_chan *uc)
851{
852 struct udma_dev *ud = uc->ud;
853
854 if (uc->rflow) {
855 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
856 uc->rflow->id);
857 __clear_bit(uc->rflow->id, ud->rflow_map);
858 uc->rflow = NULL;
859 }
860}
861
862static void udma_free_tx_resources(struct udma_chan *uc)
863{
864 if (!uc->tchan)
865 return;
866
867 k3_nav_ringacc_ring_free(uc->tchan->t_ring);
868 k3_nav_ringacc_ring_free(uc->tchan->tc_ring);
869 uc->tchan->t_ring = NULL;
870 uc->tchan->tc_ring = NULL;
871
872 udma_put_tchan(uc);
873}
874
875static int udma_alloc_tx_resources(struct udma_chan *uc)
876{
877 struct k3_nav_ring_cfg ring_cfg;
878 struct udma_dev *ud = uc->ud;
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]879 struct udma_tchan *tchan;
880 int ring_idx, ret;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]881
882 ret = udma_get_tchan(uc);
883 if (ret)
884 return ret;
885
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]886 tchan = uc->tchan;
Udit Kumarf084e402024-02-21 19:53:44 +0530[diff] [blame]887 if (tchan->tflow_id > 0)
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]888 ring_idx = tchan->tflow_id;
889 else
Udit Kumarf084e402024-02-21 19:53:44 +0530[diff] [blame]890 ring_idx = tchan->id;
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]891
892 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1,
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]893 &uc->tchan->t_ring,
894 &uc->tchan->tc_ring);
895 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]896 ret = -EBUSY;
897 goto err_tx_ring;
898 }
899
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]900 memset(&ring_cfg, 0, sizeof(ring_cfg));
901 ring_cfg.size = 16;
902 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
Vignesh Raghavendra0fe24d32019-12-09 10:25:37 +0530[diff] [blame]903 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]904
905 ret = k3_nav_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg);
906 ret |= k3_nav_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg);
907
908 if (ret)
909 goto err_ringcfg;
910
911 return 0;
912
913err_ringcfg:
914 k3_nav_ringacc_ring_free(uc->tchan->tc_ring);
915 uc->tchan->tc_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]916 k3_nav_ringacc_ring_free(uc->tchan->t_ring);
917 uc->tchan->t_ring = NULL;
918err_tx_ring:
919 udma_put_tchan(uc);
920
921 return ret;
922}
923
924static void udma_free_rx_resources(struct udma_chan *uc)
925{
926 if (!uc->rchan)
927 return;
928
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]929 if (uc->rflow) {
930 k3_nav_ringacc_ring_free(uc->rflow->fd_ring);
931 k3_nav_ringacc_ring_free(uc->rflow->r_ring);
932 uc->rflow->fd_ring = NULL;
933 uc->rflow->r_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]934
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]935 udma_put_rflow(uc);
936 }
937
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]938 udma_put_rchan(uc);
939}
940
941static int udma_alloc_rx_resources(struct udma_chan *uc)
942{
943 struct k3_nav_ring_cfg ring_cfg;
944 struct udma_dev *ud = uc->ud;
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]945 struct udma_rflow *rflow;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]946 int fd_ring_id;
947 int ret;
948
949 ret = udma_get_rchan(uc);
950 if (ret)
951 return ret;
952
953 /* For MEM_TO_MEM we don't need rflow or rings */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]954 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]955 return 0;
956
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]957 if (uc->config.default_flow_id >= 0)
958 ret = udma_get_rflow(uc, uc->config.default_flow_id);
959 else
960 ret = udma_get_rflow(uc, uc->rchan->id);
961
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]962 if (ret) {
963 ret = -EBUSY;
964 goto err_rflow;
965 }
966
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]967 rflow = uc->rflow;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]968 if (ud->tflow_cnt) {
969 fd_ring_id = ud->tflow_cnt + rflow->id;
970 } else {
971 fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt +
972 uc->rchan->id;
973 }
974
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]975 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
976 &rflow->fd_ring, &rflow->r_ring);
977 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]978 ret = -EBUSY;
979 goto err_rx_ring;
980 }
981
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]982 memset(&ring_cfg, 0, sizeof(ring_cfg));
983 ring_cfg.size = 16;
984 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
Vignesh Raghavendra0fe24d32019-12-09 10:25:37 +0530[diff] [blame]985 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]986
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]987 ret = k3_nav_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
988 ret |= k3_nav_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]989 if (ret)
990 goto err_ringcfg;
991
992 return 0;
993
994err_ringcfg:
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]995 k3_nav_ringacc_ring_free(rflow->r_ring);
996 rflow->r_ring = NULL;
997 k3_nav_ringacc_ring_free(rflow->fd_ring);
998 rflow->fd_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]999err_rx_ring:
1000 udma_put_rflow(uc);
1001err_rflow:
1002 udma_put_rchan(uc);
1003
1004 return ret;
1005}
1006
1007static int udma_alloc_tchan_sci_req(struct udma_chan *uc)
1008{
1009 struct udma_dev *ud = uc->ud;
1010 int tc_ring = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1011 struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1012 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1013 u32 mode;
1014 int ret;
1015
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1016 if (uc->config.pkt_mode)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1017 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1018 else
1019 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1020
1021 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
1022 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
1023 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1024 req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1025 req.index = uc->tchan->id;
1026 req.tx_chan_type = mode;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1027 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1028 req.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1029 else
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1030 req.tx_fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1031 uc->config.psd_size,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1032 0) >> 2;
1033 req.txcq_qnum = tc_ring;
1034
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1035 ret = tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1036 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1037 dev_err(ud->dev, "tisci tx alloc failed %d\n", ret);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1038 return ret;
1039 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1040
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1041 /*
1042 * Above TI SCI call handles firewall configuration, cfg
1043 * register configuration still has to be done locally in
1044 * absence of RM services.
1045 */
1046 if (IS_ENABLED(CONFIG_K3_DM_FW))
1047 udma_alloc_tchan_raw(uc);
1048
1049 return 0;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1050}
1051
1052static int udma_alloc_rchan_sci_req(struct udma_chan *uc)
1053{
1054 struct udma_dev *ud = uc->ud;
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]1055 int fd_ring = k3_nav_ringacc_get_ring_id(uc->rflow->fd_ring);
1056 int rx_ring = k3_nav_ringacc_get_ring_id(uc->rflow->r_ring);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1057 int tc_ring = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1058 struct ti_sci_msg_rm_udmap_rx_ch_cfg req = { 0 };
1059 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1060 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1061 u32 mode;
1062 int ret;
1063
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1064 if (uc->config.pkt_mode)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1065 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1066 else
1067 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1068
1069 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
1070 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1071 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1072 req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1073 req.index = uc->rchan->id;
1074 req.rx_chan_type = mode;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1075 if (uc->config.dir == DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1076 req.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1077 req.rxcq_qnum = tc_ring;
1078 } else {
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1079 req.rx_fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1080 uc->config.psd_size,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1081 0) >> 2;
1082 req.rxcq_qnum = rx_ring;
1083 }
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1084 if (ud->match_data->type == DMA_TYPE_UDMA &&
1085 uc->rflow->id != uc->rchan->id &&
1086 uc->config.dir != DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1087 req.flowid_start = uc->rflow->id;
1088 req.flowid_cnt = 1;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1089 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
1090 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1091 }
1092
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1093 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1094 if (ret) {
1095 dev_err(ud->dev, "tisci rx %u cfg failed %d\n",
1096 uc->rchan->id, ret);
1097 return ret;
1098 }
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1099 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1100 return ret;
1101
1102 flow_req.valid_params =
1103 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
1104 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
1105 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
1106 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
1107 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1108 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
1109 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
1110 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
1111 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
1112 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1113 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1114 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1115 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID |
1116 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PS_LOCATION_VALID;
1117
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1118 flow_req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1119 flow_req.flow_index = uc->rflow->id;
1120
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1121 if (uc->config.needs_epib)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1122 flow_req.rx_einfo_present = 1;
1123 else
1124 flow_req.rx_einfo_present = 0;
1125
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1126 if (uc->config.psd_size)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1127 flow_req.rx_psinfo_present = 1;
1128 else
1129 flow_req.rx_psinfo_present = 0;
1130
1131 flow_req.rx_error_handling = 0;
1132 flow_req.rx_desc_type = 0;
1133 flow_req.rx_dest_qnum = rx_ring;
1134 flow_req.rx_src_tag_hi_sel = 2;
1135 flow_req.rx_src_tag_lo_sel = 4;
1136 flow_req.rx_dest_tag_hi_sel = 5;
1137 flow_req.rx_dest_tag_lo_sel = 4;
1138 flow_req.rx_fdq0_sz0_qnum = fd_ring;
1139 flow_req.rx_fdq1_qnum = fd_ring;
1140 flow_req.rx_fdq2_qnum = fd_ring;
1141 flow_req.rx_fdq3_qnum = fd_ring;
1142 flow_req.rx_ps_location = 0;
1143
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1144 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci,
1145 &flow_req);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1146 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1147 dev_err(ud->dev, "tisci rx %u flow %u cfg failed %d\n",
1148 uc->rchan->id, uc->rflow->id, ret);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1149 return ret;
1150 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1151
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1152 /*
1153 * Above TI SCI call handles firewall configuration, cfg
1154 * register configuration still has to be done locally in
1155 * absence of RM services.
1156 */
1157 if (IS_ENABLED(CONFIG_K3_DM_FW))
1158 udma_alloc_rchan_raw(uc);
1159
1160 return 0;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1161}
1162
1163static int udma_alloc_chan_resources(struct udma_chan *uc)
1164{
1165 struct udma_dev *ud = uc->ud;
1166 int ret;
1167
1168 pr_debug("%s: chan:%d as %s\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1169 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1170
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1171 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1172 case DMA_MEM_TO_MEM:
1173 /* Non synchronized - mem to mem type of transfer */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1174 uc->config.pkt_mode = false;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1175 ret = udma_get_chan_pair(uc);
1176 if (ret)
1177 return ret;
1178
1179 ret = udma_alloc_tx_resources(uc);
1180 if (ret)
1181 goto err_free_res;
1182
1183 ret = udma_alloc_rx_resources(uc);
1184 if (ret)
1185 goto err_free_res;
1186
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1187 uc->config.src_thread = ud->psil_base + uc->tchan->id;
1188 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1189 break;
1190 case DMA_MEM_TO_DEV:
1191 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
1192 ret = udma_alloc_tx_resources(uc);
1193 if (ret)
1194 goto err_free_res;
1195
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1196 uc->config.src_thread = ud->psil_base + uc->tchan->id;
1197 uc->config.dst_thread = uc->config.remote_thread_id;
1198 uc->config.dst_thread |= 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1199
1200 break;
1201 case DMA_DEV_TO_MEM:
1202 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
1203 ret = udma_alloc_rx_resources(uc);
1204 if (ret)
1205 goto err_free_res;
1206
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1207 uc->config.src_thread = uc->config.remote_thread_id;
1208 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1209
1210 break;
1211 default:
1212 /* Can not happen */
1213 pr_debug("%s: chan:%d invalid direction (%u)\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1214 __func__, uc->id, uc->config.dir);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1215 return -EINVAL;
1216 }
1217
1218 /* We have channel indexes and rings */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1219 if (uc->config.dir == DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1220 ret = udma_alloc_tchan_sci_req(uc);
1221 if (ret)
1222 goto err_free_res;
1223
1224 ret = udma_alloc_rchan_sci_req(uc);
1225 if (ret)
1226 goto err_free_res;
1227 } else {
1228 /* Slave transfer */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1229 if (uc->config.dir == DMA_MEM_TO_DEV) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1230 ret = udma_alloc_tchan_sci_req(uc);
1231 if (ret)
1232 goto err_free_res;
1233 } else {
1234 ret = udma_alloc_rchan_sci_req(uc);
1235 if (ret)
1236 goto err_free_res;
1237 }
1238 }
1239
Peter Ujfalusid15f8652019-04-25 12:08:15 +0530[diff] [blame]1240 if (udma_is_chan_running(uc)) {
1241 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
1242 udma_stop(uc);
1243 if (udma_is_chan_running(uc)) {
1244 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
1245 goto err_free_res;
1246 }
1247 }
1248
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1249 /* PSI-L pairing */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1250 ret = udma_navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1251 if (ret) {
1252 dev_err(ud->dev, "k3_nav_psil_request_link fail\n");
1253 goto err_free_res;
1254 }
1255
1256 return 0;
1257
1258err_free_res:
1259 udma_free_tx_resources(uc);
1260 udma_free_rx_resources(uc);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1261 uc->config.remote_thread_id = -1;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1262 return ret;
1263}
1264
1265static void udma_free_chan_resources(struct udma_chan *uc)
1266{
Vignesh Raghavendrabe7bdcc2020-09-17 20:11:22 +0530[diff] [blame]1267 /* Hard reset UDMA channel */
1268 udma_stop_hard(uc);
1269 udma_reset_counters(uc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1270
1271 /* Release PSI-L pairing */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1272 udma_navss_psil_unpair(uc->ud, uc->config.src_thread, uc->config.dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1273
1274 /* Reset the rings for a new start */
1275 udma_reset_rings(uc);
1276 udma_free_tx_resources(uc);
1277 udma_free_rx_resources(uc);
1278
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1279 uc->config.remote_thread_id = -1;
1280 uc->config.dir = DMA_MEM_TO_MEM;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1281}
1282
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1283static const char * const range_names[] = {
1284 [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan",
1285 [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan",
1286 [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan",
1287 [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow",
1288 [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow",
1289};
1290
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1291static int udma_get_mmrs(struct udevice *dev)
1292{
1293 struct udma_dev *ud = dev_get_priv(dev);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1294 u32 cap2, cap3, cap4;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1295 int i;
1296
Matthias Schiffer47331932023-09-27 15:33:34 +0200[diff] [blame]1297 ud->mmrs[MMR_GCFG] = dev_read_addr_name_ptr(dev, mmr_names[MMR_GCFG]);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1298 if (!ud->mmrs[MMR_GCFG])
1299 return -EINVAL;
1300
1301 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);
1302 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
1303
1304 switch (ud->match_data->type) {
1305 case DMA_TYPE_UDMA:
1306 ud->rflow_cnt = cap3 & 0x3fff;
1307 ud->tchan_cnt = cap2 & 0x1ff;
1308 ud->echan_cnt = (cap2 >> 9) & 0x1ff;
1309 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1310 break;
1311 case DMA_TYPE_BCDMA:
1312 ud->bchan_cnt = cap2 & 0x1ff;
1313 ud->tchan_cnt = (cap2 >> 9) & 0x1ff;
1314 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1315 break;
1316 case DMA_TYPE_PKTDMA:
1317 cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30);
1318 ud->tchan_cnt = cap2 & 0x1ff;
1319 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1320 ud->rflow_cnt = cap3 & 0x3fff;
1321 ud->tflow_cnt = cap4 & 0x3fff;
1322 break;
1323 default:
1324 return -EINVAL;
1325 }
1326
1327 for (i = 1; i < MMR_LAST; i++) {
1328 if (i == MMR_BCHANRT && ud->bchan_cnt == 0)
1329 continue;
1330 if (i == MMR_TCHANRT && ud->tchan_cnt == 0)
1331 continue;
1332 if (i == MMR_RCHANRT && ud->rchan_cnt == 0)
1333 continue;
1334
Matthias Schiffer47331932023-09-27 15:33:34 +0200[diff] [blame]1335 ud->mmrs[i] = dev_read_addr_name_ptr(dev, mmr_names[i]);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1336 if (!ud->mmrs[i])
1337 return -EINVAL;
1338 }
1339
1340 return 0;
1341}
1342
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1343static int udma_setup_resources(struct udma_dev *ud)
1344{
1345 struct udevice *dev = ud->dev;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1346 int i;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1347 struct ti_sci_resource_desc *rm_desc;
1348 struct ti_sci_resource *rm_res;
1349 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1350
1351 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1352 sizeof(unsigned long), GFP_KERNEL);
1353 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1354 GFP_KERNEL);
1355 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1356 sizeof(unsigned long), GFP_KERNEL);
1357 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1358 GFP_KERNEL);
1359 ud->rflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
1360 sizeof(unsigned long), GFP_KERNEL);
1361 ud->rflow_map_reserved = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
1362 sizeof(unsigned long),
1363 GFP_KERNEL);
1364 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
1365 GFP_KERNEL);
1366
1367 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_map ||
1368 !ud->rflow_map_reserved || !ud->tchans || !ud->rchans ||
1369 !ud->rflows)
1370 return -ENOMEM;
1371
1372 /*
1373 * RX flows with the same Ids as RX channels are reserved to be used
1374 * as default flows if remote HW can't generate flow_ids. Those
1375 * RX flows can be requested only explicitly by id.
1376 */
1377 bitmap_set(ud->rflow_map_reserved, 0, ud->rchan_cnt);
1378
1379 /* Get resource ranges from tisci */
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1380 for (i = 0; i < RM_RANGE_LAST; i++) {
1381 if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW)
1382 continue;
1383
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1384 tisci_rm->rm_ranges[i] =
1385 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1386 tisci_rm->tisci_dev_id,
1387 (char *)range_names[i]);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1388 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1389
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1390 /* tchan ranges */
1391 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1392 if (IS_ERR(rm_res)) {
1393 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1394 } else {
1395 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1396 for (i = 0; i < rm_res->sets; i++) {
1397 rm_desc = &rm_res->desc[i];
1398 bitmap_clear(ud->tchan_map, rm_desc->start,
1399 rm_desc->num);
1400 }
1401 }
1402
1403 /* rchan and matching default flow ranges */
1404 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1405 if (IS_ERR(rm_res)) {
1406 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1407 bitmap_zero(ud->rflow_map, ud->rchan_cnt);
1408 } else {
1409 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1410 bitmap_fill(ud->rflow_map, ud->rchan_cnt);
1411 for (i = 0; i < rm_res->sets; i++) {
1412 rm_desc = &rm_res->desc[i];
1413 bitmap_clear(ud->rchan_map, rm_desc->start,
1414 rm_desc->num);
1415 bitmap_clear(ud->rflow_map, rm_desc->start,
1416 rm_desc->num);
1417 }
1418 }
1419
1420 /* GP rflow ranges */
1421 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
1422 if (IS_ERR(rm_res)) {
1423 bitmap_clear(ud->rflow_map, ud->rchan_cnt,
1424 ud->rflow_cnt - ud->rchan_cnt);
1425 } else {
1426 bitmap_set(ud->rflow_map, ud->rchan_cnt,
1427 ud->rflow_cnt - ud->rchan_cnt);
1428 for (i = 0; i < rm_res->sets; i++) {
1429 rm_desc = &rm_res->desc[i];
1430 bitmap_clear(ud->rflow_map, rm_desc->start,
1431 rm_desc->num);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1432 }
1433 }
1434
1435 return 0;
1436}
1437
1438static int bcdma_setup_resources(struct udma_dev *ud)
1439{
1440 int i;
1441 struct udevice *dev = ud->dev;
1442 struct ti_sci_resource_desc *rm_desc;
1443 struct ti_sci_resource *rm_res;
1444 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1445
1446 ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt),
1447 sizeof(unsigned long), GFP_KERNEL);
1448 ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans),
1449 GFP_KERNEL);
1450 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1451 sizeof(unsigned long), GFP_KERNEL);
1452 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1453 GFP_KERNEL);
1454 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1455 sizeof(unsigned long), GFP_KERNEL);
1456 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1457 GFP_KERNEL);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1458 ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows),
1459 GFP_KERNEL);
1460
1461 if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map ||
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1462 !ud->bchans || !ud->tchans || !ud->rchans ||
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1463 !ud->rflows)
1464 return -ENOMEM;
1465
1466 /* Get resource ranges from tisci */
1467 for (i = 0; i < RM_RANGE_LAST; i++) {
1468 if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW)
1469 continue;
1470
1471 tisci_rm->rm_ranges[i] =
1472 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1473 tisci_rm->tisci_dev_id,
1474 (char *)range_names[i]);
1475 }
1476
1477 /* bchan ranges */
1478 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
1479 if (IS_ERR(rm_res)) {
1480 bitmap_zero(ud->bchan_map, ud->bchan_cnt);
1481 } else {
1482 bitmap_fill(ud->bchan_map, ud->bchan_cnt);
1483 for (i = 0; i < rm_res->sets; i++) {
1484 rm_desc = &rm_res->desc[i];
1485 bitmap_clear(ud->bchan_map, rm_desc->start,
1486 rm_desc->num);
1487 dev_dbg(dev, "ti-sci-res: bchan: %d:%d\n",
1488 rm_desc->start, rm_desc->num);
1489 }
1490 }
1491
1492 /* tchan ranges */
1493 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1494 if (IS_ERR(rm_res)) {
1495 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1496 } else {
1497 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1498 for (i = 0; i < rm_res->sets; i++) {
1499 rm_desc = &rm_res->desc[i];
1500 bitmap_clear(ud->tchan_map, rm_desc->start,
1501 rm_desc->num);
1502 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
1503 rm_desc->start, rm_desc->num);
1504 }
1505 }
1506
1507 /* rchan ranges */
1508 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1509 if (IS_ERR(rm_res)) {
1510 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1511 } else {
1512 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1513 for (i = 0; i < rm_res->sets; i++) {
1514 rm_desc = &rm_res->desc[i];
1515 bitmap_clear(ud->rchan_map, rm_desc->start,
1516 rm_desc->num);
1517 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
1518 rm_desc->start, rm_desc->num);
1519 }
1520 }
1521
1522 return 0;
1523}
1524
1525static int pktdma_setup_resources(struct udma_dev *ud)
1526{
1527 int i;
1528 struct udevice *dev = ud->dev;
1529 struct ti_sci_resource *rm_res;
1530 struct ti_sci_resource_desc *rm_desc;
1531 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1532
1533 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1534 sizeof(unsigned long), GFP_KERNEL);
1535 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1536 GFP_KERNEL);
1537 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1538 sizeof(unsigned long), GFP_KERNEL);
1539 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1540 GFP_KERNEL);
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1541 ud->rflow_map = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
1542 sizeof(unsigned long),
1543 GFP_KERNEL);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1544 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
1545 GFP_KERNEL);
1546 ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt),
1547 sizeof(unsigned long), GFP_KERNEL);
1548
1549 if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans ||
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1550 !ud->rchans || !ud->rflows || !ud->rflow_map)
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1551 return -ENOMEM;
1552
1553 /* Get resource ranges from tisci */
1554 for (i = 0; i < RM_RANGE_LAST; i++) {
1555 if (i == RM_RANGE_BCHAN)
1556 continue;
1557
1558 tisci_rm->rm_ranges[i] =
1559 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1560 tisci_rm->tisci_dev_id,
1561 (char *)range_names[i]);
1562 }
1563
1564 /* tchan ranges */
1565 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1566 if (IS_ERR(rm_res)) {
1567 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1568 } else {
1569 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1570 for (i = 0; i < rm_res->sets; i++) {
1571 rm_desc = &rm_res->desc[i];
1572 bitmap_clear(ud->tchan_map, rm_desc->start,
1573 rm_desc->num);
1574 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
1575 rm_desc->start, rm_desc->num);
1576 }
1577 }
1578
1579 /* rchan ranges */
1580 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1581 if (IS_ERR(rm_res)) {
1582 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1583 } else {
1584 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1585 for (i = 0; i < rm_res->sets; i++) {
1586 rm_desc = &rm_res->desc[i];
1587 bitmap_clear(ud->rchan_map, rm_desc->start,
1588 rm_desc->num);
1589 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
1590 rm_desc->start, rm_desc->num);
1591 }
1592 }
1593
1594 /* rflow ranges */
1595 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
1596 if (IS_ERR(rm_res)) {
1597 /* all rflows are assigned exclusively to Linux */
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1598 bitmap_zero(ud->rflow_map, ud->rflow_cnt);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1599 } else {
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1600 bitmap_fill(ud->rflow_map, ud->rflow_cnt);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1601 for (i = 0; i < rm_res->sets; i++) {
1602 rm_desc = &rm_res->desc[i];
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1603 bitmap_clear(ud->rflow_map, rm_desc->start,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1604 rm_desc->num);
1605 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n",
1606 rm_desc->start, rm_desc->num);
1607 }
1608 }
1609
1610 /* tflow ranges */
1611 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
1612 if (IS_ERR(rm_res)) {
1613 /* all tflows are assigned exclusively to Linux */
1614 bitmap_zero(ud->tflow_map, ud->tflow_cnt);
1615 } else {
1616 bitmap_fill(ud->tflow_map, ud->tflow_cnt);
1617 for (i = 0; i < rm_res->sets; i++) {
1618 rm_desc = &rm_res->desc[i];
1619 bitmap_clear(ud->tflow_map, rm_desc->start,
1620 rm_desc->num);
1621 dev_dbg(dev, "ti-sci-res: tflow: %d:%d\n",
1622 rm_desc->start, rm_desc->num);
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1623 }
1624 }
1625
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1626 return 0;
1627}
1628
1629static int setup_resources(struct udma_dev *ud)
1630{
1631 struct udevice *dev = ud->dev;
1632 int ch_count, ret;
1633
1634 switch (ud->match_data->type) {
1635 case DMA_TYPE_UDMA:
1636 ret = udma_setup_resources(ud);
1637 break;
1638 case DMA_TYPE_BCDMA:
1639 ret = bcdma_setup_resources(ud);
1640 break;
1641 case DMA_TYPE_PKTDMA:
1642 ret = pktdma_setup_resources(ud);
1643 break;
1644 default:
1645 return -EINVAL;
1646 }
1647
1648 if (ret)
1649 return ret;
1650
1651 ch_count = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt;
1652 if (ud->bchan_cnt)
1653 ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt);
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1654 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
1655 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
1656 if (!ch_count)
1657 return -ENODEV;
1658
1659 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
1660 GFP_KERNEL);
1661 if (!ud->channels)
1662 return -ENOMEM;
1663
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1664 switch (ud->match_data->type) {
1665 case DMA_TYPE_UDMA:
1666 dev_dbg(dev,
1667 "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
1668 ch_count,
1669 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1670 ud->tchan_cnt),
1671 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1672 ud->rchan_cnt),
1673 ud->rflow_cnt - bitmap_weight(ud->rflow_map,
1674 ud->rflow_cnt));
1675 break;
1676 case DMA_TYPE_BCDMA:
1677 dev_dbg(dev,
1678 "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n",
1679 ch_count,
1680 ud->bchan_cnt - bitmap_weight(ud->bchan_map,
1681 ud->bchan_cnt),
1682 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1683 ud->tchan_cnt),
1684 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1685 ud->rchan_cnt));
1686 break;
1687 case DMA_TYPE_PKTDMA:
1688 dev_dbg(dev,
1689 "Channels: %d (tchan: %u, rchan: %u)\n",
1690 ch_count,
1691 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1692 ud->tchan_cnt),
1693 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1694 ud->rchan_cnt));
1695 break;
1696 default:
1697 break;
1698 }
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1699
1700 return ch_count;
1701}
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1702
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]1703static int udma_push_to_ring(struct k3_nav_ring *ring, void *elem)
1704{
1705 u64 addr = 0;
1706
1707 memcpy(&addr, &elem, sizeof(elem));
1708 return k3_nav_ringacc_ring_push(ring, &addr);
1709}
1710
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1711static int *udma_prep_dma_memcpy(struct udma_chan *uc, dma_addr_t dest,
1712 dma_addr_t src, size_t len)
1713{
1714 u32 tc_ring_id = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1715 struct cppi5_tr_type15_t *tr_req;
1716 int num_tr;
1717 size_t tr_size = sizeof(struct cppi5_tr_type15_t);
1718 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
1719 unsigned long dummy;
1720 void *tr_desc;
1721 size_t desc_size;
1722
1723 if (len < SZ_64K) {
1724 num_tr = 1;
1725 tr0_cnt0 = len;
1726 tr0_cnt1 = 1;
1727 } else {
1728 unsigned long align_to = __ffs(src | dest);
1729
1730 if (align_to > 3)
1731 align_to = 3;
1732 /*
1733 * Keep simple: tr0: SZ_64K-alignment blocks,
1734 * tr1: the remaining
1735 */
1736 num_tr = 2;
1737 tr0_cnt0 = (SZ_64K - BIT(align_to));
1738 if (len / tr0_cnt0 >= SZ_64K) {
1739 dev_err(uc->ud->dev, "size %zu is not supported\n",
1740 len);
1741 return NULL;
1742 }
1743
1744 tr0_cnt1 = len / tr0_cnt0;
1745 tr1_cnt0 = len % tr0_cnt0;
1746 }
1747
1748 desc_size = cppi5_trdesc_calc_size(num_tr, tr_size);
1749 tr_desc = dma_alloc_coherent(desc_size, &dummy);
1750 if (!tr_desc)
1751 return NULL;
1752 memset(tr_desc, 0, desc_size);
1753
1754 cppi5_trdesc_init(tr_desc, num_tr, tr_size, 0, 0);
1755 cppi5_desc_set_pktids(tr_desc, uc->id, 0x3fff);
1756 cppi5_desc_set_retpolicy(tr_desc, 0, tc_ring_id);
1757
1758 tr_req = tr_desc + tr_size;
1759
1760 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
1761 CPPI5_TR_EVENT_SIZE_COMPLETION, 1);
1762 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT);
1763
1764 tr_req[0].addr = src;
1765 tr_req[0].icnt0 = tr0_cnt0;
1766 tr_req[0].icnt1 = tr0_cnt1;
1767 tr_req[0].icnt2 = 1;
1768 tr_req[0].icnt3 = 1;
1769 tr_req[0].dim1 = tr0_cnt0;
1770
1771 tr_req[0].daddr = dest;
1772 tr_req[0].dicnt0 = tr0_cnt0;
1773 tr_req[0].dicnt1 = tr0_cnt1;
1774 tr_req[0].dicnt2 = 1;
1775 tr_req[0].dicnt3 = 1;
1776 tr_req[0].ddim1 = tr0_cnt0;
1777
1778 if (num_tr == 2) {
1779 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
1780 CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
1781 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT);
1782
1783 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
1784 tr_req[1].icnt0 = tr1_cnt0;
1785 tr_req[1].icnt1 = 1;
1786 tr_req[1].icnt2 = 1;
1787 tr_req[1].icnt3 = 1;
1788
1789 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
1790 tr_req[1].dicnt0 = tr1_cnt0;
1791 tr_req[1].dicnt1 = 1;
1792 tr_req[1].dicnt2 = 1;
1793 tr_req[1].dicnt3 = 1;
1794 }
1795
1796 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, CPPI5_TR_CSF_EOP);
1797
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]1798 flush_dcache_range((unsigned long)tr_desc,
1799 ALIGN((unsigned long)tr_desc + desc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]1800 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1801
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]1802 udma_push_to_ring(uc->tchan->t_ring, tr_desc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1803
1804 return 0;
1805}
1806
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1807#define TISCI_BCDMA_BCHAN_VALID_PARAMS ( \
1808 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1809 TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID)
1810
1811#define TISCI_BCDMA_TCHAN_VALID_PARAMS ( \
1812 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1813 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID)
1814
1815#define TISCI_BCDMA_RCHAN_VALID_PARAMS ( \
1816 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID)
1817
1818#define TISCI_UDMA_TCHAN_VALID_PARAMS ( \
1819 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1820 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \
1821 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \
1822 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
1823 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \
1824 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
1825 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
1826 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1827
1828#define TISCI_UDMA_RCHAN_VALID_PARAMS ( \
1829 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1830 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
1831 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
1832 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
1833 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \
1834 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \
1835 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \
1836 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \
1837 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1838
1839static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc)
1840{
1841 struct udma_dev *ud = uc->ud;
1842 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1843 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1844 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1845 struct udma_bchan *bchan = uc->bchan;
1846 int ret = 0;
1847
1848 req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS;
1849 req_tx.nav_id = tisci_rm->tisci_dev_id;
1850 req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN;
1851 req_tx.index = bchan->id;
1852
1853 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1854 if (ret)
1855 dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret);
1856
1857 return ret;
1858}
1859
1860static struct udma_bchan *__bcdma_reserve_bchan(struct udma_dev *ud, int id)
1861{
1862 if (id >= 0) {
1863 if (test_bit(id, ud->bchan_map)) {
1864 dev_err(ud->dev, "bchan%d is in use\n", id);
1865 return ERR_PTR(-ENOENT);
1866 }
1867 } else {
1868 id = find_next_zero_bit(ud->bchan_map, ud->bchan_cnt, 0);
1869 if (id == ud->bchan_cnt)
1870 return ERR_PTR(-ENOENT);
1871 }
1872 __set_bit(id, ud->bchan_map);
1873 return &ud->bchans[id];
1874}
1875
1876static int bcdma_get_bchan(struct udma_chan *uc)
1877{
1878 struct udma_dev *ud = uc->ud;
1879
1880 if (uc->bchan) {
1881 dev_err(ud->dev, "chan%d: already have bchan%d allocated\n",
1882 uc->id, uc->bchan->id);
1883 return 0;
1884 }
1885
1886 uc->bchan = __bcdma_reserve_bchan(ud, -1);
1887 if (IS_ERR(uc->bchan))
1888 return PTR_ERR(uc->bchan);
1889
1890 uc->tchan = uc->bchan;
1891
1892 return 0;
1893}
1894
1895static void bcdma_put_bchan(struct udma_chan *uc)
1896{
1897 struct udma_dev *ud = uc->ud;
1898
1899 if (uc->bchan) {
1900 dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id,
1901 uc->bchan->id);
1902 __clear_bit(uc->bchan->id, ud->bchan_map);
1903 uc->bchan = NULL;
1904 uc->tchan = NULL;
1905 }
1906}
1907
1908static void bcdma_free_bchan_resources(struct udma_chan *uc)
1909{
1910 if (!uc->bchan)
1911 return;
1912
1913 k3_nav_ringacc_ring_free(uc->bchan->tc_ring);
1914 k3_nav_ringacc_ring_free(uc->bchan->t_ring);
1915 uc->bchan->tc_ring = NULL;
1916 uc->bchan->t_ring = NULL;
1917
1918 bcdma_put_bchan(uc);
1919}
1920
1921static int bcdma_alloc_bchan_resources(struct udma_chan *uc)
1922{
1923 struct k3_nav_ring_cfg ring_cfg;
1924 struct udma_dev *ud = uc->ud;
1925 int ret;
1926
1927 ret = bcdma_get_bchan(uc);
1928 if (ret)
1929 return ret;
1930
1931 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1,
1932 &uc->bchan->t_ring,
1933 &uc->bchan->tc_ring);
1934 if (ret) {
1935 ret = -EBUSY;
1936 goto err_ring;
1937 }
1938
1939 memset(&ring_cfg, 0, sizeof(ring_cfg));
1940 ring_cfg.size = 16;
1941 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
1942 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
1943
1944 ret = k3_nav_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg);
1945 if (ret)
1946 goto err_ringcfg;
1947
1948 return 0;
1949
1950err_ringcfg:
1951 k3_nav_ringacc_ring_free(uc->bchan->tc_ring);
1952 uc->bchan->tc_ring = NULL;
1953 k3_nav_ringacc_ring_free(uc->bchan->t_ring);
1954 uc->bchan->t_ring = NULL;
1955err_ring:
1956 bcdma_put_bchan(uc);
1957
1958 return ret;
1959}
1960
1961static int bcdma_tisci_tx_channel_config(struct udma_chan *uc)
1962{
1963 struct udma_dev *ud = uc->ud;
1964 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1965 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1966 struct udma_tchan *tchan = uc->tchan;
1967 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1968 int ret = 0;
1969
1970 req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS;
1971 req_tx.nav_id = tisci_rm->tisci_dev_id;
1972 req_tx.index = tchan->id;
1973 req_tx.tx_supr_tdpkt = uc->config.notdpkt;
1974 if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
1975 ud->match_data->flags & UDMA_FLAG_TDTYPE) {
1976 /* wait for peer to complete the teardown for PDMAs */
1977 req_tx.valid_params |=
1978 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
1979 req_tx.tx_tdtype = 1;
1980 }
1981
1982 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1983 if (ret)
1984 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1985
Kishon Vijay Abraham I8878dad2024-08-26 15:55:10 +0530[diff] [blame]1986 if (IS_ENABLED(CONFIG_K3_DM_FW))
1987 udma_alloc_tchan_raw(uc);
1988
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1989 return ret;
1990}
1991
1992#define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config
1993
1994static int pktdma_tisci_rx_channel_config(struct udma_chan *uc)
1995{
1996 struct udma_dev *ud = uc->ud;
1997 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1998 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1999 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2000 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
2001 int ret = 0;
2002
2003 req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2004 req_rx.nav_id = tisci_rm->tisci_dev_id;
2005 req_rx.index = uc->rchan->id;
2006
2007 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2008 if (ret) {
2009 dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret);
2010 return ret;
2011 }
2012
2013 flow_req.valid_params =
2014 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2015 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2016 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID;
2017
2018 flow_req.nav_id = tisci_rm->tisci_dev_id;
2019 flow_req.flow_index = uc->rflow->id;
2020
2021 if (uc->config.needs_epib)
2022 flow_req.rx_einfo_present = 1;
2023 else
2024 flow_req.rx_einfo_present = 0;
2025 if (uc->config.psd_size)
2026 flow_req.rx_psinfo_present = 1;
2027 else
2028 flow_req.rx_psinfo_present = 0;
Vignesh Raghavendra87fa0d62023-03-08 09:42:57 +0530[diff] [blame]2029 flow_req.rx_error_handling = 0;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2030
2031 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2032
2033 if (ret)
2034 dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id,
2035 ret);
2036
Kishon Vijay Abraham I8878dad2024-08-26 15:55:10 +0530[diff] [blame]2037 if (IS_ENABLED(CONFIG_K3_DM_FW))
2038 udma_alloc_rchan_raw(uc);
2039
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2040 return ret;
2041}
2042
2043static int bcdma_alloc_chan_resources(struct udma_chan *uc)
2044{
2045 int ret;
2046
2047 uc->config.pkt_mode = false;
2048
2049 switch (uc->config.dir) {
2050 case DMA_MEM_TO_MEM:
2051 /* Non synchronized - mem to mem type of transfer */
2052 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2053 uc->id);
2054
2055 ret = bcdma_alloc_bchan_resources(uc);
2056 if (ret)
2057 return ret;
2058
2059 ret = bcdma_tisci_m2m_channel_config(uc);
2060 break;
2061 default:
2062 /* Can not happen */
2063 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2064 __func__, uc->id, uc->config.dir);
2065 return -EINVAL;
2066 }
2067
2068 /* check if the channel configuration was successful */
2069 if (ret)
2070 goto err_res_free;
2071
2072 if (udma_is_chan_running(uc)) {
2073 dev_warn(uc->ud->dev, "chan%d: is running!\n", uc->id);
2074 udma_stop(uc);
2075 if (udma_is_chan_running(uc)) {
2076 dev_err(uc->ud->dev, "chan%d: won't stop!\n", uc->id);
2077 goto err_res_free;
2078 }
2079 }
2080
2081 udma_reset_rings(uc);
2082
2083 return 0;
2084
2085err_res_free:
2086 bcdma_free_bchan_resources(uc);
2087 udma_free_tx_resources(uc);
2088 udma_free_rx_resources(uc);
2089
2090 udma_reset_uchan(uc);
2091
2092 return ret;
2093}
2094
2095static int pktdma_alloc_chan_resources(struct udma_chan *uc)
2096{
2097 struct udma_dev *ud = uc->ud;
2098 int ret;
2099
2100 switch (uc->config.dir) {
2101 case DMA_MEM_TO_DEV:
2102 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2103 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2104 uc->id);
2105
2106 ret = udma_alloc_tx_resources(uc);
2107 if (ret) {
2108 uc->config.remote_thread_id = -1;
2109 return ret;
2110 }
2111
2112 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2113 uc->config.dst_thread = uc->config.remote_thread_id;
2114 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2115
2116 ret = pktdma_tisci_tx_channel_config(uc);
2117 break;
2118 case DMA_DEV_TO_MEM:
2119 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2120 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2121 uc->id);
2122
2123 ret = udma_alloc_rx_resources(uc);
2124 if (ret) {
2125 uc->config.remote_thread_id = -1;
2126 return ret;
2127 }
2128
2129 uc->config.src_thread = uc->config.remote_thread_id;
2130 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2131 K3_PSIL_DST_THREAD_ID_OFFSET;
2132
2133 ret = pktdma_tisci_rx_channel_config(uc);
2134 break;
2135 default:
2136 /* Can not happen */
2137 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2138 __func__, uc->id, uc->config.dir);
2139 return -EINVAL;
2140 }
2141
2142 /* check if the channel configuration was successful */
2143 if (ret)
2144 goto err_res_free;
2145
2146 /* PSI-L pairing */
2147 ret = udma_navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2148 if (ret) {
2149 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2150 uc->config.src_thread, uc->config.dst_thread);
2151 goto err_res_free;
2152 }
2153
2154 if (udma_is_chan_running(uc)) {
2155 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2156 udma_stop(uc);
2157 if (udma_is_chan_running(uc)) {
2158 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2159 goto err_res_free;
2160 }
2161 }
2162
2163 udma_reset_rings(uc);
2164
2165 if (uc->tchan)
2166 dev_dbg(ud->dev,
2167 "chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n",
2168 uc->id, uc->tchan->id, uc->tchan->tflow_id,
2169 uc->config.remote_thread_id);
2170 else if (uc->rchan)
2171 dev_dbg(ud->dev,
2172 "chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n",
2173 uc->id, uc->rchan->id, uc->rflow->id,
2174 uc->config.remote_thread_id);
2175 return 0;
2176
2177err_res_free:
2178 udma_free_tx_resources(uc);
2179 udma_free_rx_resources(uc);
2180
2181 udma_reset_uchan(uc);
2182
2183 return ret;
2184}
2185
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2186static int udma_transfer(struct udevice *dev, int direction,
Andrew Davisd2da2842022-10-07 12:11:13 -0500[diff] [blame]2187 dma_addr_t dst, dma_addr_t src, size_t len)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2188{
2189 struct udma_dev *ud = dev_get_priv(dev);
2190 /* Channel0 is reserved for memcpy */
2191 struct udma_chan *uc = &ud->channels[0];
2192 dma_addr_t paddr = 0;
2193 int ret;
2194
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2195 switch (ud->match_data->type) {
2196 case DMA_TYPE_UDMA:
2197 ret = udma_alloc_chan_resources(uc);
2198 break;
2199 case DMA_TYPE_BCDMA:
2200 ret = bcdma_alloc_chan_resources(uc);
2201 break;
2202 default:
2203 return -EINVAL;
2204 };
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2205 if (ret)
2206 return ret;
2207
Andrew Davisd2da2842022-10-07 12:11:13 -0500[diff] [blame]2208 udma_prep_dma_memcpy(uc, dst, src, len);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2209 udma_start(uc);
2210 udma_poll_completion(uc, &paddr);
2211 udma_stop(uc);
2212
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2213 switch (ud->match_data->type) {
2214 case DMA_TYPE_UDMA:
2215 udma_free_chan_resources(uc);
2216 break;
2217 case DMA_TYPE_BCDMA:
2218 bcdma_free_bchan_resources(uc);
2219 break;
2220 default:
2221 return -EINVAL;
2222 };
2223
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2224 return 0;
2225}
2226
2227static int udma_request(struct dma *dma)
2228{
2229 struct udma_dev *ud = dev_get_priv(dma->dev);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2230 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2231 struct udma_chan *uc;
2232 unsigned long dummy;
2233 int ret;
2234
2235 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2236 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2237 return -EINVAL;
2238 }
2239
2240 uc = &ud->channels[dma->id];
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2241 ucc = &uc->config;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2242 switch (ud->match_data->type) {
2243 case DMA_TYPE_UDMA:
2244 ret = udma_alloc_chan_resources(uc);
2245 break;
2246 case DMA_TYPE_BCDMA:
2247 ret = bcdma_alloc_chan_resources(uc);
2248 break;
2249 case DMA_TYPE_PKTDMA:
2250 ret = pktdma_alloc_chan_resources(uc);
2251 break;
2252 default:
2253 return -EINVAL;
2254 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2255 if (ret) {
2256 dev_err(dma->dev, "alloc dma res failed %d\n", ret);
2257 return -EINVAL;
2258 }
2259
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2260 if (uc->config.dir == DMA_MEM_TO_DEV) {
2261 uc->desc_tx = dma_alloc_coherent(ucc->hdesc_size, &dummy);
2262 memset(uc->desc_tx, 0, ucc->hdesc_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2263 } else {
2264 uc->desc_rx = dma_alloc_coherent(
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2265 ucc->hdesc_size * UDMA_RX_DESC_NUM, &dummy);
2266 memset(uc->desc_rx, 0, ucc->hdesc_size * UDMA_RX_DESC_NUM);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2267 }
2268
2269 uc->in_use = true;
2270 uc->desc_rx_cur = 0;
2271 uc->num_rx_bufs = 0;
2272
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2273 if (uc->config.dir == DMA_DEV_TO_MEM) {
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2274 uc->cfg_data.flow_id_base = uc->rflow->id;
2275 uc->cfg_data.flow_id_cnt = 1;
2276 }
2277
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2278 return 0;
2279}
2280
Simon Glass75c0ad62020-02-03 07:35:55 -0700[diff] [blame]2281static int udma_rfree(struct dma *dma)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2282{
2283 struct udma_dev *ud = dev_get_priv(dma->dev);
2284 struct udma_chan *uc;
2285
2286 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2287 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2288 return -EINVAL;
2289 }
2290 uc = &ud->channels[dma->id];
2291
2292 if (udma_is_chan_running(uc))
2293 udma_stop(uc);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2294
2295 udma_navss_psil_unpair(ud, uc->config.src_thread,
2296 uc->config.dst_thread);
2297
2298 bcdma_free_bchan_resources(uc);
2299 udma_free_tx_resources(uc);
2300 udma_free_rx_resources(uc);
2301 udma_reset_uchan(uc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2302
2303 uc->in_use = false;
2304
2305 return 0;
2306}
2307
2308static int udma_enable(struct dma *dma)
2309{
2310 struct udma_dev *ud = dev_get_priv(dma->dev);
2311 struct udma_chan *uc;
2312 int ret;
2313
2314 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2315 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2316 return -EINVAL;
2317 }
2318 uc = &ud->channels[dma->id];
2319
2320 ret = udma_start(uc);
2321
2322 return ret;
2323}
2324
2325static int udma_disable(struct dma *dma)
2326{
2327 struct udma_dev *ud = dev_get_priv(dma->dev);
2328 struct udma_chan *uc;
2329 int ret = 0;
2330
2331 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2332 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2333 return -EINVAL;
2334 }
2335 uc = &ud->channels[dma->id];
2336
2337 if (udma_is_chan_running(uc))
2338 ret = udma_stop(uc);
2339 else
2340 dev_err(dma->dev, "%s not running\n", __func__);
2341
2342 return ret;
2343}
2344
2345static int udma_send(struct dma *dma, void *src, size_t len, void *metadata)
2346{
2347 struct udma_dev *ud = dev_get_priv(dma->dev);
2348 struct cppi5_host_desc_t *desc_tx;
2349 dma_addr_t dma_src = (dma_addr_t)src;
2350 struct ti_udma_drv_packet_data packet_data = { 0 };
2351 dma_addr_t paddr;
2352 struct udma_chan *uc;
2353 u32 tc_ring_id;
2354 int ret;
2355
Keerthya3c8bb12019-04-24 16:33:54 +0530[diff] [blame]2356 if (metadata)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2357 packet_data = *((struct ti_udma_drv_packet_data *)metadata);
2358
2359 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2360 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2361 return -EINVAL;
2362 }
2363 uc = &ud->channels[dma->id];
2364
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2365 if (uc->config.dir != DMA_MEM_TO_DEV)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2366 return -EINVAL;
2367
2368 tc_ring_id = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
2369
2370 desc_tx = uc->desc_tx;
2371
2372 cppi5_hdesc_reset_hbdesc(desc_tx);
2373
2374 cppi5_hdesc_init(desc_tx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2375 uc->config.needs_epib ? CPPI5_INFO0_HDESC_EPIB_PRESENT : 0,
2376 uc->config.psd_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2377 cppi5_hdesc_set_pktlen(desc_tx, len);
2378 cppi5_hdesc_attach_buf(desc_tx, dma_src, len, dma_src, len);
2379 cppi5_desc_set_pktids(&desc_tx->hdr, uc->id, 0x3fff);
2380 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, tc_ring_id);
2381 /* pass below information from caller */
2382 cppi5_hdesc_set_pkttype(desc_tx, packet_data.pkt_type);
2383 cppi5_desc_set_tags_ids(&desc_tx->hdr, 0, packet_data.dest_tag);
2384
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2385 flush_dcache_range((unsigned long)dma_src,
2386 ALIGN((unsigned long)dma_src + len,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2387 ARCH_DMA_MINALIGN));
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2388 flush_dcache_range((unsigned long)desc_tx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2389 ALIGN((unsigned long)desc_tx + uc->config.hdesc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2390 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2391
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]2392 ret = udma_push_to_ring(uc->tchan->t_ring, uc->desc_tx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2393 if (ret) {
2394 dev_err(dma->dev, "TX dma push fail ch_id %lu %d\n",
2395 dma->id, ret);
2396 return ret;
2397 }
2398
2399 udma_poll_completion(uc, &paddr);
2400
2401 return 0;
2402}
2403
2404static int udma_receive(struct dma *dma, void **dst, void *metadata)
2405{
2406 struct udma_dev *ud = dev_get_priv(dma->dev);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2407 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2408 struct cppi5_host_desc_t *desc_rx;
2409 dma_addr_t buf_dma;
2410 struct udma_chan *uc;
2411 u32 buf_dma_len, pkt_len;
2412 u32 port_id = 0;
2413 int ret;
2414
2415 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2416 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2417 return -EINVAL;
2418 }
2419 uc = &ud->channels[dma->id];
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2420 ucc = &uc->config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2421
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2422 if (uc->config.dir != DMA_DEV_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2423 return -EINVAL;
2424 if (!uc->num_rx_bufs)
2425 return -EINVAL;
2426
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]2427 ret = k3_nav_ringacc_ring_pop(uc->rflow->r_ring, &desc_rx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2428 if (ret && ret != -ENODATA) {
2429 dev_err(dma->dev, "rx dma fail ch_id:%lu %d\n", dma->id, ret);
2430 return ret;
2431 } else if (ret == -ENODATA) {
2432 return 0;
2433 }
2434
2435 /* invalidate cache data */
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2436 invalidate_dcache_range((ulong)desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2437 (ulong)(desc_rx + ucc->hdesc_size));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2438
2439 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
2440 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
2441
2442 /* invalidate cache data */
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2443 invalidate_dcache_range((ulong)buf_dma,
2444 (ulong)(buf_dma + buf_dma_len));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2445
2446 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
2447
2448 *dst = (void *)buf_dma;
2449 uc->num_rx_bufs--;
2450
2451 return pkt_len;
2452}
2453
2454static int udma_of_xlate(struct dma *dma, struct ofnode_phandle_args *args)
2455{
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2456 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2457 struct udma_dev *ud = dev_get_priv(dma->dev);
2458 struct udma_chan *uc = &ud->channels[0];
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2459 struct psil_endpoint_config *ep_config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2460 u32 val;
2461
2462 for (val = 0; val < ud->ch_count; val++) {
2463 uc = &ud->channels[val];
2464 if (!uc->in_use)
2465 break;
2466 }
2467
2468 if (val == ud->ch_count)
2469 return -EBUSY;
2470
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2471 ucc = &uc->config;
2472 ucc->remote_thread_id = args->args[0];
2473 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)
2474 ucc->dir = DMA_MEM_TO_DEV;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2475 else
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2476 ucc->dir = DMA_DEV_TO_MEM;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2477
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2478 ep_config = psil_get_ep_config(ucc->remote_thread_id);
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2479 if (IS_ERR(ep_config)) {
2480 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2481 uc->config.remote_thread_id);
2482 ucc->dir = DMA_MEM_TO_MEM;
2483 ucc->remote_thread_id = -1;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2484 return false;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2485 }
2486
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2487 ucc->pkt_mode = ep_config->pkt_mode;
2488 ucc->channel_tpl = ep_config->channel_tpl;
2489 ucc->notdpkt = ep_config->notdpkt;
2490 ucc->ep_type = ep_config->ep_type;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2491
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2492 if (ud->match_data->type == DMA_TYPE_PKTDMA &&
2493 ep_config->mapped_channel_id >= 0) {
2494 ucc->mapped_channel_id = ep_config->mapped_channel_id;
2495 ucc->default_flow_id = ep_config->default_flow_id;
2496 } else {
2497 ucc->mapped_channel_id = -1;
2498 ucc->default_flow_id = -1;
2499 }
2500
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2501 ucc->needs_epib = ep_config->needs_epib;
2502 ucc->psd_size = ep_config->psd_size;
2503 ucc->metadata_size = (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + ucc->psd_size;
2504
2505 ucc->hdesc_size = cppi5_hdesc_calc_size(ucc->needs_epib,
2506 ucc->psd_size, 0);
2507 ucc->hdesc_size = ALIGN(ucc->hdesc_size, ARCH_DMA_MINALIGN);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2508
2509 dma->id = uc->id;
2510 pr_debug("Allocated dma chn:%lu epib:%d psdata:%u meta:%u thread_id:%x\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2511 dma->id, ucc->needs_epib,
2512 ucc->psd_size, ucc->metadata_size,
2513 ucc->remote_thread_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2514
2515 return 0;
2516}
2517
2518int udma_prepare_rcv_buf(struct dma *dma, void *dst, size_t size)
2519{
2520 struct udma_dev *ud = dev_get_priv(dma->dev);
2521 struct cppi5_host_desc_t *desc_rx;
2522 dma_addr_t dma_dst;
2523 struct udma_chan *uc;
2524 u32 desc_num;
2525
2526 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2527 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2528 return -EINVAL;
2529 }
2530 uc = &ud->channels[dma->id];
2531
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2532 if (uc->config.dir != DMA_DEV_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2533 return -EINVAL;
2534
2535 if (uc->num_rx_bufs >= UDMA_RX_DESC_NUM)
2536 return -EINVAL;
2537
2538 desc_num = uc->desc_rx_cur % UDMA_RX_DESC_NUM;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2539 desc_rx = uc->desc_rx + (desc_num * uc->config.hdesc_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2540 dma_dst = (dma_addr_t)dst;
2541
2542 cppi5_hdesc_reset_hbdesc(desc_rx);
2543
2544 cppi5_hdesc_init(desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2545 uc->config.needs_epib ? CPPI5_INFO0_HDESC_EPIB_PRESENT : 0,
2546 uc->config.psd_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2547 cppi5_hdesc_set_pktlen(desc_rx, size);
2548 cppi5_hdesc_attach_buf(desc_rx, dma_dst, size, dma_dst, size);
2549
Matthias Schiffer65aef702024-04-26 10:02:28 +0200[diff] [blame]2550 invalidate_dcache_range((unsigned long)dma_dst,
2551 (unsigned long)(dma_dst + size));
2552
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2553 flush_dcache_range((unsigned long)desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2554 ALIGN((unsigned long)desc_rx + uc->config.hdesc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2555 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2556
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]2557 udma_push_to_ring(uc->rflow->fd_ring, desc_rx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2558
2559 uc->num_rx_bufs++;
2560 uc->desc_rx_cur++;
2561
2562 return 0;
2563}
2564
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2565static int udma_get_cfg(struct dma *dma, u32 id, void **data)
2566{
2567 struct udma_dev *ud = dev_get_priv(dma->dev);
2568 struct udma_chan *uc;
2569
2570 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2571 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2572 return -EINVAL;
2573 }
2574
2575 switch (id) {
2576 case TI_UDMA_CHAN_PRIV_INFO:
2577 uc = &ud->channels[dma->id];
2578 *data = &uc->cfg_data;
2579 return 0;
2580 }
2581
2582 return -EINVAL;
2583}
2584
Santhosh Kumar K976edc62024-10-09 20:27:02 +0530[diff] [blame^]2585static int udma_probe(struct udevice *dev)
2586{
2587 struct dma_dev_priv *uc_priv = dev_get_uclass_priv(dev);
2588 struct udma_dev *ud = dev_get_priv(dev);
2589 int i, ret;
2590 struct udevice *tmp;
2591 struct udevice *tisci_dev = NULL;
2592 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2593 ofnode navss_ofnode = ofnode_get_parent(dev_ofnode(dev));
2594
2595 ud->match_data = (void *)dev_get_driver_data(dev);
2596 ret = udma_get_mmrs(dev);
2597 if (ret)
2598 return ret;
2599
2600 ud->psil_base = ud->match_data->psil_base;
2601
2602 ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, dev,
2603 "ti,sci", &tisci_dev);
2604 if (ret) {
2605 debug("Failed to get TISCI phandle (%d)\n", ret);
2606 tisci_rm->tisci = NULL;
2607 return -EINVAL;
2608 }
2609 tisci_rm->tisci = (struct ti_sci_handle *)
2610 (ti_sci_get_handle_from_sysfw(tisci_dev));
2611
2612 tisci_rm->tisci_dev_id = -1;
2613 ret = dev_read_u32(dev, "ti,sci-dev-id", &tisci_rm->tisci_dev_id);
2614 if (ret) {
2615 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
2616 return ret;
2617 }
2618
2619 tisci_rm->tisci_navss_dev_id = -1;
2620 ret = ofnode_read_u32(navss_ofnode, "ti,sci-dev-id",
2621 &tisci_rm->tisci_navss_dev_id);
2622 if (ret) {
2623 dev_err(dev, "navss sci-dev-id read failure %d\n", ret);
2624 return ret;
2625 }
2626
2627 tisci_rm->tisci_udmap_ops = &tisci_rm->tisci->ops.rm_udmap_ops;
2628 tisci_rm->tisci_psil_ops = &tisci_rm->tisci->ops.rm_psil_ops;
2629
2630 if (ud->match_data->type == DMA_TYPE_UDMA) {
2631 ret = uclass_get_device_by_phandle(UCLASS_MISC, dev,
2632 "ti,ringacc", &tmp);
2633 ud->ringacc = dev_get_priv(tmp);
2634 } else {
2635 struct k3_ringacc_init_data ring_init_data;
2636
2637 ring_init_data.tisci = ud->tisci_rm.tisci;
2638 ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id;
2639 if (ud->match_data->type == DMA_TYPE_BCDMA) {
2640 ring_init_data.num_rings = ud->bchan_cnt +
2641 ud->tchan_cnt +
2642 ud->rchan_cnt;
2643 } else {
2644 ring_init_data.num_rings = ud->rflow_cnt +
2645 ud->tflow_cnt;
2646 }
2647
2648 ud->ringacc = k3_ringacc_dmarings_init(dev, &ring_init_data);
2649 }
2650 if (IS_ERR(ud->ringacc))
2651 return PTR_ERR(ud->ringacc);
2652
2653 ud->dev = dev;
2654 ret = setup_resources(ud);
2655 if (ret < 0)
2656 return ret;
2657
2658 ud->ch_count = ret;
2659
2660 for (i = 0; i < ud->bchan_cnt; i++) {
2661 struct udma_bchan *bchan = &ud->bchans[i];
2662
2663 bchan->id = i;
2664 bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000;
2665 }
2666
2667 for (i = 0; i < ud->tchan_cnt; i++) {
2668 struct udma_tchan *tchan = &ud->tchans[i];
2669
2670 tchan->id = i;
2671 tchan->reg_chan = ud->mmrs[MMR_TCHAN] + UDMA_CH_100(i);
2672 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + UDMA_CH_1000(i);
2673 }
2674
2675 for (i = 0; i < ud->rchan_cnt; i++) {
2676 struct udma_rchan *rchan = &ud->rchans[i];
2677
2678 rchan->id = i;
2679 rchan->reg_chan = ud->mmrs[MMR_RCHAN] + UDMA_CH_100(i);
2680 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + UDMA_CH_1000(i);
2681 }
2682
2683 for (i = 0; i < ud->rflow_cnt; i++) {
2684 struct udma_rflow *rflow = &ud->rflows[i];
2685
2686 rflow->id = i;
2687 rflow->reg_rflow = ud->mmrs[MMR_RFLOW] + UDMA_CH_40(i);
2688 }
2689
2690 for (i = 0; i < ud->ch_count; i++) {
2691 struct udma_chan *uc = &ud->channels[i];
2692
2693 uc->ud = ud;
2694 uc->id = i;
2695 uc->config.remote_thread_id = -1;
2696 uc->bchan = NULL;
2697 uc->tchan = NULL;
2698 uc->rchan = NULL;
2699 uc->config.mapped_channel_id = -1;
2700 uc->config.default_flow_id = -1;
2701 uc->config.dir = DMA_MEM_TO_MEM;
2702 sprintf(uc->name, "UDMA chan%d\n", i);
2703 if (!i)
2704 uc->in_use = true;
2705 }
2706
2707 pr_debug("%s(rev: 0x%08x) CAP0-3: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
2708 dev->name,
2709 udma_read(ud->mmrs[MMR_GCFG], 0),
2710 udma_read(ud->mmrs[MMR_GCFG], 0x20),
2711 udma_read(ud->mmrs[MMR_GCFG], 0x24),
2712 udma_read(ud->mmrs[MMR_GCFG], 0x28),
2713 udma_read(ud->mmrs[MMR_GCFG], 0x2c));
2714
2715 uc_priv->supported = DMA_SUPPORTS_MEM_TO_MEM | DMA_SUPPORTS_MEM_TO_DEV;
2716
2717 return 0;
2718}
2719
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2720static const struct dma_ops udma_ops = {
2721 .transfer = udma_transfer,
2722 .of_xlate = udma_of_xlate,
2723 .request = udma_request,
Simon Glass75c0ad62020-02-03 07:35:55 -0700[diff] [blame]2724 .rfree = udma_rfree,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2725 .enable = udma_enable,
2726 .disable = udma_disable,
2727 .send = udma_send,
2728 .receive = udma_receive,
2729 .prepare_rcv_buf = udma_prepare_rcv_buf,
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2730 .get_cfg = udma_get_cfg,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2731};
2732
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2733static struct udma_match_data am654_main_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2734 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2735 .psil_base = 0x1000,
2736 .enable_memcpy_support = true,
2737 .statictr_z_mask = GENMASK(11, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2738 .oes = {
2739 .udma_rchan = 0x200,
2740 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2741 .tpl_levels = 2,
2742 .level_start_idx = {
2743 [0] = 8, /* Normal channels */
2744 [1] = 0, /* High Throughput channels */
2745 },
2746};
2747
2748static struct udma_match_data am654_mcu_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2749 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2750 .psil_base = 0x6000,
2751 .enable_memcpy_support = true,
2752 .statictr_z_mask = GENMASK(11, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2753 .oes = {
2754 .udma_rchan = 0x200,
2755 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2756 .tpl_levels = 2,
2757 .level_start_idx = {
2758 [0] = 2, /* Normal channels */
2759 [1] = 0, /* High Throughput channels */
2760 },
2761};
2762
2763static struct udma_match_data j721e_main_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2764 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2765 .psil_base = 0x1000,
2766 .enable_memcpy_support = true,
2767 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2768 .statictr_z_mask = GENMASK(23, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2769 .oes = {
2770 .udma_rchan = 0x400,
2771 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2772 .tpl_levels = 3,
2773 .level_start_idx = {
2774 [0] = 16, /* Normal channels */
2775 [1] = 4, /* High Throughput channels */
2776 [2] = 0, /* Ultra High Throughput channels */
2777 },
2778};
2779
2780static struct udma_match_data j721e_mcu_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2781 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2782 .psil_base = 0x6000,
2783 .enable_memcpy_support = true,
2784 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2785 .statictr_z_mask = GENMASK(23, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2786 .oes = {
2787 .udma_rchan = 0x400,
2788 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2789 .tpl_levels = 2,
2790 .level_start_idx = {
2791 [0] = 2, /* Normal channels */
2792 [1] = 0, /* High Throughput channels */
2793 },
2794};
2795
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2796static struct udma_match_data am64_bcdma_data = {
2797 .type = DMA_TYPE_BCDMA,
2798 .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */
2799 .enable_memcpy_support = true, /* Supported via bchan */
2800 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2801 .statictr_z_mask = GENMASK(23, 0),
2802 .oes = {
2803 .bcdma_bchan_data = 0x2200,
2804 .bcdma_bchan_ring = 0x2400,
2805 .bcdma_tchan_data = 0x2800,
2806 .bcdma_tchan_ring = 0x2a00,
2807 .bcdma_rchan_data = 0x2e00,
2808 .bcdma_rchan_ring = 0x3000,
2809 },
2810 /* No throughput levels */
2811};
2812
2813static struct udma_match_data am64_pktdma_data = {
2814 .type = DMA_TYPE_PKTDMA,
2815 .psil_base = 0x1000,
2816 .enable_memcpy_support = false,
2817 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2818 .statictr_z_mask = GENMASK(23, 0),
2819 .oes = {
2820 .pktdma_tchan_flow = 0x1200,
2821 .pktdma_rchan_flow = 0x1600,
2822 },
2823 /* No throughput levels */
2824};
2825
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2826static const struct udevice_id udma_ids[] = {
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2827 {
2828 .compatible = "ti,am654-navss-main-udmap",
2829 .data = (ulong)&am654_main_data,
2830 },
2831 {
2832 .compatible = "ti,am654-navss-mcu-udmap",
2833 .data = (ulong)&am654_mcu_data,
2834 }, {
2835 .compatible = "ti,j721e-navss-main-udmap",
2836 .data = (ulong)&j721e_main_data,
2837 }, {
2838 .compatible = "ti,j721e-navss-mcu-udmap",
2839 .data = (ulong)&j721e_mcu_data,
2840 },
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2841 {
2842 .compatible = "ti,am64-dmss-bcdma",
2843 .data = (ulong)&am64_bcdma_data,
2844 },
2845 {
2846 .compatible = "ti,am64-dmss-pktdma",
2847 .data = (ulong)&am64_pktdma_data,
2848 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2849 { /* Sentinel */ },
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2850};
2851
2852U_BOOT_DRIVER(ti_edma3) = {
2853 .name = "ti-udma",
2854 .id = UCLASS_DMA,
2855 .of_match = udma_ids,
2856 .ops = &udma_ops,
2857 .probe = udma_probe,
Simon Glass8a2b47f2020-12-03 16:55:17 -0700[diff] [blame]2858 .priv_auto = sizeof(struct udma_dev),
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2859};