Gitiles
Code Review Sign In
git01.mediatek.com / filogic / uboot / 94ef0166f2cd373ee29a9bb11cb8150561750139 / . / drivers / dma / ti / k3-udma.c
blob: da341a2477839ccd53ee8e6d5b0378fccf88738f [file] [log] [blame]
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1// SPDX-License-Identifier: GPL-2.0+
2/*
Nishanth Menoneaa39c62023-11-01 15:56:03 -0500[diff] [blame]3 * Copyright (C) 2018 Texas Instruments Incorporated - https://www.ti.com
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
5 */
6#define pr_fmt(fmt) "udma: " fmt
7
Simon Glass63334482019-11-14 12:57:39 -0700[diff] [blame]8#include <cpu_func.h>
Simon Glass0f2af882020-05-10 11:40:05 -0600[diff] [blame]9#include <log.h>
Simon Glass274e0b02020-05-10 11:39:56 -0600[diff] [blame]10#include <asm/cache.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]11#include <asm/io.h>
12#include <asm/bitops.h>
13#include <malloc.h>
Matthias Schifferfe2ee882024-04-26 10:02:27 +0200[diff] [blame]14#include <net.h>
Simon Glass4dcacfc2020-05-10 11:40:13 -0600[diff] [blame]15#include <linux/bitops.h>
Masahiro Yamada6373a172020-02-14 16:40:19 +0900[diff] [blame]16#include <linux/dma-mapping.h>
Dhruva Golee6b42392022-09-20 10:56:02 +0530[diff] [blame]17#include <linux/sizes.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]18#include <dm.h>
Simon Glass9bc15642020-02-03 07:36:16 -0700[diff] [blame]19#include <dm/device_compat.h>
Simon Glassd66c5f72020-02-03 07:36:15 -0700[diff] [blame]20#include <dm/devres.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]21#include <dm/read.h>
22#include <dm/of_access.h>
23#include <dma.h>
24#include <dma-uclass.h>
25#include <linux/delay.h>
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]26#include <linux/bitmap.h>
Simon Glassd66c5f72020-02-03 07:36:15 -0700[diff] [blame]27#include <linux/err.h>
Simon Glassbdd5f812023-09-14 18:21:46 -0600[diff] [blame]28#include <linux/printk.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]29#include <linux/soc/ti/k3-navss-ringacc.h>
30#include <linux/soc/ti/cppi5.h>
31#include <linux/soc/ti/ti-udma.h>
32#include <linux/soc/ti/ti_sci_protocol.h>
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]33#include <linux/soc/ti/cppi5.h>
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]34
35#include "k3-udma-hwdef.h"
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]36#include "k3-psil-priv.h"
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]37
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]38#define K3_UDMA_MAX_RFLOWS 1024
39
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]40struct udma_chan;
41
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]42enum k3_dma_type {
43 DMA_TYPE_UDMA = 0,
44 DMA_TYPE_BCDMA,
45 DMA_TYPE_PKTDMA,
46};
47
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]48enum udma_mmr {
49 MMR_GCFG = 0,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]50 MMR_BCHANRT,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]51 MMR_RCHANRT,
52 MMR_TCHANRT,
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]53 MMR_RCHAN,
54 MMR_TCHAN,
55 MMR_RFLOW,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]56 MMR_LAST,
57};
58
59static const char * const mmr_names[] = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]60 [MMR_GCFG] = "gcfg",
61 [MMR_BCHANRT] = "bchanrt",
62 [MMR_RCHANRT] = "rchanrt",
63 [MMR_TCHANRT] = "tchanrt",
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]64 [MMR_RCHAN] = "rchan",
65 [MMR_TCHAN] = "tchan",
66 [MMR_RFLOW] = "rflow",
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]67};
68
69struct udma_tchan {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]70 void __iomem *reg_chan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]71 void __iomem *reg_rt;
72
73 int id;
74 struct k3_nav_ring *t_ring; /* Transmit ring */
75 struct k3_nav_ring *tc_ring; /* Transmit Completion ring */
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]76 int tflow_id; /* applicable only for PKTDMA */
77
78};
79
80#define udma_bchan udma_tchan
81
82struct udma_rflow {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]83 void __iomem *reg_rflow;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]84 int id;
85 struct k3_nav_ring *fd_ring; /* Free Descriptor ring */
86 struct k3_nav_ring *r_ring; /* Receive ring */
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]87};
88
89struct udma_rchan {
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]90 void __iomem *reg_chan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]91 void __iomem *reg_rt;
92
93 int id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]94};
95
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]96struct udma_oes_offsets {
97 /* K3 UDMA Output Event Offset */
98 u32 udma_rchan;
99
100 /* BCDMA Output Event Offsets */
101 u32 bcdma_bchan_data;
102 u32 bcdma_bchan_ring;
103 u32 bcdma_tchan_data;
104 u32 bcdma_tchan_ring;
105 u32 bcdma_rchan_data;
106 u32 bcdma_rchan_ring;
107
108 /* PKTDMA Output Event Offsets */
109 u32 pktdma_tchan_flow;
110 u32 pktdma_rchan_flow;
111};
112
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]113#define UDMA_FLAG_PDMA_ACC32 BIT(0)
114#define UDMA_FLAG_PDMA_BURST BIT(1)
115#define UDMA_FLAG_TDTYPE BIT(2)
116
117struct udma_match_data {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]118 enum k3_dma_type type;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]119 u32 psil_base;
120 bool enable_memcpy_support;
121 u32 flags;
122 u32 statictr_z_mask;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]123 struct udma_oes_offsets oes;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]124
125 u8 tpl_levels;
126 u32 level_start_idx[];
127};
128
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]129enum udma_rm_range {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]130 RM_RANGE_BCHAN = 0,
131 RM_RANGE_TCHAN,
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]132 RM_RANGE_RCHAN,
133 RM_RANGE_RFLOW,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]134 RM_RANGE_TFLOW,
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]135 RM_RANGE_LAST,
136};
137
138struct udma_tisci_rm {
139 const struct ti_sci_handle *tisci;
140 const struct ti_sci_rm_udmap_ops *tisci_udmap_ops;
141 u32 tisci_dev_id;
142
143 /* tisci information for PSI-L thread pairing/unpairing */
144 const struct ti_sci_rm_psil_ops *tisci_psil_ops;
145 u32 tisci_navss_dev_id;
146
147 struct ti_sci_resource *rm_ranges[RM_RANGE_LAST];
148};
149
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]150struct udma_dev {
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]151 struct udevice *dev;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]152 void __iomem *mmrs[MMR_LAST];
153
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]154 struct udma_tisci_rm tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]155 struct k3_nav_ringacc *ringacc;
156
157 u32 features;
158
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]159 int bchan_cnt;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]160 int tchan_cnt;
161 int echan_cnt;
162 int rchan_cnt;
163 int rflow_cnt;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]164 int tflow_cnt;
165 unsigned long *bchan_map;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]166 unsigned long *tchan_map;
167 unsigned long *rchan_map;
168 unsigned long *rflow_map;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]169 unsigned long *rflow_map_reserved;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]170 unsigned long *tflow_map;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]171
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]172 struct udma_bchan *bchans;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]173 struct udma_tchan *tchans;
174 struct udma_rchan *rchans;
175 struct udma_rflow *rflows;
176
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]177 struct udma_match_data *match_data;
178
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]179 struct udma_chan *channels;
180 u32 psil_base;
181
182 u32 ch_count;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]183};
184
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]185struct udma_chan_config {
186 u32 psd_size; /* size of Protocol Specific Data */
187 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
188 u32 hdesc_size; /* Size of a packet descriptor in packet mode */
189 int remote_thread_id;
190 u32 atype;
191 u32 src_thread;
192 u32 dst_thread;
193 enum psil_endpoint_type ep_type;
194 enum udma_tp_level channel_tpl; /* Channel Throughput Level */
195
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]196 /* PKTDMA mapped channel */
197 int mapped_channel_id;
198 /* PKTDMA default tflow or rflow for mapped channel */
199 int default_flow_id;
200
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]201 enum dma_direction dir;
202
203 unsigned int pkt_mode:1; /* TR or packet */
204 unsigned int needs_epib:1; /* EPIB is needed for the communication or not */
205 unsigned int enable_acc32:1;
206 unsigned int enable_burst:1;
207 unsigned int notdpkt:1; /* Suppress sending TDC packet */
208};
209
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]210struct udma_chan {
211 struct udma_dev *ud;
212 char name[20];
213
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]214 struct udma_bchan *bchan;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]215 struct udma_tchan *tchan;
216 struct udma_rchan *rchan;
217 struct udma_rflow *rflow;
218
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]219 struct ti_udma_drv_chan_cfg_data cfg_data;
220
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]221 u32 bcnt; /* number of bytes completed since the start of the channel */
222
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]223 struct udma_chan_config config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]224
225 u32 id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]226
227 struct cppi5_host_desc_t *desc_tx;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]228 bool in_use;
229 void *desc_rx;
230 u32 num_rx_bufs;
231 u32 desc_rx_cur;
232
233};
234
235#define UDMA_CH_1000(ch) (ch * 0x1000)
236#define UDMA_CH_100(ch) (ch * 0x100)
237#define UDMA_CH_40(ch) (ch * 0x40)
238
239#ifdef PKTBUFSRX
240#define UDMA_RX_DESC_NUM PKTBUFSRX
241#else
242#define UDMA_RX_DESC_NUM 4
243#endif
244
245/* Generic register access functions */
246static inline u32 udma_read(void __iomem *base, int reg)
247{
248 u32 v;
249
250 v = __raw_readl(base + reg);
251 pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, base + reg);
252 return v;
253}
254
255static inline void udma_write(void __iomem *base, int reg, u32 val)
256{
257 pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", val, base + reg);
258 __raw_writel(val, base + reg);
259}
260
261static inline void udma_update_bits(void __iomem *base, int reg,
262 u32 mask, u32 val)
263{
264 u32 tmp, orig;
265
266 orig = udma_read(base, reg);
267 tmp = orig & ~mask;
268 tmp |= (val & mask);
269
270 if (tmp != orig)
271 udma_write(base, reg, tmp);
272}
273
274/* TCHANRT */
275static inline u32 udma_tchanrt_read(struct udma_tchan *tchan, int reg)
276{
277 if (!tchan)
278 return 0;
279 return udma_read(tchan->reg_rt, reg);
280}
281
282static inline void udma_tchanrt_write(struct udma_tchan *tchan,
283 int reg, u32 val)
284{
285 if (!tchan)
286 return;
287 udma_write(tchan->reg_rt, reg, val);
288}
289
290/* RCHANRT */
291static inline u32 udma_rchanrt_read(struct udma_rchan *rchan, int reg)
292{
293 if (!rchan)
294 return 0;
295 return udma_read(rchan->reg_rt, reg);
296}
297
298static inline void udma_rchanrt_write(struct udma_rchan *rchan,
299 int reg, u32 val)
300{
301 if (!rchan)
302 return;
303 udma_write(rchan->reg_rt, reg, val);
304}
305
306static inline int udma_navss_psil_pair(struct udma_dev *ud, u32 src_thread,
307 u32 dst_thread)
308{
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]309 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
310
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]311 dst_thread |= UDMA_PSIL_DST_THREAD_ID_OFFSET;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]312
313 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
314 tisci_rm->tisci_navss_dev_id,
315 src_thread, dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]316}
317
318static inline int udma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
319 u32 dst_thread)
320{
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]321 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
322
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]323 dst_thread |= UDMA_PSIL_DST_THREAD_ID_OFFSET;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]324
325 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
326 tisci_rm->tisci_navss_dev_id,
327 src_thread, dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]328}
329
330static inline char *udma_get_dir_text(enum dma_direction dir)
331{
332 switch (dir) {
333 case DMA_DEV_TO_MEM:
334 return "DEV_TO_MEM";
335 case DMA_MEM_TO_DEV:
336 return "MEM_TO_DEV";
337 case DMA_MEM_TO_MEM:
338 return "MEM_TO_MEM";
339 case DMA_DEV_TO_DEV:
340 return "DEV_TO_DEV";
341 default:
342 break;
343 }
344
345 return "invalid";
346}
347
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]348#include "k3-udma-u-boot.c"
349
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]350static void udma_reset_uchan(struct udma_chan *uc)
351{
352 memset(&uc->config, 0, sizeof(uc->config));
353 uc->config.remote_thread_id = -1;
354 uc->config.mapped_channel_id = -1;
355 uc->config.default_flow_id = -1;
356}
357
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]358static inline bool udma_is_chan_running(struct udma_chan *uc)
359{
360 u32 trt_ctl = 0;
361 u32 rrt_ctl = 0;
362
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]363 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]364 case DMA_DEV_TO_MEM:
365 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
366 pr_debug("%s: rrt_ctl: 0x%08x (peer: 0x%08x)\n",
367 __func__, rrt_ctl,
368 udma_rchanrt_read(uc->rchan,
369 UDMA_RCHAN_RT_PEER_RT_EN_REG));
370 break;
371 case DMA_MEM_TO_DEV:
372 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
373 pr_debug("%s: trt_ctl: 0x%08x (peer: 0x%08x)\n",
374 __func__, trt_ctl,
375 udma_tchanrt_read(uc->tchan,
376 UDMA_TCHAN_RT_PEER_RT_EN_REG));
377 break;
378 case DMA_MEM_TO_MEM:
379 trt_ctl = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
380 rrt_ctl = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
381 break;
382 default:
383 break;
384 }
385
386 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
387 return true;
388
389 return false;
390}
391
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]392static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
393{
394 struct k3_nav_ring *ring = NULL;
395 int ret = -ENOENT;
396
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]397 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]398 case DMA_DEV_TO_MEM:
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]399 ring = uc->rflow->r_ring;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]400 break;
401 case DMA_MEM_TO_DEV:
402 ring = uc->tchan->tc_ring;
403 break;
404 case DMA_MEM_TO_MEM:
405 ring = uc->tchan->tc_ring;
406 break;
407 default:
408 break;
409 }
410
411 if (ring && k3_nav_ringacc_ring_get_occ(ring))
412 ret = k3_nav_ringacc_ring_pop(ring, addr);
413
414 return ret;
415}
416
417static void udma_reset_rings(struct udma_chan *uc)
418{
419 struct k3_nav_ring *ring1 = NULL;
420 struct k3_nav_ring *ring2 = NULL;
421
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]422 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]423 case DMA_DEV_TO_MEM:
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]424 ring1 = uc->rflow->fd_ring;
425 ring2 = uc->rflow->r_ring;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]426 break;
427 case DMA_MEM_TO_DEV:
428 ring1 = uc->tchan->t_ring;
429 ring2 = uc->tchan->tc_ring;
430 break;
431 case DMA_MEM_TO_MEM:
432 ring1 = uc->tchan->t_ring;
433 ring2 = uc->tchan->tc_ring;
434 break;
435 default:
436 break;
437 }
438
439 if (ring1)
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]440 k3_nav_ringacc_ring_reset_dma(ring1, k3_nav_ringacc_ring_get_occ(ring1));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]441 if (ring2)
442 k3_nav_ringacc_ring_reset(ring2);
443}
444
445static void udma_reset_counters(struct udma_chan *uc)
446{
447 u32 val;
448
449 if (uc->tchan) {
450 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG);
451 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_BCNT_REG, val);
452
453 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG);
454 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_SBCNT_REG, val);
455
456 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PCNT_REG);
457 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PCNT_REG, val);
458
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]459 if (!uc->bchan) {
460 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG);
461 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG, val);
462 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]463 }
464
465 if (uc->rchan) {
466 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_BCNT_REG);
467 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_BCNT_REG, val);
468
469 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG);
470 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_SBCNT_REG, val);
471
472 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PCNT_REG);
473 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PCNT_REG, val);
474
475 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG);
476 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_BCNT_REG, val);
477 }
478
479 uc->bcnt = 0;
480}
481
482static inline int udma_stop_hard(struct udma_chan *uc)
483{
484 pr_debug("%s: ENTER (chan%d)\n", __func__, uc->id);
485
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]486 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]487 case DMA_DEV_TO_MEM:
488 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, 0);
489 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
490 break;
491 case DMA_MEM_TO_DEV:
492 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
493 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG, 0);
494 break;
495 case DMA_MEM_TO_MEM:
496 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
497 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
498 break;
499 default:
500 return -EINVAL;
501 }
502
503 return 0;
504}
505
506static int udma_start(struct udma_chan *uc)
507{
508 /* Channel is already running, no need to proceed further */
509 if (udma_is_chan_running(uc))
510 goto out;
511
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]512 pr_debug("%s: chan:%d dir:%s\n",
513 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]514
515 /* Make sure that we clear the teardown bit, if it is set */
516 udma_stop_hard(uc);
517
518 /* Reset all counters */
519 udma_reset_counters(uc);
520
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]521 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]522 case DMA_DEV_TO_MEM:
523 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG,
524 UDMA_CHAN_RT_CTL_EN);
525
526 /* Enable remote */
527 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG,
528 UDMA_PEER_RT_EN_ENABLE);
529
530 pr_debug("%s(rx): RT_CTL:0x%08x PEER RT_ENABLE:0x%08x\n",
531 __func__,
532 udma_rchanrt_read(uc->rchan,
533 UDMA_RCHAN_RT_CTL_REG),
534 udma_rchanrt_read(uc->rchan,
535 UDMA_RCHAN_RT_PEER_RT_EN_REG));
536 break;
537 case DMA_MEM_TO_DEV:
538 /* Enable remote */
539 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG,
540 UDMA_PEER_RT_EN_ENABLE);
541
542 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
543 UDMA_CHAN_RT_CTL_EN);
544
545 pr_debug("%s(tx): RT_CTL:0x%08x PEER RT_ENABLE:0x%08x\n",
546 __func__,
Vignesh Raghavendrac2237992019-12-09 10:25:36 +0530[diff] [blame]547 udma_tchanrt_read(uc->tchan,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]548 UDMA_TCHAN_RT_CTL_REG),
Vignesh Raghavendrac2237992019-12-09 10:25:36 +0530[diff] [blame]549 udma_tchanrt_read(uc->tchan,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]550 UDMA_TCHAN_RT_PEER_RT_EN_REG));
551 break;
552 case DMA_MEM_TO_MEM:
553 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG,
554 UDMA_CHAN_RT_CTL_EN);
555 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
556 UDMA_CHAN_RT_CTL_EN);
557
558 break;
559 default:
560 return -EINVAL;
561 }
562
563 pr_debug("%s: DONE chan:%d\n", __func__, uc->id);
564out:
565 return 0;
566}
567
568static inline void udma_stop_mem2dev(struct udma_chan *uc, bool sync)
569{
570 int i = 0;
571 u32 val;
572
573 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG,
574 UDMA_CHAN_RT_CTL_EN |
575 UDMA_CHAN_RT_CTL_TDOWN);
576
577 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
578
579 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
580 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_CTL_REG);
581 udelay(1);
582 if (i > 1000) {
583 printf(" %s TIMEOUT !\n", __func__);
584 break;
585 }
586 i++;
587 }
588
589 val = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_RT_EN_REG);
590 if (val & UDMA_PEER_RT_EN_ENABLE)
591 printf("%s: peer not stopped TIMEOUT !\n", __func__);
592}
593
594static inline void udma_stop_dev2mem(struct udma_chan *uc, bool sync)
595{
596 int i = 0;
597 u32 val;
598
599 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG,
600 UDMA_PEER_RT_EN_ENABLE |
601 UDMA_PEER_RT_EN_TEARDOWN);
602
603 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
604
605 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
606 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_CTL_REG);
607 udelay(1);
608 if (i > 1000) {
609 printf("%s TIMEOUT !\n", __func__);
610 break;
611 }
612 i++;
613 }
614
615 val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG);
616 if (val & UDMA_PEER_RT_EN_ENABLE)
617 printf("%s: peer not stopped TIMEOUT !\n", __func__);
618}
619
620static inline int udma_stop(struct udma_chan *uc)
621{
622 pr_debug("%s: chan:%d dir:%s\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]623 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]624
625 udma_reset_counters(uc);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]626 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]627 case DMA_DEV_TO_MEM:
628 udma_stop_dev2mem(uc, true);
629 break;
630 case DMA_MEM_TO_DEV:
631 udma_stop_mem2dev(uc, true);
632 break;
633 case DMA_MEM_TO_MEM:
634 udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_CTL_REG, 0);
635 udma_tchanrt_write(uc->tchan, UDMA_TCHAN_RT_CTL_REG, 0);
636 break;
637 default:
638 return -EINVAL;
639 }
640
641 return 0;
642}
643
644static void udma_poll_completion(struct udma_chan *uc, dma_addr_t *paddr)
645{
646 int i = 1;
647
648 while (udma_pop_from_ring(uc, paddr)) {
649 udelay(1);
650 if (!(i % 1000000))
651 printf(".");
652 i++;
653 }
654}
655
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]656static struct udma_rflow *__udma_reserve_rflow(struct udma_dev *ud, int id)
657{
658 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
659
660 if (id >= 0) {
661 if (test_bit(id, ud->rflow_map)) {
662 dev_err(ud->dev, "rflow%d is in use\n", id);
663 return ERR_PTR(-ENOENT);
664 }
665 } else {
666 bitmap_or(tmp, ud->rflow_map, ud->rflow_map_reserved,
667 ud->rflow_cnt);
668
669 id = find_next_zero_bit(tmp, ud->rflow_cnt, ud->rchan_cnt);
670 if (id >= ud->rflow_cnt)
671 return ERR_PTR(-ENOENT);
672 }
673
674 __set_bit(id, ud->rflow_map);
675 return &ud->rflows[id];
676}
677
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]678#define UDMA_RESERVE_RESOURCE(res) \
679static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \
680 int id) \
681{ \
682 if (id >= 0) { \
683 if (test_bit(id, ud->res##_map)) { \
684 dev_err(ud->dev, "res##%d is in use\n", id); \
685 return ERR_PTR(-ENOENT); \
686 } \
687 } else { \
688 id = find_first_zero_bit(ud->res##_map, ud->res##_cnt); \
689 if (id == ud->res##_cnt) { \
690 return ERR_PTR(-ENOENT); \
691 } \
692 } \
693 \
694 __set_bit(id, ud->res##_map); \
695 return &ud->res##s[id]; \
696}
697
698UDMA_RESERVE_RESOURCE(tchan);
699UDMA_RESERVE_RESOURCE(rchan);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]700
701static int udma_get_tchan(struct udma_chan *uc)
702{
703 struct udma_dev *ud = uc->ud;
704
705 if (uc->tchan) {
706 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
707 uc->id, uc->tchan->id);
708 return 0;
709 }
710
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]711 uc->tchan = __udma_reserve_tchan(ud, uc->config.mapped_channel_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]712 if (IS_ERR(uc->tchan))
713 return PTR_ERR(uc->tchan);
714
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]715 if (ud->tflow_cnt) {
716 int tflow_id;
717
718 /* Only PKTDMA have support for tx flows */
719 if (uc->config.default_flow_id >= 0)
720 tflow_id = uc->config.default_flow_id;
721 else
722 tflow_id = uc->tchan->id;
723
724 if (test_bit(tflow_id, ud->tflow_map)) {
725 dev_err(ud->dev, "tflow%d is in use\n", tflow_id);
726 __clear_bit(uc->tchan->id, ud->tchan_map);
727 uc->tchan = NULL;
728 return -ENOENT;
729 }
730
731 uc->tchan->tflow_id = tflow_id;
732 __set_bit(tflow_id, ud->tflow_map);
733 } else {
734 uc->tchan->tflow_id = -1;
735 }
736
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]737 pr_debug("chan%d: got tchan%d\n", uc->id, uc->tchan->id);
738
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]739 return 0;
740}
741
742static int udma_get_rchan(struct udma_chan *uc)
743{
744 struct udma_dev *ud = uc->ud;
745
746 if (uc->rchan) {
747 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
748 uc->id, uc->rchan->id);
749 return 0;
750 }
751
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]752 uc->rchan = __udma_reserve_rchan(ud, uc->config.mapped_channel_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]753 if (IS_ERR(uc->rchan))
754 return PTR_ERR(uc->rchan);
755
756 pr_debug("chan%d: got rchan%d\n", uc->id, uc->rchan->id);
757
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]758 return 0;
759}
760
761static int udma_get_chan_pair(struct udma_chan *uc)
762{
763 struct udma_dev *ud = uc->ud;
764 int chan_id, end;
765
766 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
767 dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
768 uc->id, uc->tchan->id);
769 return 0;
770 }
771
772 if (uc->tchan) {
773 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
774 uc->id, uc->tchan->id);
775 return -EBUSY;
776 } else if (uc->rchan) {
777 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
778 uc->id, uc->rchan->id);
779 return -EBUSY;
780 }
781
782 /* Can be optimized, but let's have it like this for now */
783 end = min(ud->tchan_cnt, ud->rchan_cnt);
784 for (chan_id = 0; chan_id < end; chan_id++) {
785 if (!test_bit(chan_id, ud->tchan_map) &&
786 !test_bit(chan_id, ud->rchan_map))
787 break;
788 }
789
790 if (chan_id == end)
791 return -ENOENT;
792
793 __set_bit(chan_id, ud->tchan_map);
794 __set_bit(chan_id, ud->rchan_map);
795 uc->tchan = &ud->tchans[chan_id];
796 uc->rchan = &ud->rchans[chan_id];
797
798 pr_debug("chan%d: got t/rchan%d pair\n", uc->id, chan_id);
799
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]800 return 0;
801}
802
803static int udma_get_rflow(struct udma_chan *uc, int flow_id)
804{
805 struct udma_dev *ud = uc->ud;
806
807 if (uc->rflow) {
808 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
809 uc->id, uc->rflow->id);
810 return 0;
811 }
812
813 if (!uc->rchan)
814 dev_warn(ud->dev, "chan%d: does not have rchan??\n", uc->id);
815
816 uc->rflow = __udma_reserve_rflow(ud, flow_id);
817 if (IS_ERR(uc->rflow))
818 return PTR_ERR(uc->rflow);
819
820 pr_debug("chan%d: got rflow%d\n", uc->id, uc->rflow->id);
821 return 0;
822}
823
824static void udma_put_rchan(struct udma_chan *uc)
825{
826 struct udma_dev *ud = uc->ud;
827
828 if (uc->rchan) {
829 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
830 uc->rchan->id);
831 __clear_bit(uc->rchan->id, ud->rchan_map);
832 uc->rchan = NULL;
833 }
834}
835
836static void udma_put_tchan(struct udma_chan *uc)
837{
838 struct udma_dev *ud = uc->ud;
839
840 if (uc->tchan) {
841 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
842 uc->tchan->id);
843 __clear_bit(uc->tchan->id, ud->tchan_map);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]844 if (uc->tchan->tflow_id >= 0)
845 __clear_bit(uc->tchan->tflow_id, ud->tflow_map);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]846 uc->tchan = NULL;
847 }
848}
849
850static void udma_put_rflow(struct udma_chan *uc)
851{
852 struct udma_dev *ud = uc->ud;
853
854 if (uc->rflow) {
855 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
856 uc->rflow->id);
857 __clear_bit(uc->rflow->id, ud->rflow_map);
858 uc->rflow = NULL;
859 }
860}
861
862static void udma_free_tx_resources(struct udma_chan *uc)
863{
864 if (!uc->tchan)
865 return;
866
867 k3_nav_ringacc_ring_free(uc->tchan->t_ring);
868 k3_nav_ringacc_ring_free(uc->tchan->tc_ring);
869 uc->tchan->t_ring = NULL;
870 uc->tchan->tc_ring = NULL;
871
872 udma_put_tchan(uc);
873}
874
875static int udma_alloc_tx_resources(struct udma_chan *uc)
876{
877 struct k3_nav_ring_cfg ring_cfg;
878 struct udma_dev *ud = uc->ud;
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]879 struct udma_tchan *tchan;
880 int ring_idx, ret;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]881
882 ret = udma_get_tchan(uc);
883 if (ret)
884 return ret;
885
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]886 tchan = uc->tchan;
Udit Kumarf084e402024-02-21 19:53:44 +0530[diff] [blame]887 if (tchan->tflow_id > 0)
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]888 ring_idx = tchan->tflow_id;
889 else
Udit Kumarf084e402024-02-21 19:53:44 +0530[diff] [blame]890 ring_idx = tchan->id;
MD Danish Anwar25abdb32024-01-30 11:48:04 +0530[diff] [blame]891
892 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1,
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]893 &uc->tchan->t_ring,
894 &uc->tchan->tc_ring);
895 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]896 ret = -EBUSY;
897 goto err_tx_ring;
898 }
899
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]900 memset(&ring_cfg, 0, sizeof(ring_cfg));
901 ring_cfg.size = 16;
902 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
Vignesh Raghavendra0fe24d32019-12-09 10:25:37 +0530[diff] [blame]903 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]904
905 ret = k3_nav_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg);
906 ret |= k3_nav_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg);
907
908 if (ret)
909 goto err_ringcfg;
910
911 return 0;
912
913err_ringcfg:
914 k3_nav_ringacc_ring_free(uc->tchan->tc_ring);
915 uc->tchan->tc_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]916 k3_nav_ringacc_ring_free(uc->tchan->t_ring);
917 uc->tchan->t_ring = NULL;
918err_tx_ring:
919 udma_put_tchan(uc);
920
921 return ret;
922}
923
924static void udma_free_rx_resources(struct udma_chan *uc)
925{
926 if (!uc->rchan)
927 return;
928
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]929 if (uc->rflow) {
930 k3_nav_ringacc_ring_free(uc->rflow->fd_ring);
931 k3_nav_ringacc_ring_free(uc->rflow->r_ring);
932 uc->rflow->fd_ring = NULL;
933 uc->rflow->r_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]934
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]935 udma_put_rflow(uc);
936 }
937
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]938 udma_put_rchan(uc);
939}
940
941static int udma_alloc_rx_resources(struct udma_chan *uc)
942{
943 struct k3_nav_ring_cfg ring_cfg;
944 struct udma_dev *ud = uc->ud;
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]945 struct udma_rflow *rflow;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]946 int fd_ring_id;
947 int ret;
948
949 ret = udma_get_rchan(uc);
950 if (ret)
951 return ret;
952
953 /* For MEM_TO_MEM we don't need rflow or rings */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]954 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]955 return 0;
956
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]957 if (uc->config.default_flow_id >= 0)
958 ret = udma_get_rflow(uc, uc->config.default_flow_id);
959 else
960 ret = udma_get_rflow(uc, uc->rchan->id);
961
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]962 if (ret) {
963 ret = -EBUSY;
964 goto err_rflow;
965 }
966
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]967 rflow = uc->rflow;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]968 if (ud->tflow_cnt) {
969 fd_ring_id = ud->tflow_cnt + rflow->id;
970 } else {
971 fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt +
972 uc->rchan->id;
973 }
974
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]975 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
976 &rflow->fd_ring, &rflow->r_ring);
977 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]978 ret = -EBUSY;
979 goto err_rx_ring;
980 }
981
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]982 memset(&ring_cfg, 0, sizeof(ring_cfg));
983 ring_cfg.size = 16;
984 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
Vignesh Raghavendra0fe24d32019-12-09 10:25:37 +0530[diff] [blame]985 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]986
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]987 ret = k3_nav_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
988 ret |= k3_nav_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]989 if (ret)
990 goto err_ringcfg;
991
992 return 0;
993
994err_ringcfg:
Vignesh Raghavendrad7c3eb02020-07-06 13:26:27 +0530[diff] [blame]995 k3_nav_ringacc_ring_free(rflow->r_ring);
996 rflow->r_ring = NULL;
997 k3_nav_ringacc_ring_free(rflow->fd_ring);
998 rflow->fd_ring = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]999err_rx_ring:
1000 udma_put_rflow(uc);
1001err_rflow:
1002 udma_put_rchan(uc);
1003
1004 return ret;
1005}
1006
1007static int udma_alloc_tchan_sci_req(struct udma_chan *uc)
1008{
1009 struct udma_dev *ud = uc->ud;
1010 int tc_ring = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1011 struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1012 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1013 u32 mode;
1014 int ret;
1015
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1016 if (uc->config.pkt_mode)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1017 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1018 else
1019 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1020
1021 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
1022 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
1023 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1024 req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1025 req.index = uc->tchan->id;
1026 req.tx_chan_type = mode;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1027 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1028 req.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1029 else
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1030 req.tx_fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1031 uc->config.psd_size,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1032 0) >> 2;
1033 req.txcq_qnum = tc_ring;
1034
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1035 ret = tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1036 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1037 dev_err(ud->dev, "tisci tx alloc failed %d\n", ret);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1038 return ret;
1039 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1040
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1041 /*
1042 * Above TI SCI call handles firewall configuration, cfg
1043 * register configuration still has to be done locally in
1044 * absence of RM services.
1045 */
1046 if (IS_ENABLED(CONFIG_K3_DM_FW))
1047 udma_alloc_tchan_raw(uc);
1048
1049 return 0;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1050}
1051
1052static int udma_alloc_rchan_sci_req(struct udma_chan *uc)
1053{
1054 struct udma_dev *ud = uc->ud;
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]1055 int fd_ring = k3_nav_ringacc_get_ring_id(uc->rflow->fd_ring);
1056 int rx_ring = k3_nav_ringacc_get_ring_id(uc->rflow->r_ring);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1057 int tc_ring = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1058 struct ti_sci_msg_rm_udmap_rx_ch_cfg req = { 0 };
1059 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1060 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1061 u32 mode;
1062 int ret;
1063
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1064 if (uc->config.pkt_mode)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1065 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1066 else
1067 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1068
1069 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
1070 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1071 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1072 req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1073 req.index = uc->rchan->id;
1074 req.rx_chan_type = mode;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1075 if (uc->config.dir == DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1076 req.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1077 req.rxcq_qnum = tc_ring;
1078 } else {
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1079 req.rx_fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1080 uc->config.psd_size,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1081 0) >> 2;
1082 req.rxcq_qnum = rx_ring;
1083 }
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1084 if (ud->match_data->type == DMA_TYPE_UDMA &&
1085 uc->rflow->id != uc->rchan->id &&
1086 uc->config.dir != DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1087 req.flowid_start = uc->rflow->id;
1088 req.flowid_cnt = 1;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1089 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
1090 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1091 }
1092
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1093 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1094 if (ret) {
1095 dev_err(ud->dev, "tisci rx %u cfg failed %d\n",
1096 uc->rchan->id, ret);
1097 return ret;
1098 }
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1099 if (uc->config.dir == DMA_MEM_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1100 return ret;
1101
1102 flow_req.valid_params =
1103 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
1104 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
1105 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
1106 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
1107 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1108 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
1109 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
1110 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
1111 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
1112 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1113 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1114 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1115 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID |
1116 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PS_LOCATION_VALID;
1117
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1118 flow_req.nav_id = tisci_rm->tisci_dev_id;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1119 flow_req.flow_index = uc->rflow->id;
1120
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1121 if (uc->config.needs_epib)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1122 flow_req.rx_einfo_present = 1;
1123 else
1124 flow_req.rx_einfo_present = 0;
1125
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1126 if (uc->config.psd_size)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1127 flow_req.rx_psinfo_present = 1;
1128 else
1129 flow_req.rx_psinfo_present = 0;
1130
1131 flow_req.rx_error_handling = 0;
1132 flow_req.rx_desc_type = 0;
1133 flow_req.rx_dest_qnum = rx_ring;
1134 flow_req.rx_src_tag_hi_sel = 2;
1135 flow_req.rx_src_tag_lo_sel = 4;
1136 flow_req.rx_dest_tag_hi_sel = 5;
1137 flow_req.rx_dest_tag_lo_sel = 4;
1138 flow_req.rx_fdq0_sz0_qnum = fd_ring;
1139 flow_req.rx_fdq1_qnum = fd_ring;
1140 flow_req.rx_fdq2_qnum = fd_ring;
1141 flow_req.rx_fdq3_qnum = fd_ring;
1142 flow_req.rx_ps_location = 0;
1143
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1144 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci,
1145 &flow_req);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1146 if (ret) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1147 dev_err(ud->dev, "tisci rx %u flow %u cfg failed %d\n",
1148 uc->rchan->id, uc->rflow->id, ret);
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1149 return ret;
1150 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1151
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1152 /*
1153 * Above TI SCI call handles firewall configuration, cfg
1154 * register configuration still has to be done locally in
1155 * absence of RM services.
1156 */
1157 if (IS_ENABLED(CONFIG_K3_DM_FW))
1158 udma_alloc_rchan_raw(uc);
1159
1160 return 0;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1161}
1162
1163static int udma_alloc_chan_resources(struct udma_chan *uc)
1164{
1165 struct udma_dev *ud = uc->ud;
1166 int ret;
1167
1168 pr_debug("%s: chan:%d as %s\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1169 __func__, uc->id, udma_get_dir_text(uc->config.dir));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1170
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1171 switch (uc->config.dir) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1172 case DMA_MEM_TO_MEM:
1173 /* Non synchronized - mem to mem type of transfer */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1174 uc->config.pkt_mode = false;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1175 ret = udma_get_chan_pair(uc);
1176 if (ret)
1177 return ret;
1178
1179 ret = udma_alloc_tx_resources(uc);
1180 if (ret)
1181 goto err_free_res;
1182
1183 ret = udma_alloc_rx_resources(uc);
1184 if (ret)
1185 goto err_free_res;
1186
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1187 uc->config.src_thread = ud->psil_base + uc->tchan->id;
1188 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1189 break;
1190 case DMA_MEM_TO_DEV:
1191 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
1192 ret = udma_alloc_tx_resources(uc);
1193 if (ret)
1194 goto err_free_res;
1195
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1196 uc->config.src_thread = ud->psil_base + uc->tchan->id;
1197 uc->config.dst_thread = uc->config.remote_thread_id;
1198 uc->config.dst_thread |= 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1199
1200 break;
1201 case DMA_DEV_TO_MEM:
1202 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
1203 ret = udma_alloc_rx_resources(uc);
1204 if (ret)
1205 goto err_free_res;
1206
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1207 uc->config.src_thread = uc->config.remote_thread_id;
1208 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | 0x8000;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1209
1210 break;
1211 default:
1212 /* Can not happen */
1213 pr_debug("%s: chan:%d invalid direction (%u)\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1214 __func__, uc->id, uc->config.dir);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1215 return -EINVAL;
1216 }
1217
1218 /* We have channel indexes and rings */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1219 if (uc->config.dir == DMA_MEM_TO_MEM) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1220 ret = udma_alloc_tchan_sci_req(uc);
1221 if (ret)
1222 goto err_free_res;
1223
1224 ret = udma_alloc_rchan_sci_req(uc);
1225 if (ret)
1226 goto err_free_res;
1227 } else {
1228 /* Slave transfer */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1229 if (uc->config.dir == DMA_MEM_TO_DEV) {
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1230 ret = udma_alloc_tchan_sci_req(uc);
1231 if (ret)
1232 goto err_free_res;
1233 } else {
1234 ret = udma_alloc_rchan_sci_req(uc);
1235 if (ret)
1236 goto err_free_res;
1237 }
1238 }
1239
Peter Ujfalusid15f8652019-04-25 12:08:15 +0530[diff] [blame]1240 if (udma_is_chan_running(uc)) {
1241 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
1242 udma_stop(uc);
1243 if (udma_is_chan_running(uc)) {
1244 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
1245 goto err_free_res;
1246 }
1247 }
1248
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1249 /* PSI-L pairing */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1250 ret = udma_navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1251 if (ret) {
1252 dev_err(ud->dev, "k3_nav_psil_request_link fail\n");
1253 goto err_free_res;
1254 }
1255
1256 return 0;
1257
1258err_free_res:
1259 udma_free_tx_resources(uc);
1260 udma_free_rx_resources(uc);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1261 uc->config.remote_thread_id = -1;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1262 return ret;
1263}
1264
1265static void udma_free_chan_resources(struct udma_chan *uc)
1266{
Vignesh Raghavendrabe7bdcc2020-09-17 20:11:22 +0530[diff] [blame]1267 /* Hard reset UDMA channel */
1268 udma_stop_hard(uc);
1269 udma_reset_counters(uc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1270
1271 /* Release PSI-L pairing */
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1272 udma_navss_psil_unpair(uc->ud, uc->config.src_thread, uc->config.dst_thread);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1273
1274 /* Reset the rings for a new start */
1275 udma_reset_rings(uc);
1276 udma_free_tx_resources(uc);
1277 udma_free_rx_resources(uc);
1278
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1279 uc->config.remote_thread_id = -1;
1280 uc->config.dir = DMA_MEM_TO_MEM;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1281}
1282
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1283static const char * const range_names[] = {
1284 [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan",
1285 [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan",
1286 [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan",
1287 [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow",
1288 [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow",
1289};
1290
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1291static int udma_get_mmrs(struct udevice *dev)
1292{
1293 struct udma_dev *ud = dev_get_priv(dev);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1294 u32 cap2, cap3, cap4;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1295 int i;
1296
Matthias Schiffer47331932023-09-27 15:33:34 +0200[diff] [blame]1297 ud->mmrs[MMR_GCFG] = dev_read_addr_name_ptr(dev, mmr_names[MMR_GCFG]);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1298 if (!ud->mmrs[MMR_GCFG])
1299 return -EINVAL;
1300
1301 cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);
1302 cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
1303
1304 switch (ud->match_data->type) {
1305 case DMA_TYPE_UDMA:
1306 ud->rflow_cnt = cap3 & 0x3fff;
1307 ud->tchan_cnt = cap2 & 0x1ff;
1308 ud->echan_cnt = (cap2 >> 9) & 0x1ff;
1309 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1310 break;
1311 case DMA_TYPE_BCDMA:
1312 ud->bchan_cnt = cap2 & 0x1ff;
1313 ud->tchan_cnt = (cap2 >> 9) & 0x1ff;
1314 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1315 break;
1316 case DMA_TYPE_PKTDMA:
1317 cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30);
1318 ud->tchan_cnt = cap2 & 0x1ff;
1319 ud->rchan_cnt = (cap2 >> 18) & 0x1ff;
1320 ud->rflow_cnt = cap3 & 0x3fff;
1321 ud->tflow_cnt = cap4 & 0x3fff;
1322 break;
1323 default:
1324 return -EINVAL;
1325 }
1326
1327 for (i = 1; i < MMR_LAST; i++) {
1328 if (i == MMR_BCHANRT && ud->bchan_cnt == 0)
1329 continue;
1330 if (i == MMR_TCHANRT && ud->tchan_cnt == 0)
1331 continue;
1332 if (i == MMR_RCHANRT && ud->rchan_cnt == 0)
1333 continue;
1334
Matthias Schiffer47331932023-09-27 15:33:34 +0200[diff] [blame]1335 ud->mmrs[i] = dev_read_addr_name_ptr(dev, mmr_names[i]);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1336 if (!ud->mmrs[i])
1337 return -EINVAL;
1338 }
1339
1340 return 0;
1341}
1342
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1343static int udma_setup_resources(struct udma_dev *ud)
1344{
1345 struct udevice *dev = ud->dev;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1346 int i;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1347 struct ti_sci_resource_desc *rm_desc;
1348 struct ti_sci_resource *rm_res;
1349 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1350
1351 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1352 sizeof(unsigned long), GFP_KERNEL);
1353 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1354 GFP_KERNEL);
1355 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1356 sizeof(unsigned long), GFP_KERNEL);
1357 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1358 GFP_KERNEL);
1359 ud->rflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
1360 sizeof(unsigned long), GFP_KERNEL);
1361 ud->rflow_map_reserved = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
1362 sizeof(unsigned long),
1363 GFP_KERNEL);
1364 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
1365 GFP_KERNEL);
1366
1367 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_map ||
1368 !ud->rflow_map_reserved || !ud->tchans || !ud->rchans ||
1369 !ud->rflows)
1370 return -ENOMEM;
1371
1372 /*
1373 * RX flows with the same Ids as RX channels are reserved to be used
1374 * as default flows if remote HW can't generate flow_ids. Those
1375 * RX flows can be requested only explicitly by id.
1376 */
1377 bitmap_set(ud->rflow_map_reserved, 0, ud->rchan_cnt);
1378
1379 /* Get resource ranges from tisci */
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1380 for (i = 0; i < RM_RANGE_LAST; i++) {
1381 if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW)
1382 continue;
1383
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1384 tisci_rm->rm_ranges[i] =
1385 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1386 tisci_rm->tisci_dev_id,
1387 (char *)range_names[i]);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1388 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1389
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1390 /* tchan ranges */
1391 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1392 if (IS_ERR(rm_res)) {
1393 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1394 } else {
1395 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1396 for (i = 0; i < rm_res->sets; i++) {
1397 rm_desc = &rm_res->desc[i];
1398 bitmap_clear(ud->tchan_map, rm_desc->start,
1399 rm_desc->num);
1400 }
1401 }
1402
1403 /* rchan and matching default flow ranges */
1404 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1405 if (IS_ERR(rm_res)) {
1406 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1407 bitmap_zero(ud->rflow_map, ud->rchan_cnt);
1408 } else {
1409 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1410 bitmap_fill(ud->rflow_map, ud->rchan_cnt);
1411 for (i = 0; i < rm_res->sets; i++) {
1412 rm_desc = &rm_res->desc[i];
1413 bitmap_clear(ud->rchan_map, rm_desc->start,
1414 rm_desc->num);
1415 bitmap_clear(ud->rflow_map, rm_desc->start,
1416 rm_desc->num);
1417 }
1418 }
1419
1420 /* GP rflow ranges */
1421 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
1422 if (IS_ERR(rm_res)) {
1423 bitmap_clear(ud->rflow_map, ud->rchan_cnt,
1424 ud->rflow_cnt - ud->rchan_cnt);
1425 } else {
1426 bitmap_set(ud->rflow_map, ud->rchan_cnt,
1427 ud->rflow_cnt - ud->rchan_cnt);
1428 for (i = 0; i < rm_res->sets; i++) {
1429 rm_desc = &rm_res->desc[i];
1430 bitmap_clear(ud->rflow_map, rm_desc->start,
1431 rm_desc->num);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1432 }
1433 }
1434
1435 return 0;
1436}
1437
1438static int bcdma_setup_resources(struct udma_dev *ud)
1439{
1440 int i;
1441 struct udevice *dev = ud->dev;
1442 struct ti_sci_resource_desc *rm_desc;
1443 struct ti_sci_resource *rm_res;
1444 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1445
1446 ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt),
1447 sizeof(unsigned long), GFP_KERNEL);
1448 ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans),
1449 GFP_KERNEL);
1450 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1451 sizeof(unsigned long), GFP_KERNEL);
1452 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1453 GFP_KERNEL);
1454 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1455 sizeof(unsigned long), GFP_KERNEL);
1456 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1457 GFP_KERNEL);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1458 ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows),
1459 GFP_KERNEL);
1460
1461 if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map ||
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1462 !ud->bchans || !ud->tchans || !ud->rchans ||
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1463 !ud->rflows)
1464 return -ENOMEM;
1465
1466 /* Get resource ranges from tisci */
1467 for (i = 0; i < RM_RANGE_LAST; i++) {
1468 if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW)
1469 continue;
1470
1471 tisci_rm->rm_ranges[i] =
1472 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1473 tisci_rm->tisci_dev_id,
1474 (char *)range_names[i]);
1475 }
1476
1477 /* bchan ranges */
1478 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
1479 if (IS_ERR(rm_res)) {
1480 bitmap_zero(ud->bchan_map, ud->bchan_cnt);
1481 } else {
1482 bitmap_fill(ud->bchan_map, ud->bchan_cnt);
1483 for (i = 0; i < rm_res->sets; i++) {
1484 rm_desc = &rm_res->desc[i];
1485 bitmap_clear(ud->bchan_map, rm_desc->start,
1486 rm_desc->num);
1487 dev_dbg(dev, "ti-sci-res: bchan: %d:%d\n",
1488 rm_desc->start, rm_desc->num);
1489 }
1490 }
1491
1492 /* tchan ranges */
1493 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1494 if (IS_ERR(rm_res)) {
1495 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1496 } else {
1497 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1498 for (i = 0; i < rm_res->sets; i++) {
1499 rm_desc = &rm_res->desc[i];
1500 bitmap_clear(ud->tchan_map, rm_desc->start,
1501 rm_desc->num);
1502 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
1503 rm_desc->start, rm_desc->num);
1504 }
1505 }
1506
1507 /* rchan ranges */
1508 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1509 if (IS_ERR(rm_res)) {
1510 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1511 } else {
1512 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1513 for (i = 0; i < rm_res->sets; i++) {
1514 rm_desc = &rm_res->desc[i];
1515 bitmap_clear(ud->rchan_map, rm_desc->start,
1516 rm_desc->num);
1517 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
1518 rm_desc->start, rm_desc->num);
1519 }
1520 }
1521
1522 return 0;
1523}
1524
1525static int pktdma_setup_resources(struct udma_dev *ud)
1526{
1527 int i;
1528 struct udevice *dev = ud->dev;
1529 struct ti_sci_resource *rm_res;
1530 struct ti_sci_resource_desc *rm_desc;
1531 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1532
1533 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
1534 sizeof(unsigned long), GFP_KERNEL);
1535 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
1536 GFP_KERNEL);
1537 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
1538 sizeof(unsigned long), GFP_KERNEL);
1539 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
1540 GFP_KERNEL);
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1541 ud->rflow_map = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
1542 sizeof(unsigned long),
1543 GFP_KERNEL);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1544 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
1545 GFP_KERNEL);
1546 ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt),
1547 sizeof(unsigned long), GFP_KERNEL);
1548
1549 if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans ||
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1550 !ud->rchans || !ud->rflows || !ud->rflow_map)
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1551 return -ENOMEM;
1552
1553 /* Get resource ranges from tisci */
1554 for (i = 0; i < RM_RANGE_LAST; i++) {
1555 if (i == RM_RANGE_BCHAN)
1556 continue;
1557
1558 tisci_rm->rm_ranges[i] =
1559 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
1560 tisci_rm->tisci_dev_id,
1561 (char *)range_names[i]);
1562 }
1563
1564 /* tchan ranges */
1565 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
1566 if (IS_ERR(rm_res)) {
1567 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
1568 } else {
1569 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
1570 for (i = 0; i < rm_res->sets; i++) {
1571 rm_desc = &rm_res->desc[i];
1572 bitmap_clear(ud->tchan_map, rm_desc->start,
1573 rm_desc->num);
1574 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
1575 rm_desc->start, rm_desc->num);
1576 }
1577 }
1578
1579 /* rchan ranges */
1580 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
1581 if (IS_ERR(rm_res)) {
1582 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
1583 } else {
1584 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
1585 for (i = 0; i < rm_res->sets; i++) {
1586 rm_desc = &rm_res->desc[i];
1587 bitmap_clear(ud->rchan_map, rm_desc->start,
1588 rm_desc->num);
1589 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
1590 rm_desc->start, rm_desc->num);
1591 }
1592 }
1593
1594 /* rflow ranges */
1595 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
1596 if (IS_ERR(rm_res)) {
1597 /* all rflows are assigned exclusively to Linux */
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1598 bitmap_zero(ud->rflow_map, ud->rflow_cnt);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1599 } else {
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1600 bitmap_fill(ud->rflow_map, ud->rflow_cnt);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1601 for (i = 0; i < rm_res->sets; i++) {
1602 rm_desc = &rm_res->desc[i];
Vignesh Raghavendra8ade6b02021-12-23 19:27:30 +0530[diff] [blame]1603 bitmap_clear(ud->rflow_map, rm_desc->start,
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1604 rm_desc->num);
1605 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n",
1606 rm_desc->start, rm_desc->num);
1607 }
1608 }
1609
1610 /* tflow ranges */
1611 rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
1612 if (IS_ERR(rm_res)) {
1613 /* all tflows are assigned exclusively to Linux */
1614 bitmap_zero(ud->tflow_map, ud->tflow_cnt);
1615 } else {
1616 bitmap_fill(ud->tflow_map, ud->tflow_cnt);
1617 for (i = 0; i < rm_res->sets; i++) {
1618 rm_desc = &rm_res->desc[i];
1619 bitmap_clear(ud->tflow_map, rm_desc->start,
1620 rm_desc->num);
1621 dev_dbg(dev, "ti-sci-res: tflow: %d:%d\n",
1622 rm_desc->start, rm_desc->num);
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1623 }
1624 }
1625
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1626 return 0;
1627}
1628
1629static int setup_resources(struct udma_dev *ud)
1630{
1631 struct udevice *dev = ud->dev;
1632 int ch_count, ret;
1633
1634 switch (ud->match_data->type) {
1635 case DMA_TYPE_UDMA:
1636 ret = udma_setup_resources(ud);
1637 break;
1638 case DMA_TYPE_BCDMA:
1639 ret = bcdma_setup_resources(ud);
1640 break;
1641 case DMA_TYPE_PKTDMA:
1642 ret = pktdma_setup_resources(ud);
1643 break;
1644 default:
1645 return -EINVAL;
1646 }
1647
1648 if (ret)
1649 return ret;
1650
1651 ch_count = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt;
1652 if (ud->bchan_cnt)
1653 ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt);
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1654 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
1655 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
1656 if (!ch_count)
1657 return -ENODEV;
1658
1659 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
1660 GFP_KERNEL);
1661 if (!ud->channels)
1662 return -ENOMEM;
1663
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1664 switch (ud->match_data->type) {
1665 case DMA_TYPE_UDMA:
1666 dev_dbg(dev,
1667 "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
1668 ch_count,
1669 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1670 ud->tchan_cnt),
1671 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1672 ud->rchan_cnt),
1673 ud->rflow_cnt - bitmap_weight(ud->rflow_map,
1674 ud->rflow_cnt));
1675 break;
1676 case DMA_TYPE_BCDMA:
1677 dev_dbg(dev,
1678 "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n",
1679 ch_count,
1680 ud->bchan_cnt - bitmap_weight(ud->bchan_map,
1681 ud->bchan_cnt),
1682 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1683 ud->tchan_cnt),
1684 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1685 ud->rchan_cnt));
1686 break;
1687 case DMA_TYPE_PKTDMA:
1688 dev_dbg(dev,
1689 "Channels: %d (tchan: %u, rchan: %u)\n",
1690 ch_count,
1691 ud->tchan_cnt - bitmap_weight(ud->tchan_map,
1692 ud->tchan_cnt),
1693 ud->rchan_cnt - bitmap_weight(ud->rchan_map,
1694 ud->rchan_cnt));
1695 break;
1696 default:
1697 break;
1698 }
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1699
1700 return ch_count;
1701}
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1702
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1703static int udma_probe(struct udevice *dev)
1704{
1705 struct dma_dev_priv *uc_priv = dev_get_uclass_priv(dev);
1706 struct udma_dev *ud = dev_get_priv(dev);
1707 int i, ret;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1708 struct udevice *tmp;
1709 struct udevice *tisci_dev = NULL;
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1710 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1711 ofnode navss_ofnode = ofnode_get_parent(dev_ofnode(dev));
1712
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1713
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1714 ud->match_data = (void *)dev_get_driver_data(dev);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1715 ret = udma_get_mmrs(dev);
1716 if (ret)
1717 return ret;
1718
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]1719 ud->psil_base = ud->match_data->psil_base;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1720
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1721 ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, dev,
1722 "ti,sci", &tisci_dev);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1723 if (ret) {
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1724 debug("Failed to get TISCI phandle (%d)\n", ret);
1725 tisci_rm->tisci = NULL;
1726 return -EINVAL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1727 }
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1728 tisci_rm->tisci = (struct ti_sci_handle *)
1729 (ti_sci_get_handle_from_sysfw(tisci_dev));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1730
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1731 tisci_rm->tisci_dev_id = -1;
1732 ret = dev_read_u32(dev, "ti,sci-dev-id", &tisci_rm->tisci_dev_id);
1733 if (ret) {
1734 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
1735 return ret;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1736 }
1737
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1738 tisci_rm->tisci_navss_dev_id = -1;
1739 ret = ofnode_read_u32(navss_ofnode, "ti,sci-dev-id",
1740 &tisci_rm->tisci_navss_dev_id);
1741 if (ret) {
1742 dev_err(dev, "navss sci-dev-id read failure %d\n", ret);
1743 return ret;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1744 }
1745
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1746 tisci_rm->tisci_udmap_ops = &tisci_rm->tisci->ops.rm_udmap_ops;
1747 tisci_rm->tisci_psil_ops = &tisci_rm->tisci->ops.rm_psil_ops;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1748
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1749 if (ud->match_data->type == DMA_TYPE_UDMA) {
1750 ret = uclass_get_device_by_phandle(UCLASS_MISC, dev,
1751 "ti,ringacc", &tmp);
1752 ud->ringacc = dev_get_priv(tmp);
1753 } else {
1754 struct k3_ringacc_init_data ring_init_data;
1755
1756 ring_init_data.tisci = ud->tisci_rm.tisci;
1757 ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id;
1758 if (ud->match_data->type == DMA_TYPE_BCDMA) {
1759 ring_init_data.num_rings = ud->bchan_cnt +
1760 ud->tchan_cnt +
1761 ud->rchan_cnt;
1762 } else {
1763 ring_init_data.num_rings = ud->rflow_cnt +
1764 ud->tflow_cnt;
1765 }
1766
1767 ud->ringacc = k3_ringacc_dmarings_init(dev, &ring_init_data);
1768 }
1769 if (IS_ERR(ud->ringacc))
1770 return PTR_ERR(ud->ringacc);
1771
Vignesh Raghavendrac4106862019-12-09 10:25:32 +0530[diff] [blame]1772 ud->dev = dev;
Siddharth Vadapalli679bc0f2024-02-20 15:34:51 +0530[diff] [blame]1773 ret = setup_resources(ud);
1774 if (ret < 0)
1775 return ret;
1776
1777 ud->ch_count = ret;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1778
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1779 for (i = 0; i < ud->bchan_cnt; i++) {
1780 struct udma_bchan *bchan = &ud->bchans[i];
1781
1782 bchan->id = i;
1783 bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000;
1784 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1785
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1786 for (i = 0; i < ud->tchan_cnt; i++) {
1787 struct udma_tchan *tchan = &ud->tchans[i];
1788
1789 tchan->id = i;
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1790 tchan->reg_chan = ud->mmrs[MMR_TCHAN] + UDMA_CH_100(i);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1791 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + UDMA_CH_1000(i);
1792 }
1793
1794 for (i = 0; i < ud->rchan_cnt; i++) {
1795 struct udma_rchan *rchan = &ud->rchans[i];
1796
1797 rchan->id = i;
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1798 rchan->reg_chan = ud->mmrs[MMR_RCHAN] + UDMA_CH_100(i);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1799 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + UDMA_CH_1000(i);
1800 }
1801
1802 for (i = 0; i < ud->rflow_cnt; i++) {
1803 struct udma_rflow *rflow = &ud->rflows[i];
1804
1805 rflow->id = i;
Vignesh Raghavendra27e72502021-06-07 19:47:53 +0530[diff] [blame]1806 rflow->reg_rflow = ud->mmrs[MMR_RFLOW] + UDMA_CH_40(i);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1807 }
1808
1809 for (i = 0; i < ud->ch_count; i++) {
1810 struct udma_chan *uc = &ud->channels[i];
1811
1812 uc->ud = ud;
1813 uc->id = i;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1814 uc->config.remote_thread_id = -1;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1815 uc->bchan = NULL;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1816 uc->tchan = NULL;
1817 uc->rchan = NULL;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1818 uc->config.mapped_channel_id = -1;
1819 uc->config.default_flow_id = -1;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]1820 uc->config.dir = DMA_MEM_TO_MEM;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1821 sprintf(uc->name, "UDMA chan%d\n", i);
1822 if (!i)
1823 uc->in_use = true;
1824 }
1825
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1826 pr_debug("%s(rev: 0x%08x) CAP0-3: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
1827 dev->name,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1828 udma_read(ud->mmrs[MMR_GCFG], 0),
1829 udma_read(ud->mmrs[MMR_GCFG], 0x20),
1830 udma_read(ud->mmrs[MMR_GCFG], 0x24),
1831 udma_read(ud->mmrs[MMR_GCFG], 0x28),
1832 udma_read(ud->mmrs[MMR_GCFG], 0x2c));
1833
1834 uc_priv->supported = DMA_SUPPORTS_MEM_TO_MEM | DMA_SUPPORTS_MEM_TO_DEV;
1835
Siddharth Vadapalli679bc0f2024-02-20 15:34:51 +0530[diff] [blame]1836 return 0;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1837}
1838
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]1839static int udma_push_to_ring(struct k3_nav_ring *ring, void *elem)
1840{
1841 u64 addr = 0;
1842
1843 memcpy(&addr, &elem, sizeof(elem));
1844 return k3_nav_ringacc_ring_push(ring, &addr);
1845}
1846
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1847static int *udma_prep_dma_memcpy(struct udma_chan *uc, dma_addr_t dest,
1848 dma_addr_t src, size_t len)
1849{
1850 u32 tc_ring_id = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
1851 struct cppi5_tr_type15_t *tr_req;
1852 int num_tr;
1853 size_t tr_size = sizeof(struct cppi5_tr_type15_t);
1854 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
1855 unsigned long dummy;
1856 void *tr_desc;
1857 size_t desc_size;
1858
1859 if (len < SZ_64K) {
1860 num_tr = 1;
1861 tr0_cnt0 = len;
1862 tr0_cnt1 = 1;
1863 } else {
1864 unsigned long align_to = __ffs(src | dest);
1865
1866 if (align_to > 3)
1867 align_to = 3;
1868 /*
1869 * Keep simple: tr0: SZ_64K-alignment blocks,
1870 * tr1: the remaining
1871 */
1872 num_tr = 2;
1873 tr0_cnt0 = (SZ_64K - BIT(align_to));
1874 if (len / tr0_cnt0 >= SZ_64K) {
1875 dev_err(uc->ud->dev, "size %zu is not supported\n",
1876 len);
1877 return NULL;
1878 }
1879
1880 tr0_cnt1 = len / tr0_cnt0;
1881 tr1_cnt0 = len % tr0_cnt0;
1882 }
1883
1884 desc_size = cppi5_trdesc_calc_size(num_tr, tr_size);
1885 tr_desc = dma_alloc_coherent(desc_size, &dummy);
1886 if (!tr_desc)
1887 return NULL;
1888 memset(tr_desc, 0, desc_size);
1889
1890 cppi5_trdesc_init(tr_desc, num_tr, tr_size, 0, 0);
1891 cppi5_desc_set_pktids(tr_desc, uc->id, 0x3fff);
1892 cppi5_desc_set_retpolicy(tr_desc, 0, tc_ring_id);
1893
1894 tr_req = tr_desc + tr_size;
1895
1896 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
1897 CPPI5_TR_EVENT_SIZE_COMPLETION, 1);
1898 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT);
1899
1900 tr_req[0].addr = src;
1901 tr_req[0].icnt0 = tr0_cnt0;
1902 tr_req[0].icnt1 = tr0_cnt1;
1903 tr_req[0].icnt2 = 1;
1904 tr_req[0].icnt3 = 1;
1905 tr_req[0].dim1 = tr0_cnt0;
1906
1907 tr_req[0].daddr = dest;
1908 tr_req[0].dicnt0 = tr0_cnt0;
1909 tr_req[0].dicnt1 = tr0_cnt1;
1910 tr_req[0].dicnt2 = 1;
1911 tr_req[0].dicnt3 = 1;
1912 tr_req[0].ddim1 = tr0_cnt0;
1913
1914 if (num_tr == 2) {
1915 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
1916 CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
1917 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT);
1918
1919 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
1920 tr_req[1].icnt0 = tr1_cnt0;
1921 tr_req[1].icnt1 = 1;
1922 tr_req[1].icnt2 = 1;
1923 tr_req[1].icnt3 = 1;
1924
1925 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
1926 tr_req[1].dicnt0 = tr1_cnt0;
1927 tr_req[1].dicnt1 = 1;
1928 tr_req[1].dicnt2 = 1;
1929 tr_req[1].dicnt3 = 1;
1930 }
1931
1932 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, CPPI5_TR_CSF_EOP);
1933
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]1934 flush_dcache_range((unsigned long)tr_desc,
1935 ALIGN((unsigned long)tr_desc + desc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]1936 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1937
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]1938 udma_push_to_ring(uc->tchan->t_ring, tr_desc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]1939
1940 return 0;
1941}
1942
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]1943#define TISCI_BCDMA_BCHAN_VALID_PARAMS ( \
1944 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1945 TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID)
1946
1947#define TISCI_BCDMA_TCHAN_VALID_PARAMS ( \
1948 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1949 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID)
1950
1951#define TISCI_BCDMA_RCHAN_VALID_PARAMS ( \
1952 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID)
1953
1954#define TISCI_UDMA_TCHAN_VALID_PARAMS ( \
1955 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1956 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \
1957 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \
1958 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
1959 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \
1960 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
1961 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
1962 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1963
1964#define TISCI_UDMA_RCHAN_VALID_PARAMS ( \
1965 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1966 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
1967 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
1968 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
1969 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \
1970 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \
1971 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \
1972 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \
1973 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1974
1975static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc)
1976{
1977 struct udma_dev *ud = uc->ud;
1978 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1979 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1980 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1981 struct udma_bchan *bchan = uc->bchan;
1982 int ret = 0;
1983
1984 req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS;
1985 req_tx.nav_id = tisci_rm->tisci_dev_id;
1986 req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN;
1987 req_tx.index = bchan->id;
1988
1989 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1990 if (ret)
1991 dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret);
1992
1993 return ret;
1994}
1995
1996static struct udma_bchan *__bcdma_reserve_bchan(struct udma_dev *ud, int id)
1997{
1998 if (id >= 0) {
1999 if (test_bit(id, ud->bchan_map)) {
2000 dev_err(ud->dev, "bchan%d is in use\n", id);
2001 return ERR_PTR(-ENOENT);
2002 }
2003 } else {
2004 id = find_next_zero_bit(ud->bchan_map, ud->bchan_cnt, 0);
2005 if (id == ud->bchan_cnt)
2006 return ERR_PTR(-ENOENT);
2007 }
2008 __set_bit(id, ud->bchan_map);
2009 return &ud->bchans[id];
2010}
2011
2012static int bcdma_get_bchan(struct udma_chan *uc)
2013{
2014 struct udma_dev *ud = uc->ud;
2015
2016 if (uc->bchan) {
2017 dev_err(ud->dev, "chan%d: already have bchan%d allocated\n",
2018 uc->id, uc->bchan->id);
2019 return 0;
2020 }
2021
2022 uc->bchan = __bcdma_reserve_bchan(ud, -1);
2023 if (IS_ERR(uc->bchan))
2024 return PTR_ERR(uc->bchan);
2025
2026 uc->tchan = uc->bchan;
2027
2028 return 0;
2029}
2030
2031static void bcdma_put_bchan(struct udma_chan *uc)
2032{
2033 struct udma_dev *ud = uc->ud;
2034
2035 if (uc->bchan) {
2036 dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id,
2037 uc->bchan->id);
2038 __clear_bit(uc->bchan->id, ud->bchan_map);
2039 uc->bchan = NULL;
2040 uc->tchan = NULL;
2041 }
2042}
2043
2044static void bcdma_free_bchan_resources(struct udma_chan *uc)
2045{
2046 if (!uc->bchan)
2047 return;
2048
2049 k3_nav_ringacc_ring_free(uc->bchan->tc_ring);
2050 k3_nav_ringacc_ring_free(uc->bchan->t_ring);
2051 uc->bchan->tc_ring = NULL;
2052 uc->bchan->t_ring = NULL;
2053
2054 bcdma_put_bchan(uc);
2055}
2056
2057static int bcdma_alloc_bchan_resources(struct udma_chan *uc)
2058{
2059 struct k3_nav_ring_cfg ring_cfg;
2060 struct udma_dev *ud = uc->ud;
2061 int ret;
2062
2063 ret = bcdma_get_bchan(uc);
2064 if (ret)
2065 return ret;
2066
2067 ret = k3_nav_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1,
2068 &uc->bchan->t_ring,
2069 &uc->bchan->tc_ring);
2070 if (ret) {
2071 ret = -EBUSY;
2072 goto err_ring;
2073 }
2074
2075 memset(&ring_cfg, 0, sizeof(ring_cfg));
2076 ring_cfg.size = 16;
2077 ring_cfg.elm_size = K3_NAV_RINGACC_RING_ELSIZE_8;
2078 ring_cfg.mode = K3_NAV_RINGACC_RING_MODE_RING;
2079
2080 ret = k3_nav_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg);
2081 if (ret)
2082 goto err_ringcfg;
2083
2084 return 0;
2085
2086err_ringcfg:
2087 k3_nav_ringacc_ring_free(uc->bchan->tc_ring);
2088 uc->bchan->tc_ring = NULL;
2089 k3_nav_ringacc_ring_free(uc->bchan->t_ring);
2090 uc->bchan->t_ring = NULL;
2091err_ring:
2092 bcdma_put_bchan(uc);
2093
2094 return ret;
2095}
2096
2097static int bcdma_tisci_tx_channel_config(struct udma_chan *uc)
2098{
2099 struct udma_dev *ud = uc->ud;
2100 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2101 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2102 struct udma_tchan *tchan = uc->tchan;
2103 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
2104 int ret = 0;
2105
2106 req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS;
2107 req_tx.nav_id = tisci_rm->tisci_dev_id;
2108 req_tx.index = tchan->id;
2109 req_tx.tx_supr_tdpkt = uc->config.notdpkt;
2110 if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
2111 ud->match_data->flags & UDMA_FLAG_TDTYPE) {
2112 /* wait for peer to complete the teardown for PDMAs */
2113 req_tx.valid_params |=
2114 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
2115 req_tx.tx_tdtype = 1;
2116 }
2117
2118 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
2119 if (ret)
2120 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
2121
2122 return ret;
2123}
2124
2125#define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config
2126
2127static int pktdma_tisci_rx_channel_config(struct udma_chan *uc)
2128{
2129 struct udma_dev *ud = uc->ud;
2130 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2131 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2132 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2133 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
2134 int ret = 0;
2135
2136 req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2137 req_rx.nav_id = tisci_rm->tisci_dev_id;
2138 req_rx.index = uc->rchan->id;
2139
2140 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2141 if (ret) {
2142 dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret);
2143 return ret;
2144 }
2145
2146 flow_req.valid_params =
2147 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2148 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2149 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID;
2150
2151 flow_req.nav_id = tisci_rm->tisci_dev_id;
2152 flow_req.flow_index = uc->rflow->id;
2153
2154 if (uc->config.needs_epib)
2155 flow_req.rx_einfo_present = 1;
2156 else
2157 flow_req.rx_einfo_present = 0;
2158 if (uc->config.psd_size)
2159 flow_req.rx_psinfo_present = 1;
2160 else
2161 flow_req.rx_psinfo_present = 0;
Vignesh Raghavendra87fa0d62023-03-08 09:42:57 +0530[diff] [blame]2162 flow_req.rx_error_handling = 0;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2163
2164 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2165
2166 if (ret)
2167 dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id,
2168 ret);
2169
2170 return ret;
2171}
2172
2173static int bcdma_alloc_chan_resources(struct udma_chan *uc)
2174{
2175 int ret;
2176
2177 uc->config.pkt_mode = false;
2178
2179 switch (uc->config.dir) {
2180 case DMA_MEM_TO_MEM:
2181 /* Non synchronized - mem to mem type of transfer */
2182 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2183 uc->id);
2184
2185 ret = bcdma_alloc_bchan_resources(uc);
2186 if (ret)
2187 return ret;
2188
2189 ret = bcdma_tisci_m2m_channel_config(uc);
2190 break;
2191 default:
2192 /* Can not happen */
2193 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2194 __func__, uc->id, uc->config.dir);
2195 return -EINVAL;
2196 }
2197
2198 /* check if the channel configuration was successful */
2199 if (ret)
2200 goto err_res_free;
2201
2202 if (udma_is_chan_running(uc)) {
2203 dev_warn(uc->ud->dev, "chan%d: is running!\n", uc->id);
2204 udma_stop(uc);
2205 if (udma_is_chan_running(uc)) {
2206 dev_err(uc->ud->dev, "chan%d: won't stop!\n", uc->id);
2207 goto err_res_free;
2208 }
2209 }
2210
2211 udma_reset_rings(uc);
2212
2213 return 0;
2214
2215err_res_free:
2216 bcdma_free_bchan_resources(uc);
2217 udma_free_tx_resources(uc);
2218 udma_free_rx_resources(uc);
2219
2220 udma_reset_uchan(uc);
2221
2222 return ret;
2223}
2224
2225static int pktdma_alloc_chan_resources(struct udma_chan *uc)
2226{
2227 struct udma_dev *ud = uc->ud;
2228 int ret;
2229
2230 switch (uc->config.dir) {
2231 case DMA_MEM_TO_DEV:
2232 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2233 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2234 uc->id);
2235
2236 ret = udma_alloc_tx_resources(uc);
2237 if (ret) {
2238 uc->config.remote_thread_id = -1;
2239 return ret;
2240 }
2241
2242 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2243 uc->config.dst_thread = uc->config.remote_thread_id;
2244 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2245
2246 ret = pktdma_tisci_tx_channel_config(uc);
2247 break;
2248 case DMA_DEV_TO_MEM:
2249 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2250 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2251 uc->id);
2252
2253 ret = udma_alloc_rx_resources(uc);
2254 if (ret) {
2255 uc->config.remote_thread_id = -1;
2256 return ret;
2257 }
2258
2259 uc->config.src_thread = uc->config.remote_thread_id;
2260 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2261 K3_PSIL_DST_THREAD_ID_OFFSET;
2262
2263 ret = pktdma_tisci_rx_channel_config(uc);
2264 break;
2265 default:
2266 /* Can not happen */
2267 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2268 __func__, uc->id, uc->config.dir);
2269 return -EINVAL;
2270 }
2271
2272 /* check if the channel configuration was successful */
2273 if (ret)
2274 goto err_res_free;
2275
2276 /* PSI-L pairing */
2277 ret = udma_navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2278 if (ret) {
2279 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2280 uc->config.src_thread, uc->config.dst_thread);
2281 goto err_res_free;
2282 }
2283
2284 if (udma_is_chan_running(uc)) {
2285 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2286 udma_stop(uc);
2287 if (udma_is_chan_running(uc)) {
2288 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2289 goto err_res_free;
2290 }
2291 }
2292
2293 udma_reset_rings(uc);
2294
2295 if (uc->tchan)
2296 dev_dbg(ud->dev,
2297 "chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n",
2298 uc->id, uc->tchan->id, uc->tchan->tflow_id,
2299 uc->config.remote_thread_id);
2300 else if (uc->rchan)
2301 dev_dbg(ud->dev,
2302 "chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n",
2303 uc->id, uc->rchan->id, uc->rflow->id,
2304 uc->config.remote_thread_id);
2305 return 0;
2306
2307err_res_free:
2308 udma_free_tx_resources(uc);
2309 udma_free_rx_resources(uc);
2310
2311 udma_reset_uchan(uc);
2312
2313 return ret;
2314}
2315
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2316static int udma_transfer(struct udevice *dev, int direction,
Andrew Davisd2da2842022-10-07 12:11:13 -0500[diff] [blame]2317 dma_addr_t dst, dma_addr_t src, size_t len)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2318{
2319 struct udma_dev *ud = dev_get_priv(dev);
2320 /* Channel0 is reserved for memcpy */
2321 struct udma_chan *uc = &ud->channels[0];
2322 dma_addr_t paddr = 0;
2323 int ret;
2324
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2325 switch (ud->match_data->type) {
2326 case DMA_TYPE_UDMA:
2327 ret = udma_alloc_chan_resources(uc);
2328 break;
2329 case DMA_TYPE_BCDMA:
2330 ret = bcdma_alloc_chan_resources(uc);
2331 break;
2332 default:
2333 return -EINVAL;
2334 };
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2335 if (ret)
2336 return ret;
2337
Andrew Davisd2da2842022-10-07 12:11:13 -0500[diff] [blame]2338 udma_prep_dma_memcpy(uc, dst, src, len);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2339 udma_start(uc);
2340 udma_poll_completion(uc, &paddr);
2341 udma_stop(uc);
2342
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2343 switch (ud->match_data->type) {
2344 case DMA_TYPE_UDMA:
2345 udma_free_chan_resources(uc);
2346 break;
2347 case DMA_TYPE_BCDMA:
2348 bcdma_free_bchan_resources(uc);
2349 break;
2350 default:
2351 return -EINVAL;
2352 };
2353
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2354 return 0;
2355}
2356
2357static int udma_request(struct dma *dma)
2358{
2359 struct udma_dev *ud = dev_get_priv(dma->dev);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2360 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2361 struct udma_chan *uc;
2362 unsigned long dummy;
2363 int ret;
2364
2365 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2366 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2367 return -EINVAL;
2368 }
2369
2370 uc = &ud->channels[dma->id];
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2371 ucc = &uc->config;
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2372 switch (ud->match_data->type) {
2373 case DMA_TYPE_UDMA:
2374 ret = udma_alloc_chan_resources(uc);
2375 break;
2376 case DMA_TYPE_BCDMA:
2377 ret = bcdma_alloc_chan_resources(uc);
2378 break;
2379 case DMA_TYPE_PKTDMA:
2380 ret = pktdma_alloc_chan_resources(uc);
2381 break;
2382 default:
2383 return -EINVAL;
2384 }
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2385 if (ret) {
2386 dev_err(dma->dev, "alloc dma res failed %d\n", ret);
2387 return -EINVAL;
2388 }
2389
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2390 if (uc->config.dir == DMA_MEM_TO_DEV) {
2391 uc->desc_tx = dma_alloc_coherent(ucc->hdesc_size, &dummy);
2392 memset(uc->desc_tx, 0, ucc->hdesc_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2393 } else {
2394 uc->desc_rx = dma_alloc_coherent(
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2395 ucc->hdesc_size * UDMA_RX_DESC_NUM, &dummy);
2396 memset(uc->desc_rx, 0, ucc->hdesc_size * UDMA_RX_DESC_NUM);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2397 }
2398
2399 uc->in_use = true;
2400 uc->desc_rx_cur = 0;
2401 uc->num_rx_bufs = 0;
2402
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2403 if (uc->config.dir == DMA_DEV_TO_MEM) {
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2404 uc->cfg_data.flow_id_base = uc->rflow->id;
2405 uc->cfg_data.flow_id_cnt = 1;
2406 }
2407
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2408 return 0;
2409}
2410
Simon Glass75c0ad62020-02-03 07:35:55 -0700[diff] [blame]2411static int udma_rfree(struct dma *dma)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2412{
2413 struct udma_dev *ud = dev_get_priv(dma->dev);
2414 struct udma_chan *uc;
2415
2416 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2417 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2418 return -EINVAL;
2419 }
2420 uc = &ud->channels[dma->id];
2421
2422 if (udma_is_chan_running(uc))
2423 udma_stop(uc);
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2424
2425 udma_navss_psil_unpair(ud, uc->config.src_thread,
2426 uc->config.dst_thread);
2427
2428 bcdma_free_bchan_resources(uc);
2429 udma_free_tx_resources(uc);
2430 udma_free_rx_resources(uc);
2431 udma_reset_uchan(uc);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2432
2433 uc->in_use = false;
2434
2435 return 0;
2436}
2437
2438static int udma_enable(struct dma *dma)
2439{
2440 struct udma_dev *ud = dev_get_priv(dma->dev);
2441 struct udma_chan *uc;
2442 int ret;
2443
2444 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2445 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2446 return -EINVAL;
2447 }
2448 uc = &ud->channels[dma->id];
2449
2450 ret = udma_start(uc);
2451
2452 return ret;
2453}
2454
2455static int udma_disable(struct dma *dma)
2456{
2457 struct udma_dev *ud = dev_get_priv(dma->dev);
2458 struct udma_chan *uc;
2459 int ret = 0;
2460
2461 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2462 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2463 return -EINVAL;
2464 }
2465 uc = &ud->channels[dma->id];
2466
2467 if (udma_is_chan_running(uc))
2468 ret = udma_stop(uc);
2469 else
2470 dev_err(dma->dev, "%s not running\n", __func__);
2471
2472 return ret;
2473}
2474
2475static int udma_send(struct dma *dma, void *src, size_t len, void *metadata)
2476{
2477 struct udma_dev *ud = dev_get_priv(dma->dev);
2478 struct cppi5_host_desc_t *desc_tx;
2479 dma_addr_t dma_src = (dma_addr_t)src;
2480 struct ti_udma_drv_packet_data packet_data = { 0 };
2481 dma_addr_t paddr;
2482 struct udma_chan *uc;
2483 u32 tc_ring_id;
2484 int ret;
2485
Keerthya3c8bb12019-04-24 16:33:54 +0530[diff] [blame]2486 if (metadata)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2487 packet_data = *((struct ti_udma_drv_packet_data *)metadata);
2488
2489 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2490 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2491 return -EINVAL;
2492 }
2493 uc = &ud->channels[dma->id];
2494
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2495 if (uc->config.dir != DMA_MEM_TO_DEV)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2496 return -EINVAL;
2497
2498 tc_ring_id = k3_nav_ringacc_get_ring_id(uc->tchan->tc_ring);
2499
2500 desc_tx = uc->desc_tx;
2501
2502 cppi5_hdesc_reset_hbdesc(desc_tx);
2503
2504 cppi5_hdesc_init(desc_tx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2505 uc->config.needs_epib ? CPPI5_INFO0_HDESC_EPIB_PRESENT : 0,
2506 uc->config.psd_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2507 cppi5_hdesc_set_pktlen(desc_tx, len);
2508 cppi5_hdesc_attach_buf(desc_tx, dma_src, len, dma_src, len);
2509 cppi5_desc_set_pktids(&desc_tx->hdr, uc->id, 0x3fff);
2510 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, tc_ring_id);
2511 /* pass below information from caller */
2512 cppi5_hdesc_set_pkttype(desc_tx, packet_data.pkt_type);
2513 cppi5_desc_set_tags_ids(&desc_tx->hdr, 0, packet_data.dest_tag);
2514
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2515 flush_dcache_range((unsigned long)dma_src,
2516 ALIGN((unsigned long)dma_src + len,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2517 ARCH_DMA_MINALIGN));
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2518 flush_dcache_range((unsigned long)desc_tx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2519 ALIGN((unsigned long)desc_tx + uc->config.hdesc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2520 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2521
Vignesh Raghavendrafc7a33f2019-12-09 10:25:38 +0530[diff] [blame]2522 ret = udma_push_to_ring(uc->tchan->t_ring, uc->desc_tx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2523 if (ret) {
2524 dev_err(dma->dev, "TX dma push fail ch_id %lu %d\n",
2525 dma->id, ret);
2526 return ret;
2527 }
2528
2529 udma_poll_completion(uc, &paddr);
2530
2531 return 0;
2532}
2533
2534static int udma_receive(struct dma *dma, void **dst, void *metadata)
2535{
2536 struct udma_dev *ud = dev_get_priv(dma->dev);
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2537 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2538 struct cppi5_host_desc_t *desc_rx;
2539 dma_addr_t buf_dma;
2540 struct udma_chan *uc;
2541 u32 buf_dma_len, pkt_len;
2542 u32 port_id = 0;
2543 int ret;
2544
2545 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2546 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2547 return -EINVAL;
2548 }
2549 uc = &ud->channels[dma->id];
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2550 ucc = &uc->config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2551
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2552 if (uc->config.dir != DMA_DEV_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2553 return -EINVAL;
2554 if (!uc->num_rx_bufs)
2555 return -EINVAL;
2556
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]2557 ret = k3_nav_ringacc_ring_pop(uc->rflow->r_ring, &desc_rx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2558 if (ret && ret != -ENODATA) {
2559 dev_err(dma->dev, "rx dma fail ch_id:%lu %d\n", dma->id, ret);
2560 return ret;
2561 } else if (ret == -ENODATA) {
2562 return 0;
2563 }
2564
2565 /* invalidate cache data */
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2566 invalidate_dcache_range((ulong)desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2567 (ulong)(desc_rx + ucc->hdesc_size));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2568
2569 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
2570 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
2571
2572 /* invalidate cache data */
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2573 invalidate_dcache_range((ulong)buf_dma,
2574 (ulong)(buf_dma + buf_dma_len));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2575
2576 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
2577
2578 *dst = (void *)buf_dma;
2579 uc->num_rx_bufs--;
2580
2581 return pkt_len;
2582}
2583
2584static int udma_of_xlate(struct dma *dma, struct ofnode_phandle_args *args)
2585{
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2586 struct udma_chan_config *ucc;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2587 struct udma_dev *ud = dev_get_priv(dma->dev);
2588 struct udma_chan *uc = &ud->channels[0];
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2589 struct psil_endpoint_config *ep_config;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2590 u32 val;
2591
2592 for (val = 0; val < ud->ch_count; val++) {
2593 uc = &ud->channels[val];
2594 if (!uc->in_use)
2595 break;
2596 }
2597
2598 if (val == ud->ch_count)
2599 return -EBUSY;
2600
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2601 ucc = &uc->config;
2602 ucc->remote_thread_id = args->args[0];
2603 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)
2604 ucc->dir = DMA_MEM_TO_DEV;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2605 else
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2606 ucc->dir = DMA_DEV_TO_MEM;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2607
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2608 ep_config = psil_get_ep_config(ucc->remote_thread_id);
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2609 if (IS_ERR(ep_config)) {
2610 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2611 uc->config.remote_thread_id);
2612 ucc->dir = DMA_MEM_TO_MEM;
2613 ucc->remote_thread_id = -1;
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2614 return false;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2615 }
2616
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2617 ucc->pkt_mode = ep_config->pkt_mode;
2618 ucc->channel_tpl = ep_config->channel_tpl;
2619 ucc->notdpkt = ep_config->notdpkt;
2620 ucc->ep_type = ep_config->ep_type;
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2621
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2622 if (ud->match_data->type == DMA_TYPE_PKTDMA &&
2623 ep_config->mapped_channel_id >= 0) {
2624 ucc->mapped_channel_id = ep_config->mapped_channel_id;
2625 ucc->default_flow_id = ep_config->default_flow_id;
2626 } else {
2627 ucc->mapped_channel_id = -1;
2628 ucc->default_flow_id = -1;
2629 }
2630
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2631 ucc->needs_epib = ep_config->needs_epib;
2632 ucc->psd_size = ep_config->psd_size;
2633 ucc->metadata_size = (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) + ucc->psd_size;
2634
2635 ucc->hdesc_size = cppi5_hdesc_calc_size(ucc->needs_epib,
2636 ucc->psd_size, 0);
2637 ucc->hdesc_size = ALIGN(ucc->hdesc_size, ARCH_DMA_MINALIGN);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2638
2639 dma->id = uc->id;
2640 pr_debug("Allocated dma chn:%lu epib:%d psdata:%u meta:%u thread_id:%x\n",
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2641 dma->id, ucc->needs_epib,
2642 ucc->psd_size, ucc->metadata_size,
2643 ucc->remote_thread_id);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2644
2645 return 0;
2646}
2647
2648int udma_prepare_rcv_buf(struct dma *dma, void *dst, size_t size)
2649{
2650 struct udma_dev *ud = dev_get_priv(dma->dev);
2651 struct cppi5_host_desc_t *desc_rx;
2652 dma_addr_t dma_dst;
2653 struct udma_chan *uc;
2654 u32 desc_num;
2655
2656 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2657 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2658 return -EINVAL;
2659 }
2660 uc = &ud->channels[dma->id];
2661
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2662 if (uc->config.dir != DMA_DEV_TO_MEM)
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2663 return -EINVAL;
2664
2665 if (uc->num_rx_bufs >= UDMA_RX_DESC_NUM)
2666 return -EINVAL;
2667
2668 desc_num = uc->desc_rx_cur % UDMA_RX_DESC_NUM;
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2669 desc_rx = uc->desc_rx + (desc_num * uc->config.hdesc_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2670 dma_dst = (dma_addr_t)dst;
2671
2672 cppi5_hdesc_reset_hbdesc(desc_rx);
2673
2674 cppi5_hdesc_init(desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2675 uc->config.needs_epib ? CPPI5_INFO0_HDESC_EPIB_PRESENT : 0,
2676 uc->config.psd_size);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2677 cppi5_hdesc_set_pktlen(desc_rx, size);
2678 cppi5_hdesc_attach_buf(desc_rx, dma_dst, size, dma_dst, size);
2679
Matthias Schiffer65aef702024-04-26 10:02:28 +0200[diff] [blame]2680 invalidate_dcache_range((unsigned long)dma_dst,
2681 (unsigned long)(dma_dst + size));
2682
Vignesh Raghavendrace431412019-12-09 10:25:39 +0530[diff] [blame]2683 flush_dcache_range((unsigned long)desc_rx,
Vignesh Raghavendra07826212020-07-06 13:26:25 +0530[diff] [blame]2684 ALIGN((unsigned long)desc_rx + uc->config.hdesc_size,
Vignesh Raghavendra05b711f2019-12-09 10:25:35 +0530[diff] [blame]2685 ARCH_DMA_MINALIGN));
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2686
Vignesh Raghavendra2db3b282020-07-06 13:26:26 +0530[diff] [blame]2687 udma_push_to_ring(uc->rflow->fd_ring, desc_rx);
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2688
2689 uc->num_rx_bufs++;
2690 uc->desc_rx_cur++;
2691
2692 return 0;
2693}
2694
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2695static int udma_get_cfg(struct dma *dma, u32 id, void **data)
2696{
2697 struct udma_dev *ud = dev_get_priv(dma->dev);
2698 struct udma_chan *uc;
2699
2700 if (dma->id >= (ud->rchan_cnt + ud->tchan_cnt)) {
2701 dev_err(dma->dev, "invalid dma ch_id %lu\n", dma->id);
2702 return -EINVAL;
2703 }
2704
2705 switch (id) {
2706 case TI_UDMA_CHAN_PRIV_INFO:
2707 uc = &ud->channels[dma->id];
2708 *data = &uc->cfg_data;
2709 return 0;
2710 }
2711
2712 return -EINVAL;
2713}
2714
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2715static const struct dma_ops udma_ops = {
2716 .transfer = udma_transfer,
2717 .of_xlate = udma_of_xlate,
2718 .request = udma_request,
Simon Glass75c0ad62020-02-03 07:35:55 -0700[diff] [blame]2719 .rfree = udma_rfree,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2720 .enable = udma_enable,
2721 .disable = udma_disable,
2722 .send = udma_send,
2723 .receive = udma_receive,
2724 .prepare_rcv_buf = udma_prepare_rcv_buf,
Vignesh Raghavendra39349892019-12-04 22:17:21 +0530[diff] [blame]2725 .get_cfg = udma_get_cfg,
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2726};
2727
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2728static struct udma_match_data am654_main_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2729 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2730 .psil_base = 0x1000,
2731 .enable_memcpy_support = true,
2732 .statictr_z_mask = GENMASK(11, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2733 .oes = {
2734 .udma_rchan = 0x200,
2735 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2736 .tpl_levels = 2,
2737 .level_start_idx = {
2738 [0] = 8, /* Normal channels */
2739 [1] = 0, /* High Throughput channels */
2740 },
2741};
2742
2743static struct udma_match_data am654_mcu_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2744 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2745 .psil_base = 0x6000,
2746 .enable_memcpy_support = true,
2747 .statictr_z_mask = GENMASK(11, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2748 .oes = {
2749 .udma_rchan = 0x200,
2750 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2751 .tpl_levels = 2,
2752 .level_start_idx = {
2753 [0] = 2, /* Normal channels */
2754 [1] = 0, /* High Throughput channels */
2755 },
2756};
2757
2758static struct udma_match_data j721e_main_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2759 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2760 .psil_base = 0x1000,
2761 .enable_memcpy_support = true,
2762 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2763 .statictr_z_mask = GENMASK(23, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2764 .oes = {
2765 .udma_rchan = 0x400,
2766 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2767 .tpl_levels = 3,
2768 .level_start_idx = {
2769 [0] = 16, /* Normal channels */
2770 [1] = 4, /* High Throughput channels */
2771 [2] = 0, /* Ultra High Throughput channels */
2772 },
2773};
2774
2775static struct udma_match_data j721e_mcu_data = {
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2776 .type = DMA_TYPE_UDMA,
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2777 .psil_base = 0x6000,
2778 .enable_memcpy_support = true,
2779 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2780 .statictr_z_mask = GENMASK(23, 0),
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2781 .oes = {
2782 .udma_rchan = 0x400,
2783 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2784 .tpl_levels = 2,
2785 .level_start_idx = {
2786 [0] = 2, /* Normal channels */
2787 [1] = 0, /* High Throughput channels */
2788 },
2789};
2790
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2791static struct udma_match_data am64_bcdma_data = {
2792 .type = DMA_TYPE_BCDMA,
2793 .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */
2794 .enable_memcpy_support = true, /* Supported via bchan */
2795 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2796 .statictr_z_mask = GENMASK(23, 0),
2797 .oes = {
2798 .bcdma_bchan_data = 0x2200,
2799 .bcdma_bchan_ring = 0x2400,
2800 .bcdma_tchan_data = 0x2800,
2801 .bcdma_tchan_ring = 0x2a00,
2802 .bcdma_rchan_data = 0x2e00,
2803 .bcdma_rchan_ring = 0x3000,
2804 },
2805 /* No throughput levels */
2806};
2807
2808static struct udma_match_data am64_pktdma_data = {
2809 .type = DMA_TYPE_PKTDMA,
2810 .psil_base = 0x1000,
2811 .enable_memcpy_support = false,
2812 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE,
2813 .statictr_z_mask = GENMASK(23, 0),
2814 .oes = {
2815 .pktdma_tchan_flow = 0x1200,
2816 .pktdma_rchan_flow = 0x1600,
2817 },
2818 /* No throughput levels */
2819};
2820
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2821static const struct udevice_id udma_ids[] = {
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2822 {
2823 .compatible = "ti,am654-navss-main-udmap",
2824 .data = (ulong)&am654_main_data,
2825 },
2826 {
2827 .compatible = "ti,am654-navss-mcu-udmap",
2828 .data = (ulong)&am654_mcu_data,
2829 }, {
2830 .compatible = "ti,j721e-navss-main-udmap",
2831 .data = (ulong)&j721e_main_data,
2832 }, {
2833 .compatible = "ti,j721e-navss-mcu-udmap",
2834 .data = (ulong)&j721e_mcu_data,
2835 },
Vignesh Raghavendra5a7589c2021-05-10 20:06:08 +0530[diff] [blame]2836 {
2837 .compatible = "ti,am64-dmss-bcdma",
2838 .data = (ulong)&am64_bcdma_data,
2839 },
2840 {
2841 .compatible = "ti,am64-dmss-pktdma",
2842 .data = (ulong)&am64_pktdma_data,
2843 },
Vignesh Raghavendra222f5d82020-07-07 13:43:34 +0530[diff] [blame]2844 { /* Sentinel */ },
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2845};
2846
2847U_BOOT_DRIVER(ti_edma3) = {
2848 .name = "ti-udma",
2849 .id = UCLASS_DMA,
2850 .of_match = udma_ids,
2851 .ops = &udma_ops,
2852 .probe = udma_probe,
Simon Glass8a2b47f2020-12-03 16:55:17 -0700[diff] [blame]2853 .priv_auto = sizeof(struct udma_dev),
Vignesh R3a9dbf32019-02-05 17:31:24 +0530[diff] [blame]2854};