blob: cae570cf59847633ce475b32ff8e07803935318d [file] [log] [blame]
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +05301// SPDX-License-Identifier: GPL-2.0
2/*
3 * Cadence USBSS DRD Driver - gadget side.
4 *
5 * Copyright (C) 2018-2019 Cadence Design Systems.
6 * Copyright (C) 2017-2018 NXP
7 *
8 * Authors: Pawel Jez <pjez@cadence.com>,
9 * Pawel Laszczak <pawell@cadence.com>
10 * Peter Chen <peter.chen@nxp.com>
11 */
12
13/*
14 * Work around 1:
15 * At some situations, the controller may get stale data address in TRB
16 * at below sequences:
17 * 1. Controller read TRB includes data address
18 * 2. Software updates TRBs includes data address and Cycle bit
19 * 3. Controller read TRB which includes Cycle bit
20 * 4. DMA run with stale data address
21 *
22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
23 * After preparing all TRBs driver needs to check the position of DMA and
24 * if the DMA point to the first just added TRB and doorbell is 1,
25 * then driver must defer making this TRB as valid. This TRB will be make
26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
27 * interrupt.
28 *
29 * Issue has been fixed in DEV_VER_V3 version of controller.
30 *
31 * Work around 2:
32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
34 * in correct order. If the first packet in the buffer will not be handled,
35 * then the following packets directed for other endpoints and functions
36 * will be blocked.
37 * Additionally the packets directed to one endpoint can block entire on-chip
38 * buffers. In this case transfer to other endpoints also will blocked.
39 *
40 * To resolve this issue after raising the descriptor missing interrupt
41 * driver prepares internal usb_request object and use it to arm DMA transfer.
42 *
43 * The problematic situation was observed in case when endpoint has been enabled
44 * but no usb_request were queued. Driver try detects such endpoints and will
45 * use this workaround only for these endpoint.
46 *
47 * Driver use limited number of buffer. This number can be set by macro
48 * CDNS3_WA2_NUM_BUFFERS.
49 *
50 * Such blocking situation was observed on ACM gadget. For this function
51 * host send OUT data packet but ACM function is not prepared for this packet.
52 * It's cause that buffer placed in on chip memory block transfer to other
53 * endpoints.
54 *
55 * Issue has been fixed in DEV_VER_V2 version of controller.
56 *
57 */
58
59#include <dm.h>
Simon Glass9bc15642020-02-03 07:36:16 -070060#include <dm/device_compat.h>
Simon Glassd66c5f72020-02-03 07:36:15 -070061#include <dm/devres.h>
Simon Glass4dcacfc2020-05-10 11:40:13 -060062#include <linux/bitops.h>
Simon Glassdbd79542020-05-10 11:40:11 -060063#include <linux/delay.h>
Simon Glassd66c5f72020-02-03 07:36:15 -070064#include <linux/err.h>
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +053065#include <linux/usb/gadget.h>
66#include <linux/compat.h>
67#include <linux/iopoll.h>
Masahiro Yamada6373a172020-02-14 16:40:19 +090068#include <linux/dma-mapping.h>
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +053069#include <linux/bitmap.h>
70#include <linux/bug.h>
71
72#include "core.h"
73#include "gadget-export.h"
74#include "gadget.h"
75#include "trace.h"
76#include "drd.h"
77
78#define readl_poll_timeout_atomic readl_poll_timeout
79#define usleep_range(a, b) udelay((b))
80
81static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
82 struct usb_request *request,
83 gfp_t gfp_flags);
84
Ravi Gunasekarane74da732023-07-19 14:29:08 +053085static void cdns3_gadget_udc_set_speed(struct usb_gadget *gadget,
86 enum usb_device_speed speed);
87
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +053088/**
89 * cdns3_set_register_bit - set bit in given register.
90 * @ptr: address of device controller register to be read and changed
91 * @mask: bits requested to set
92 */
93void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
94{
95 mask = readl(ptr) | mask;
96 writel(mask, ptr);
97}
98
99/**
100 * cdns3_ep_addr_to_index - Macro converts endpoint address to
101 * index of endpoint object in cdns3_device.eps[] container
102 * @ep_addr: endpoint address for which endpoint object is required
103 *
104 */
105u8 cdns3_ep_addr_to_index(u8 ep_addr)
106{
107 return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
108}
109
110static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
111 struct cdns3_endpoint *priv_ep)
112{
113 int dma_index;
114
115 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
116
117 return dma_index / TRB_SIZE;
118}
119
120/**
121 * cdns3_next_request - returns next request from list
122 * @list: list containing requests
123 *
124 * Returns request or NULL if no requests in list
125 */
126struct usb_request *cdns3_next_request(struct list_head *list)
127{
128 return list_first_entry_or_null(list, struct usb_request, list);
129}
130
131/**
132 * cdns3_next_align_buf - returns next buffer from list
133 * @list: list containing buffers
134 *
135 * Returns buffer or NULL if no buffers in list
136 */
137struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
138{
139 return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
140}
141
142/**
143 * cdns3_next_priv_request - returns next request from list
144 * @list: list containing requests
145 *
146 * Returns request or NULL if no requests in list
147 */
148struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
149{
150 return list_first_entry_or_null(list, struct cdns3_request, list);
151}
152
153/**
154 * select_ep - selects endpoint
155 * @priv_dev: extended gadget object
156 * @ep: endpoint address
157 */
158void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
159{
160 if (priv_dev->selected_ep == ep)
161 return;
162
163 priv_dev->selected_ep = ep;
164 writel(ep, &priv_dev->regs->ep_sel);
165}
166
167dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
168 struct cdns3_trb *trb)
169{
170 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
171
172 return priv_ep->trb_pool_dma + offset;
173}
174
175int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
176{
177 switch (priv_ep->type) {
178 case USB_ENDPOINT_XFER_ISOC:
179 return TRB_ISO_RING_SIZE;
180 case USB_ENDPOINT_XFER_CONTROL:
181 return TRB_CTRL_RING_SIZE;
182 default:
183 return TRB_RING_SIZE;
184 }
185}
186
187/**
188 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
189 * @priv_ep: endpoint object
190 *
191 * Function will return 0 on success or -ENOMEM on allocation error
192 */
193int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
194{
195 int ring_size = cdns3_ring_size(priv_ep);
196 struct cdns3_trb *link_trb;
197
198 if (!priv_ep->trb_pool) {
199 priv_ep->trb_pool =
200 dma_alloc_coherent(ring_size,
201 (unsigned long *)&priv_ep->trb_pool_dma);
202 if (!priv_ep->trb_pool)
203 return -ENOMEM;
204 } else {
205 memset(priv_ep->trb_pool, 0, ring_size);
206 }
207
208 if (!priv_ep->num)
209 return 0;
210
211 priv_ep->num_trbs = ring_size / TRB_SIZE;
212 /* Initialize the last TRB as Link TRB. */
213 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
214 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
215 link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
216
217 return 0;
218}
219
220static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
221{
222 if (priv_ep->trb_pool) {
223 dma_free_coherent(priv_ep->trb_pool);
224 priv_ep->trb_pool = NULL;
225 }
226}
227
228/**
229 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
230 * @priv_ep: endpoint object
231 *
232 * Endpoint must be selected before call to this function
233 */
234static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
235{
236 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
237 int val;
238
239 trace_cdns3_halt(priv_ep, 1, 1);
240
241 writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
242 &priv_dev->regs->ep_cmd);
243
244 /* wait for DFLUSH cleared */
245 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
246 !(val & EP_CMD_DFLUSH), 1000);
247 priv_ep->flags |= EP_STALLED;
248 priv_ep->flags &= ~EP_STALL_PENDING;
249}
250
251/**
252 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
253 * @priv_dev: extended gadget object
254 */
255void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
256{
257 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
258
259 cdns3_allow_enable_l1(priv_dev, 0);
260 priv_dev->hw_configured_flag = 0;
261 priv_dev->onchip_used_size = 0;
262 priv_dev->out_mem_is_allocated = 0;
263 priv_dev->wait_for_setup = 0;
264}
265
266/**
267 * cdns3_ep_inc_trb - increment a trb index.
268 * @index: Pointer to the TRB index to increment.
269 * @cs: Cycle state
270 * @trb_in_seg: number of TRBs in segment
271 *
272 * The index should never point to the link TRB. After incrementing,
273 * if it is point to the link TRB, wrap around to the beginning and revert
274 * cycle state bit The
275 * link TRB is always at the last TRB entry.
276 */
277static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
278{
279 (*index)++;
280 if (*index == (trb_in_seg - 1)) {
281 *index = 0;
282 *cs ^= 1;
283 }
284}
285
286/**
287 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
288 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
289 */
290static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
291{
292 priv_ep->free_trbs--;
293 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
294}
295
296/**
297 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
298 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
299 */
300static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
301{
302 priv_ep->free_trbs++;
303 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
304}
305
306void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
307{
308 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
309 int current_trb = priv_req->start_trb;
310
311 while (current_trb != priv_req->end_trb) {
312 cdns3_ep_inc_deq(priv_ep);
313 current_trb = priv_ep->dequeue;
314 }
315
316 cdns3_ep_inc_deq(priv_ep);
317}
318
319/**
320 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
321 * @priv_dev: Extended gadget object
322 * @enable: Enable/disable permit to transition to L1.
323 *
324 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
325 * then controller answer with ACK handshake.
326 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
327 * then controller answer with NYET handshake.
328 */
329void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
330{
331 if (enable)
332 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
333 else
334 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
335}
336
337enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
338{
339 u32 reg;
340
341 reg = readl(&priv_dev->regs->usb_sts);
342
343 if (DEV_SUPERSPEED(reg))
344 return USB_SPEED_SUPER;
345 else if (DEV_HIGHSPEED(reg))
346 return USB_SPEED_HIGH;
347 else if (DEV_FULLSPEED(reg))
348 return USB_SPEED_FULL;
349 else if (DEV_LOWSPEED(reg))
350 return USB_SPEED_LOW;
351 return USB_SPEED_UNKNOWN;
352}
353
354/**
355 * cdns3_start_all_request - add to ring all request not started
356 * @priv_dev: Extended gadget object
357 * @priv_ep: The endpoint for whom request will be started.
358 *
359 * Returns return ENOMEM if transfer ring i not enough TRBs to start
360 * all requests.
361 */
362static int cdns3_start_all_request(struct cdns3_device *priv_dev,
363 struct cdns3_endpoint *priv_ep)
364{
365 struct usb_request *request;
366 int ret = 0;
367
368 while (!list_empty(&priv_ep->deferred_req_list)) {
369 request = cdns3_next_request(&priv_ep->deferred_req_list);
370
371 ret = cdns3_ep_run_transfer(priv_ep, request);
372 if (ret)
373 return ret;
374
375 list_del(&request->list);
376 list_add_tail(&request->list,
377 &priv_ep->pending_req_list);
378 }
379
380 priv_ep->flags &= ~EP_RING_FULL;
381 return ret;
382}
383
384/*
385 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
386 * driver try to detect whether endpoint need additional internal
387 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
388 * if before first DESCMISS interrupt the DMA will be armed.
389 */
390#define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \
391 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
392 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
393 (reg) |= EP_STS_EN_DESCMISEN; \
394 } } while (0)
395
396/**
397 * cdns3_wa2_descmiss_copy_data copy data from internal requests to
398 * request queued by class driver.
399 * @priv_ep: extended endpoint object
400 * @request: request object
401 */
402static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
403 struct usb_request *request)
404{
405 struct usb_request *descmiss_req;
406 struct cdns3_request *descmiss_priv_req;
407
408 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
409 int chunk_end;
410 int length;
411
412 descmiss_priv_req =
413 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
414 descmiss_req = &descmiss_priv_req->request;
415
416 /* driver can't touch pending request */
417 if (descmiss_priv_req->flags & REQUEST_PENDING)
418 break;
419
420 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
421 length = request->actual + descmiss_req->actual;
422
423 request->status = descmiss_req->status;
424
425 if (length <= request->length) {
426 memcpy(&((u8 *)request->buf)[request->actual],
427 descmiss_req->buf,
428 descmiss_req->actual);
429 request->actual = length;
430 } else {
431 /* It should never occur */
432 request->status = -ENOMEM;
433 }
434
435 list_del_init(&descmiss_priv_req->list);
436
437 kfree(descmiss_req->buf);
438 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
439 --priv_ep->wa2_counter;
440
441 if (!chunk_end)
442 break;
443 }
444}
445
446struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
447 struct cdns3_endpoint *priv_ep,
448 struct cdns3_request *priv_req)
449{
450 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
451 priv_req->flags & REQUEST_INTERNAL) {
452 struct usb_request *req;
453
454 req = cdns3_next_request(&priv_ep->deferred_req_list);
455
456 priv_ep->descmis_req = NULL;
457
458 if (!req)
459 return NULL;
460
461 cdns3_wa2_descmiss_copy_data(priv_ep, req);
462 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
463 req->length != req->actual) {
464 /* wait for next part of transfer */
465 return NULL;
466 }
467
468 if (req->status == -EINPROGRESS)
469 req->status = 0;
470
471 list_del_init(&req->list);
472 cdns3_start_all_request(priv_dev, priv_ep);
473 return req;
474 }
475
476 return &priv_req->request;
477}
478
479int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
480 struct cdns3_endpoint *priv_ep,
481 struct cdns3_request *priv_req)
482{
483 int deferred = 0;
484
485 /*
486 * If transfer was queued before DESCMISS appear than we
487 * can disable handling of DESCMISS interrupt. Driver assumes that it
488 * can disable special treatment for this endpoint.
489 */
490 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
491 u32 reg;
492
493 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
494 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
495 reg = readl(&priv_dev->regs->ep_sts_en);
496 reg &= ~EP_STS_EN_DESCMISEN;
497 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
498 writel(reg, &priv_dev->regs->ep_sts_en);
499 }
500
501 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
502 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
503 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
504
505 /*
506 * DESCMISS transfer has been finished, so data will be
507 * directly copied from internal allocated usb_request
508 * objects.
509 */
510 if (pending_empty && !descmiss_empty &&
511 !(priv_req->flags & REQUEST_INTERNAL)) {
512 cdns3_wa2_descmiss_copy_data(priv_ep,
513 &priv_req->request);
514
515 trace_cdns3_wa2(priv_ep, "get internal stored data");
516
517 list_add_tail(&priv_req->request.list,
518 &priv_ep->pending_req_list);
519 cdns3_gadget_giveback(priv_ep, priv_req,
520 priv_req->request.status);
521
522 /*
523 * Intentionally driver returns positive value as
524 * correct value. It informs that transfer has
525 * been finished.
526 */
527 return EINPROGRESS;
528 }
529
530 /*
531 * Driver will wait for completion DESCMISS transfer,
532 * before starts new, not DESCMISS transfer.
533 */
534 if (!pending_empty && !descmiss_empty) {
535 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
536 deferred = 1;
537 }
538
539 if (priv_req->flags & REQUEST_INTERNAL)
540 list_add_tail(&priv_req->list,
541 &priv_ep->wa2_descmiss_req_list);
542 }
543
544 return deferred;
545}
546
547static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
548{
549 struct cdns3_request *priv_req;
550
551 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
552 u8 chain;
553
554 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
555 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
556
557 trace_cdns3_wa2(priv_ep, "removes eldest request");
558
559 kfree(priv_req->request.buf);
560 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
561 &priv_req->request);
562 list_del_init(&priv_req->list);
563 --priv_ep->wa2_counter;
564
565 if (!chain)
566 break;
567 }
568}
569
570/**
571 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
572 * @priv_dev: extended gadget object
573 *
574 * This function is used only for WA2. For more information see Work around 2
575 * description.
576 */
577static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
578{
579 struct cdns3_request *priv_req;
580 struct usb_request *request;
581
582 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
583 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
584 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
585 }
586
587 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
588
589 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS)
590 cdns3_wa2_remove_old_request(priv_ep);
591
592 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
593 GFP_ATOMIC);
594 if (!request)
595 goto err;
596
597 priv_req = to_cdns3_request(request);
598 priv_req->flags |= REQUEST_INTERNAL;
599
600 /* if this field is still assigned it indicate that transfer related
601 * with this request has not been finished yet. Driver in this
602 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
603 * flag to previous one. It will indicate that current request is
604 * part of the previous one.
605 */
606 if (priv_ep->descmis_req)
607 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
608
609 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
610 GFP_ATOMIC);
611 priv_ep->wa2_counter++;
612
613 if (!priv_req->request.buf) {
614 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
615 goto err;
616 }
617
618 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
619 priv_ep->descmis_req = priv_req;
620
621 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
622 &priv_ep->descmis_req->request,
623 GFP_ATOMIC);
624
625 return;
626
627err:
628 dev_err(priv_ep->cdns3_dev->dev,
629 "Failed: No sufficient memory for DESCMIS\n");
630}
631
632/**
633 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
634 * @priv_ep: The endpoint to whom the request belongs to
635 * @priv_req: The request we're giving back
636 * @status: completion code for the request
637 *
638 * Must be called with controller's lock held and interrupts disabled. This
639 * function will unmap @req and call its ->complete() callback to notify upper
640 * layers that it has completed.
641 */
642void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
643 struct cdns3_request *priv_req,
644 int status)
645{
646 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
647 struct usb_request *request = &priv_req->request;
648
649 list_del_init(&request->list);
650
651 if (request->status == -EINPROGRESS)
652 request->status = status;
653
654 usb_gadget_unmap_request(&priv_dev->gadget, request,
655 priv_ep->dir);
656
657 if ((priv_req->flags & REQUEST_UNALIGNED) &&
658 priv_ep->dir == USB_DIR_OUT && !request->status)
659 memcpy(request->buf, priv_req->aligned_buf->buf,
660 request->length);
661
662 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
663 trace_cdns3_gadget_giveback(priv_req);
664
665 if (priv_dev->dev_ver < DEV_VER_V2) {
666 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
667 priv_req);
668 if (!request)
669 return;
670 }
671
672 if (request->complete) {
673 spin_unlock(&priv_dev->lock);
674 usb_gadget_giveback_request(&priv_ep->endpoint,
675 request);
676 spin_lock(&priv_dev->lock);
677 }
678
679 if (request->buf == priv_dev->zlp_buf)
680 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
681}
682
683void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
684{
685 /* Work around for stale data address in TRB*/
686 if (priv_ep->wa1_set) {
687 trace_cdns3_wa1(priv_ep, "restore cycle bit");
688
689 priv_ep->wa1_set = 0;
690 priv_ep->wa1_trb_index = 0xFFFF;
691 if (priv_ep->wa1_cycle_bit) {
692 priv_ep->wa1_trb->control =
693 priv_ep->wa1_trb->control | 0x1;
694 } else {
695 priv_ep->wa1_trb->control =
696 priv_ep->wa1_trb->control & ~0x1;
697 }
698 }
699}
700
701static void cdns3_free_aligned_request_buf(struct cdns3_device *priv_dev)
702{
703 struct cdns3_aligned_buf *buf, *tmp;
704 unsigned long flags;
705
706 spin_lock_irqsave(&priv_dev->lock, flags);
707
708 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
709 if (!buf->in_use) {
710 list_del(&buf->list);
711
712 /*
713 * Re-enable interrupts to free DMA capable memory.
714 * Driver can't free this memory with disabled
715 * interrupts.
716 */
717 spin_unlock_irqrestore(&priv_dev->lock, flags);
718 dma_free_coherent(buf->buf);
719 kfree(buf);
720 spin_lock_irqsave(&priv_dev->lock, flags);
721 }
722 }
723
724 spin_unlock_irqrestore(&priv_dev->lock, flags);
725}
726
727static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
728{
729 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
730 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
731 struct cdns3_aligned_buf *buf;
732
733 /* check if buffer is aligned to 8. */
734 if (!((uintptr_t)priv_req->request.buf & 0x7))
735 return 0;
736
737 buf = priv_req->aligned_buf;
738
739 if (!buf || priv_req->request.length > buf->size) {
740 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
741 if (!buf)
742 return -ENOMEM;
743
744 buf->size = priv_req->request.length;
745
746 buf->buf = dma_alloc_coherent(buf->size,
747 (unsigned long *)&buf->dma);
748 if (!buf->buf) {
749 kfree(buf);
750 return -ENOMEM;
751 }
752
753 if (priv_req->aligned_buf) {
754 trace_cdns3_free_aligned_request(priv_req);
755 priv_req->aligned_buf->in_use = 0;
756#ifndef __UBOOT__
757 queue_work(system_freezable_wq,
758 &priv_dev->aligned_buf_wq);
759#else
760 cdns3_free_aligned_request_buf(priv_dev);
761#endif
762 }
763
764 buf->in_use = 1;
765 priv_req->aligned_buf = buf;
766
767 list_add_tail(&buf->list,
768 &priv_dev->aligned_buf_list);
769 }
770
771 if (priv_ep->dir == USB_DIR_IN) {
772 memcpy(buf->buf, priv_req->request.buf,
773 priv_req->request.length);
774 }
775
776 priv_req->flags |= REQUEST_UNALIGNED;
777 trace_cdns3_prepare_aligned_request(priv_req);
778
779 return 0;
780}
781
782static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
783 struct cdns3_trb *trb)
784{
785 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
786
787 if (!priv_ep->wa1_set) {
788 u32 doorbell;
789
790 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
791
792 if (doorbell) {
793 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
794 priv_ep->wa1_set = 1;
795 priv_ep->wa1_trb = trb;
796 priv_ep->wa1_trb_index = priv_ep->enqueue;
797 trace_cdns3_wa1(priv_ep, "set guard");
798 return 0;
799 }
800 }
801 return 1;
802}
803
804static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
805 struct cdns3_endpoint *priv_ep)
806{
807 int dma_index;
808 u32 doorbell;
809
810 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
811 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
812
813 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
814 cdns3_wa1_restore_cycle_bit(priv_ep);
815}
816
817/**
818 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
819 * @priv_ep: endpoint object
820 *
821 * Returns zero on success or negative value on failure
822 */
823int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
824 struct usb_request *request)
825{
826 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
827 struct cdns3_request *priv_req;
828 struct cdns3_trb *trb;
829 dma_addr_t trb_dma;
830 u32 togle_pcs = 1;
831 int sg_iter = 0;
832 int num_trb = 1;
833 int address;
834 u32 control;
835 int pcs;
836
837 if (num_trb > priv_ep->free_trbs) {
838 priv_ep->flags |= EP_RING_FULL;
839 return -ENOBUFS;
840 }
841
842 priv_req = to_cdns3_request(request);
843 address = priv_ep->endpoint.desc->bEndpointAddress;
844
845 priv_ep->flags |= EP_PENDING_REQUEST;
846
847 /* must allocate buffer aligned to 8 */
848 if (priv_req->flags & REQUEST_UNALIGNED)
849 trb_dma = priv_req->aligned_buf->dma;
850 else
851 trb_dma = request->dma;
852
853 trb = priv_ep->trb_pool + priv_ep->enqueue;
854 priv_req->start_trb = priv_ep->enqueue;
855 priv_req->trb = trb;
856
857 cdns3_select_ep(priv_ep->cdns3_dev, address);
858
859 /* prepare ring */
860 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
861 struct cdns3_trb *link_trb;
862 int doorbell, dma_index;
863 u32 ch_bit = 0;
864
865 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
866 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
867
868 /* Driver can't update LINK TRB if it is current processed. */
869 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
870 priv_ep->flags |= EP_DEFERRED_DRDY;
871 return -ENOBUFS;
872 }
873
874 /*updating C bt in Link TRB before starting DMA*/
875 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
876 /*
877 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
878 * that DMA stuck at the LINK TRB.
879 * On the other hand, removing TRB_CHAIN for longer TRs for
880 * epXout cause that DMA stuck after handling LINK TRB.
881 * To eliminate this strange behavioral driver set TRB_CHAIN
882 * bit only for TR size > 2.
883 */
884 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
885 TRBS_PER_SEGMENT > 2)
886 ch_bit = TRB_CHAIN;
887
888 link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
889 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
890 }
891
892 if (priv_dev->dev_ver <= DEV_VER_V2)
893 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
894
895 /* set incorrect Cycle Bit for first trb*/
896 control = priv_ep->pcs ? 0 : TRB_CYCLE;
897
898 do {
899 u32 length;
900 u16 td_size = 0;
901
902 /* fill TRB */
903 control |= TRB_TYPE(TRB_NORMAL);
904 trb->buffer = TRB_BUFFER(trb_dma);
905
906 length = request->length;
907
908 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
909 td_size = DIV_ROUND_UP(length,
910 priv_ep->endpoint.maxpacket);
911
912 trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
913 TRB_LEN(length);
914 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
915 trb->length |= TRB_TDL_SS_SIZE(td_size);
916 else
917 control |= TRB_TDL_HS_SIZE(td_size);
918
919 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
920
921 /*
922 * first trb should be prepared as last to avoid processing
923 * transfer to early
924 */
925 if (sg_iter != 0)
926 control |= pcs;
927
928 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
929 control |= TRB_IOC | TRB_ISP;
930 } else {
931 /* for last element in TD or in SG list */
932 if (sg_iter == (num_trb - 1) && sg_iter != 0)
933 control |= pcs | TRB_IOC | TRB_ISP;
934 }
935
936 if (sg_iter)
937 trb->control = control;
938 else
939 priv_req->trb->control = control;
940
941 control = 0;
942 ++sg_iter;
943 priv_req->end_trb = priv_ep->enqueue;
944 cdns3_ep_inc_enq(priv_ep);
945 trb = priv_ep->trb_pool + priv_ep->enqueue;
946 } while (sg_iter < num_trb);
947
948 trb = priv_req->trb;
949
950 priv_req->flags |= REQUEST_PENDING;
951
952 if (sg_iter == 1)
953 trb->control |= TRB_IOC | TRB_ISP;
954
955 /*
956 * Memory barrier - cycle bit must be set before other filds in trb.
957 */
958 dmb();
959
960 /* give the TD to the consumer*/
961 if (togle_pcs)
962 trb->control = trb->control ^ 1;
963
964 if (priv_dev->dev_ver <= DEV_VER_V2)
965 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
966
967 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
968
969 /*
970 * Memory barrier - Cycle Bit must be set before trb->length and
971 * trb->buffer fields.
972 */
973 dmb();
974
975 /*
976 * For DMULT mode we can set address to transfer ring only once after
977 * enabling endpoint.
978 */
979 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
980 /*
981 * Until SW is not ready to handle the OUT transfer the ISO OUT
982 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
983 * EP_CFG_ENABLE must be set before updating ep_traddr.
984 */
985 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
986 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
987 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
988 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
989 EP_CFG_ENABLE);
990 }
991
992 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
993 priv_req->start_trb * TRB_SIZE),
994 &priv_dev->regs->ep_traddr);
995
996 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
997 }
998
999 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1000 trace_cdns3_ring(priv_ep);
1001 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1002 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1003 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1004 trace_cdns3_doorbell_epx(priv_ep->name,
1005 readl(&priv_dev->regs->ep_traddr));
1006 }
1007
1008 /* WORKAROUND for transition to L0 */
1009 __cdns3_gadget_wakeup(priv_dev);
1010
1011 return 0;
1012}
1013
1014void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1015{
1016 struct cdns3_endpoint *priv_ep;
1017 struct usb_ep *ep;
1018 int val;
1019
1020 if (priv_dev->hw_configured_flag)
1021 return;
1022
1023 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1024 writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
1025
1026 cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1027 USB_CONF_U1EN | USB_CONF_U2EN);
1028
1029 /* wait until configuration set */
1030 readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
1031 val & USB_STS_CFGSTS_MASK, 100);
1032
1033 priv_dev->hw_configured_flag = 1;
1034
1035 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1036 priv_ep = ep_to_cdns3_ep(ep);
1037 if (priv_ep->flags & EP_ENABLED)
1038 cdns3_start_all_request(priv_dev, priv_ep);
1039 }
1040}
1041
1042/**
1043 * cdns3_request_handled - check whether request has been handled by DMA
1044 *
1045 * @priv_ep: extended endpoint object.
1046 * @priv_req: request object for checking
1047 *
1048 * Endpoint must be selected before invoking this function.
1049 *
1050 * Returns false if request has not been handled by DMA, else returns true.
1051 *
1052 * SR - start ring
1053 * ER - end ring
1054 * DQ = priv_ep->dequeue - dequeue position
1055 * EQ = priv_ep->enqueue - enqueue position
1056 * ST = priv_req->start_trb - index of first TRB in transfer ring
1057 * ET = priv_req->end_trb - index of last TRB in transfer ring
1058 * CI = current_index - index of processed TRB by DMA.
1059 *
1060 * As first step, function checks if cycle bit for priv_req->start_trb is
1061 * correct.
1062 *
1063 * some rules:
1064 * 1. priv_ep->dequeue never exceed current_index.
1065 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1066 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1067 * and priv_ep->free_trbs is zero.
1068 * This case indicate that TR is full.
1069 *
1070 * Then We can split recognition into two parts:
1071 * Case 1 - priv_ep->dequeue < current_index
1072 * SR ... EQ ... DQ ... CI ... ER
1073 * SR ... DQ ... CI ... EQ ... ER
1074 *
1075 * Request has been handled by DMA if ST and ET is between DQ and CI.
1076 *
1077 * Case 2 - priv_ep->dequeue > current_index
1078 * This situation take place when CI go through the LINK TRB at the end of
1079 * transfer ring.
1080 * SR ... CI ... EQ ... DQ ... ER
1081 *
1082 * Request has been handled by DMA if ET is less then CI or
1083 * ET is greater or equal DQ.
1084 */
1085static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
1086 struct cdns3_request *priv_req)
1087{
1088 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1089 struct cdns3_trb *trb = priv_req->trb;
1090 int current_index = 0;
1091 int handled = 0;
1092 int doorbell;
1093
1094 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1095 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1096
1097 trb = &priv_ep->trb_pool[priv_req->start_trb];
1098
1099 if ((trb->control & TRB_CYCLE) != priv_ep->ccs)
1100 goto finish;
1101
1102 if (doorbell == 1 && current_index == priv_ep->dequeue)
1103 goto finish;
1104
1105 /* The corner case for TRBS_PER_SEGMENT equal 2). */
1106 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1107 handled = 1;
1108 goto finish;
1109 }
1110
1111 if (priv_ep->enqueue == priv_ep->dequeue &&
1112 priv_ep->free_trbs == 0) {
1113 handled = 1;
1114 } else if (priv_ep->dequeue < current_index) {
1115 if ((current_index == (priv_ep->num_trbs - 1)) &&
1116 !priv_ep->dequeue)
1117 goto finish;
1118
1119 if (priv_req->end_trb >= priv_ep->dequeue &&
1120 priv_req->end_trb < current_index)
1121 handled = 1;
1122 } else if (priv_ep->dequeue > current_index) {
1123 if (priv_req->end_trb < current_index ||
1124 priv_req->end_trb >= priv_ep->dequeue)
1125 handled = 1;
1126 }
1127
1128finish:
1129 trace_cdns3_request_handled(priv_req, current_index, handled);
1130
1131 return handled;
1132}
1133
1134static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1135 struct cdns3_endpoint *priv_ep)
1136{
1137 struct cdns3_request *priv_req;
1138 struct usb_request *request;
1139 struct cdns3_trb *trb;
1140
1141 while (!list_empty(&priv_ep->pending_req_list)) {
1142 request = cdns3_next_request(&priv_ep->pending_req_list);
1143 priv_req = to_cdns3_request(request);
1144
1145 /* Re-select endpoint. It could be changed by other CPU during
1146 * handling usb_gadget_giveback_request.
1147 */
1148#ifndef __UBOOT__
1149 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1150#else
1151 cdns3_select_ep(priv_dev,
1152 priv_ep->endpoint.desc->bEndpointAddress);
1153#endif
1154
1155 if (!cdns3_request_handled(priv_ep, priv_req))
1156 goto prepare_next_td;
1157
1158 trb = priv_ep->trb_pool + priv_ep->dequeue;
1159 trace_cdns3_complete_trb(priv_ep, trb);
1160
1161 if (trb != priv_req->trb)
1162 dev_warn(priv_dev->dev,
1163 "request_trb=0x%p, queue_trb=0x%p\n",
1164 priv_req->trb, trb);
1165
1166 request->actual = TRB_LEN(le32_to_cpu(trb->length));
1167 cdns3_move_deq_to_next_trb(priv_req);
1168 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1169
1170 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1171 TRBS_PER_SEGMENT == 2)
1172 break;
1173 }
1174 priv_ep->flags &= ~EP_PENDING_REQUEST;
1175
1176prepare_next_td:
1177 if (!(priv_ep->flags & EP_STALLED) &&
1178 !(priv_ep->flags & EP_STALL_PENDING))
1179 cdns3_start_all_request(priv_dev, priv_ep);
1180}
1181
1182void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1183{
1184 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1185
1186 cdns3_wa1_restore_cycle_bit(priv_ep);
1187
1188 if (rearm) {
1189 trace_cdns3_ring(priv_ep);
1190
1191 /* Cycle Bit must be updated before arming DMA. */
1192 dmb();
1193 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1194
1195 __cdns3_gadget_wakeup(priv_dev);
1196
1197 trace_cdns3_doorbell_epx(priv_ep->name,
1198 readl(&priv_dev->regs->ep_traddr));
1199 }
1200}
1201
1202/**
1203 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1204 * @priv_ep: endpoint object
1205 *
1206 * Returns 0
1207 */
1208static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1209{
1210 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1211 u32 ep_sts_reg;
1212
1213#ifndef __UBOOT__
1214 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1215#else
1216 cdns3_select_ep(priv_dev, priv_ep->endpoint.desc->bEndpointAddress);
1217#endif
1218
1219 trace_cdns3_epx_irq(priv_dev, priv_ep);
1220
1221 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1222 writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1223
1224 if (ep_sts_reg & EP_STS_TRBERR) {
1225 if (priv_ep->flags & EP_STALL_PENDING &&
1226 !(ep_sts_reg & EP_STS_DESCMIS &&
1227 priv_dev->dev_ver < DEV_VER_V2)) {
1228 cdns3_ep_stall_flush(priv_ep);
1229 }
1230
1231 /*
1232 * For isochronous transfer driver completes request on
1233 * IOC or on TRBERR. IOC appears only when device receive
1234 * OUT data packet. If host disable stream or lost some packet
1235 * then the only way to finish all queued transfer is to do it
1236 * on TRBERR event.
1237 */
1238 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1239 !priv_ep->wa1_set) {
1240 if (!priv_ep->dir) {
1241 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1242
1243 ep_cfg &= ~EP_CFG_ENABLE;
1244 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1245 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1246 }
1247 cdns3_transfer_completed(priv_dev, priv_ep);
1248 } else if (!(priv_ep->flags & EP_STALLED) &&
1249 !(priv_ep->flags & EP_STALL_PENDING)) {
1250 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1251 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1252 cdns3_start_all_request(priv_dev, priv_ep);
1253 } else {
1254 cdns3_rearm_transfer(priv_ep,
1255 priv_ep->wa1_set);
1256 }
1257 }
1258 }
1259
1260 if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
1261 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1262 if (ep_sts_reg & EP_STS_ISP)
1263 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1264 else
1265 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1266 }
1267
1268 cdns3_transfer_completed(priv_dev, priv_ep);
1269 }
1270
1271 /*
1272 * WA2: this condition should only be meet when
1273 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1274 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1275 * In other cases this interrupt will be disabled/
1276 */
1277 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1278 !(priv_ep->flags & EP_STALLED))
1279 cdns3_wa2_descmissing_packet(priv_ep);
1280
1281 return 0;
1282}
1283
1284static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1285{
1286 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
1287 spin_unlock(&priv_dev->lock);
1288 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1289 spin_lock(&priv_dev->lock);
1290 }
1291}
1292
1293/**
1294 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1295 * @priv_dev: extended gadget object
1296 * @usb_ists: bitmap representation of device's reported interrupts
1297 * (usb_ists register value)
1298 */
1299static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1300 u32 usb_ists)
1301{
1302 int speed = 0;
1303
1304 trace_cdns3_usb_irq(priv_dev, usb_ists);
1305 if (usb_ists & USB_ISTS_L1ENTI) {
1306 /*
1307 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1308 * from L1. To fix it, if any DMA transfer is pending driver
1309 * must starts driving resume signal immediately.
1310 */
1311 if (readl(&priv_dev->regs->drbl))
1312 __cdns3_gadget_wakeup(priv_dev);
1313 }
1314
1315 /* Connection detected */
1316 if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1317 speed = cdns3_get_speed(priv_dev);
1318 priv_dev->gadget.speed = speed;
1319 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1320 cdns3_ep0_config(priv_dev);
1321 }
1322
1323 /* Disconnection detected */
1324 if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1325 cdns3_disconnect_gadget(priv_dev);
1326 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1327 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1328 cdns3_hw_reset_eps_config(priv_dev);
1329 }
1330
1331 if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1332 if (priv_dev->gadget_driver &&
1333 priv_dev->gadget_driver->suspend) {
1334 spin_unlock(&priv_dev->lock);
1335 priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1336 spin_lock(&priv_dev->lock);
1337 }
1338 }
1339
1340 if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1341 if (priv_dev->gadget_driver &&
1342 priv_dev->gadget_driver->resume) {
1343 spin_unlock(&priv_dev->lock);
1344 priv_dev->gadget_driver->resume(&priv_dev->gadget);
1345 spin_lock(&priv_dev->lock);
1346 }
1347 }
1348
1349 /* reset*/
1350 if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1351 if (priv_dev->gadget_driver) {
1352 spin_unlock(&priv_dev->lock);
1353 usb_gadget_udc_reset(&priv_dev->gadget,
1354 priv_dev->gadget_driver);
1355 spin_lock(&priv_dev->lock);
1356
1357 /*read again to check the actual speed*/
1358 speed = cdns3_get_speed(priv_dev);
1359 priv_dev->gadget.speed = speed;
1360 cdns3_hw_reset_eps_config(priv_dev);
1361 cdns3_ep0_config(priv_dev);
1362 }
1363 }
1364}
1365
1366/**
1367 * cdns3_device_irq_handler- interrupt handler for device part of controller
1368 *
1369 * @irq: irq number for cdns3 core device
1370 * @data: structure of cdns3
1371 *
1372 * Returns IRQ_HANDLED or IRQ_NONE
1373 */
1374static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1375{
1376 struct cdns3_device *priv_dev;
1377 struct cdns3 *cdns = data;
1378 irqreturn_t ret = IRQ_NONE;
1379 u32 reg;
1380
1381 priv_dev = cdns->gadget_dev;
1382
1383 /* check USB device interrupt */
1384 reg = readl(&priv_dev->regs->usb_ists);
1385 if (reg) {
1386 /* After masking interrupts the new interrupts won't be
1387 * reported in usb_ists/ep_ists. In order to not lose some
1388 * of them driver disables only detected interrupts.
1389 * They will be enabled ASAP after clearing source of
1390 * interrupt. This an unusual behavior only applies to
1391 * usb_ists register.
1392 */
1393 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1394 /* mask deferred interrupt. */
1395 writel(reg, &priv_dev->regs->usb_ien);
1396 ret = IRQ_WAKE_THREAD;
1397 }
1398
1399 /* check endpoint interrupt */
1400 reg = readl(&priv_dev->regs->ep_ists);
1401 if (reg) {
1402 writel(0, &priv_dev->regs->ep_ien);
1403 ret = IRQ_WAKE_THREAD;
1404 }
1405
1406 return ret;
1407}
1408
1409/**
1410 * cdns3_device_thread_irq_handler- interrupt handler for device part
1411 * of controller
1412 *
1413 * @irq: irq number for cdns3 core device
1414 * @data: structure of cdns3
1415 *
1416 * Returns IRQ_HANDLED or IRQ_NONE
1417 */
1418static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1419{
1420 struct cdns3_device *priv_dev;
1421 struct cdns3 *cdns = data;
1422 irqreturn_t ret = IRQ_NONE;
1423 unsigned long flags;
1424 int bit;
1425 u32 reg;
1426
1427 priv_dev = cdns->gadget_dev;
1428 spin_lock_irqsave(&priv_dev->lock, flags);
1429
1430 reg = readl(&priv_dev->regs->usb_ists);
1431 if (reg) {
1432 writel(reg, &priv_dev->regs->usb_ists);
1433 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1434 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1435 ret = IRQ_HANDLED;
1436 }
1437
1438 reg = readl(&priv_dev->regs->ep_ists);
1439
1440 /* handle default endpoint OUT */
1441 if (reg & EP_ISTS_EP_OUT0) {
1442 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1443 ret = IRQ_HANDLED;
1444 }
1445
1446 /* handle default endpoint IN */
1447 if (reg & EP_ISTS_EP_IN0) {
1448 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1449 ret = IRQ_HANDLED;
1450 }
1451
1452 /* check if interrupt from non default endpoint, if no exit */
1453 reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1454 if (!reg)
1455 goto irqend;
1456
1457 for_each_set_bit(bit, (unsigned long *)&reg,
1458 sizeof(u32) * BITS_PER_BYTE) {
1459 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1460 ret = IRQ_HANDLED;
1461 }
1462
1463irqend:
1464 writel(~0, &priv_dev->regs->ep_ien);
1465 spin_unlock_irqrestore(&priv_dev->lock, flags);
1466
1467 return ret;
1468}
1469
1470/**
1471 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1472 *
1473 * The real reservation will occur during write to EP_CFG register,
1474 * this function is used to check if the 'size' reservation is allowed.
1475 *
1476 * @priv_dev: extended gadget object
1477 * @size: the size (KB) for EP would like to allocate
1478 * @is_in: endpoint direction
1479 *
1480 * Return 0 if the required size can met or negative value on failure
1481 */
1482static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1483 int size, int is_in)
1484{
1485 int remained;
1486
1487 /* 2KB are reserved for EP0*/
1488 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1489
1490 if (is_in) {
1491 if (remained < size)
1492 return -EPERM;
1493
1494 priv_dev->onchip_used_size += size;
1495 } else {
1496 int required;
1497
1498 /**
1499 * ALL OUT EPs are shared the same chunk onchip memory, so
1500 * driver checks if it already has assigned enough buffers
1501 */
1502 if (priv_dev->out_mem_is_allocated >= size)
1503 return 0;
1504
1505 required = size - priv_dev->out_mem_is_allocated;
1506
1507 if (required > remained)
1508 return -EPERM;
1509
1510 priv_dev->out_mem_is_allocated += required;
1511 priv_dev->onchip_used_size += required;
1512 }
1513
1514 return 0;
1515}
1516
1517void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1518 struct cdns3_endpoint *priv_ep)
1519{
1520 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1521
1522 /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1523 if (priv_dev->dev_ver <= DEV_VER_V2)
1524 writel(USB_CONF_DMULT, &regs->usb_conf);
1525
1526 if (priv_dev->dev_ver == DEV_VER_V2)
1527 writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
1528
1529 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
1530 u32 mask;
1531
1532 if (priv_ep->dir)
1533 mask = BIT(priv_ep->num + 16);
1534 else
1535 mask = BIT(priv_ep->num);
1536
1537 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1538 cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1539 cdns3_set_register_bit(&regs->tdl_beh, mask);
1540 cdns3_set_register_bit(&regs->tdl_beh2, mask);
1541 cdns3_set_register_bit(&regs->dma_adv_td, mask);
1542 }
1543
1544 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1545 cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1546
1547 cdns3_set_register_bit(&regs->dtrans, mask);
1548 }
1549}
1550
1551/**
1552 * cdns3_ep_config Configure hardware endpoint
1553 * @priv_ep: extended endpoint object
1554 */
1555void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
1556{
1557 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
1558 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1559 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
1560 u32 max_packet_size = 0;
1561 u8 maxburst = 0;
1562 u32 ep_cfg = 0;
1563 u8 buffering;
1564 u8 mult = 0;
1565 int ret;
1566
1567 buffering = CDNS3_EP_BUF_SIZE - 1;
1568
1569 cdns3_configure_dmult(priv_dev, priv_ep);
1570
1571 switch (priv_ep->type) {
1572 case USB_ENDPOINT_XFER_INT:
1573 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
1574
1575 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1576 priv_dev->dev_ver > DEV_VER_V2)
1577 ep_cfg |= EP_CFG_TDL_CHK;
1578 break;
1579 case USB_ENDPOINT_XFER_BULK:
1580 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
1581
1582 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1583 priv_dev->dev_ver > DEV_VER_V2)
1584 ep_cfg |= EP_CFG_TDL_CHK;
1585 break;
1586 default:
1587 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
1588 mult = CDNS3_EP_ISO_HS_MULT - 1;
1589 buffering = mult + 1;
1590 }
1591
1592 switch (priv_dev->gadget.speed) {
1593 case USB_SPEED_FULL:
1594 max_packet_size = is_iso_ep ? 1023 : 64;
1595 break;
1596 case USB_SPEED_HIGH:
1597 max_packet_size = is_iso_ep ? 1024 : 512;
1598 break;
1599 case USB_SPEED_SUPER:
1600 /* It's limitation that driver assumes in driver. */
1601 mult = 0;
1602 max_packet_size = 1024;
1603 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1604 maxburst = CDNS3_EP_ISO_SS_BURST - 1;
1605 buffering = (mult + 1) *
1606 (maxburst + 1);
1607
1608 if (priv_ep->interval > 1)
1609 buffering++;
1610 } else {
1611 maxburst = CDNS3_EP_BUF_SIZE - 1;
1612 }
1613 break;
1614 default:
1615 /* all other speed are not supported */
1616 return;
1617 }
1618
1619 if (max_packet_size == 1024)
1620 priv_ep->trb_burst_size = 128;
1621 else if (max_packet_size >= 512)
1622 priv_ep->trb_burst_size = 64;
1623 else
1624 priv_ep->trb_burst_size = 16;
1625
1626 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
1627 !!priv_ep->dir);
1628 if (ret) {
1629 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
1630 return;
1631 }
1632
1633 ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
1634 EP_CFG_MULT(mult) |
1635 EP_CFG_BUFFERING(buffering) |
1636 EP_CFG_MAXBURST(maxburst);
1637
1638 cdns3_select_ep(priv_dev, bEndpointAddress);
1639 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1640
1641 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
1642 priv_ep->name, ep_cfg);
1643}
1644
1645/* Find correct direction for HW endpoint according to description */
1646static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
1647 struct cdns3_endpoint *priv_ep)
1648{
1649 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
1650 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
1651}
1652
1653static struct
1654cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
1655 struct usb_endpoint_descriptor *desc)
1656{
1657 struct usb_ep *ep;
1658 struct cdns3_endpoint *priv_ep;
1659
1660 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1661 unsigned long num;
1662 /* ep name pattern likes epXin or epXout */
1663 char c[2] = {ep->name[2], '\0'};
1664
Simon Glassff9b9032021-07-24 09:03:30 -06001665 num = dectoul(c, NULL);
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +05301666
1667 priv_ep = ep_to_cdns3_ep(ep);
1668 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
1669 if (!(priv_ep->flags & EP_CLAIMED)) {
1670 priv_ep->num = num;
1671 return priv_ep;
1672 }
1673 }
1674 }
1675
1676 return ERR_PTR(-ENOENT);
1677}
1678
1679/*
1680 * Cadence IP has one limitation that all endpoints must be configured
1681 * (Type & MaxPacketSize) before setting configuration through hardware
1682 * register, it means we can't change endpoints configuration after
1683 * set_configuration.
1684 *
1685 * This function set EP_CLAIMED flag which is added when the gadget driver
1686 * uses usb_ep_autoconfig to configure specific endpoint;
1687 * When the udc driver receives set_configurion request,
1688 * it goes through all claimed endpoints, and configure all endpoints
1689 * accordingly.
1690 *
1691 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
1692 * ep_cfg register which can be changed after set_configuration, and do
1693 * some software operation accordingly.
1694 */
1695static struct
1696usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
1697 struct usb_endpoint_descriptor *desc,
1698 struct usb_ss_ep_comp_descriptor *comp_desc)
1699{
1700 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1701 struct cdns3_endpoint *priv_ep;
1702 unsigned long flags;
1703
1704 priv_ep = cdns3_find_available_ep(priv_dev, desc);
1705 if (IS_ERR(priv_ep)) {
1706 dev_err(priv_dev->dev, "no available ep\n");
1707 return NULL;
1708 }
1709
1710 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
1711
1712 spin_lock_irqsave(&priv_dev->lock, flags);
1713 priv_ep->endpoint.desc = desc;
1714 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
1715 priv_ep->type = usb_endpoint_type(desc);
1716 priv_ep->flags |= EP_CLAIMED;
1717 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1718
1719 spin_unlock_irqrestore(&priv_dev->lock, flags);
1720 return &priv_ep->endpoint;
1721}
1722
1723/**
1724 * cdns3_gadget_ep_alloc_request Allocates request
1725 * @ep: endpoint object associated with request
1726 * @gfp_flags: gfp flags
1727 *
1728 * Returns allocated request address, NULL on allocation error
1729 */
1730struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
1731 gfp_t gfp_flags)
1732{
1733 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1734 struct cdns3_request *priv_req;
1735
1736 priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
1737 if (!priv_req)
1738 return NULL;
1739
1740 priv_req->priv_ep = priv_ep;
1741
1742 trace_cdns3_alloc_request(priv_req);
1743 return &priv_req->request;
1744}
1745
1746/**
1747 * cdns3_gadget_ep_free_request Free memory occupied by request
1748 * @ep: endpoint object associated with request
1749 * @request: request to free memory
1750 */
1751void cdns3_gadget_ep_free_request(struct usb_ep *ep,
1752 struct usb_request *request)
1753{
1754 struct cdns3_request *priv_req = to_cdns3_request(request);
1755
1756 if (priv_req->aligned_buf)
1757 priv_req->aligned_buf->in_use = 0;
1758
1759 trace_cdns3_free_request(priv_req);
1760 kfree(priv_req);
1761}
1762
1763/**
1764 * cdns3_gadget_ep_enable Enable endpoint
1765 * @ep: endpoint object
1766 * @desc: endpoint descriptor
1767 *
1768 * Returns 0 on success, error code elsewhere
1769 */
1770static int cdns3_gadget_ep_enable(struct usb_ep *ep,
1771 const struct usb_endpoint_descriptor *desc)
1772{
1773 struct cdns3_endpoint *priv_ep;
1774 struct cdns3_device *priv_dev;
1775 u32 reg = EP_STS_EN_TRBERREN;
1776 u32 bEndpointAddress;
1777 unsigned long flags;
1778 int enable = 1;
1779 int ret;
1780 int val;
1781
1782 priv_ep = ep_to_cdns3_ep(ep);
1783 priv_dev = priv_ep->cdns3_dev;
1784
1785 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1786 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
1787 return -EINVAL;
1788 }
1789
1790 if (!desc->wMaxPacketSize) {
1791 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
1792 return -EINVAL;
1793 }
1794
1795 if (WARN_ON(priv_ep->flags & EP_ENABLED))
1796 return 0;
1797
1798 spin_lock_irqsave(&priv_dev->lock, flags);
1799
1800 priv_ep->endpoint.desc = desc;
1801 priv_ep->type = usb_endpoint_type(desc);
1802 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1803
1804 if (priv_ep->interval > ISO_MAX_INTERVAL &&
1805 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1806 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
1807 ISO_MAX_INTERVAL);
1808
1809 ret = -EINVAL;
1810 goto exit;
1811 }
1812
1813 ret = cdns3_allocate_trb_pool(priv_ep);
1814
1815 if (ret)
1816 goto exit;
1817
1818 bEndpointAddress = priv_ep->num | priv_ep->dir;
1819 cdns3_select_ep(priv_dev, bEndpointAddress);
1820
1821 trace_cdns3_gadget_ep_enable(priv_ep);
1822
1823 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1824
1825 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1826 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1827 1000);
1828
1829 if (unlikely(ret)) {
1830 cdns3_free_trb_pool(priv_ep);
1831 ret = -EINVAL;
1832 goto exit;
1833 }
1834
1835 /* enable interrupt for selected endpoint */
1836 cdns3_set_register_bit(&priv_dev->regs->ep_ien,
1837 BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
1838
1839 if (priv_dev->dev_ver < DEV_VER_V2)
1840 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
1841
1842 writel(reg, &priv_dev->regs->ep_sts_en);
1843
1844 /*
1845 * For some versions of controller at some point during ISO OUT traffic
1846 * DMA reads Transfer Ring for the EP which has never got doorbell.
1847 * This issue was detected only on simulation, but to avoid this issue
1848 * driver add protection against it. To fix it driver enable ISO OUT
1849 * endpoint before setting DRBL. This special treatment of ISO OUT
1850 * endpoints are recommended by controller specification.
1851 */
1852 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1853 enable = 0;
1854
1855 if (enable)
1856 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
1857
1858 ep->desc = desc;
1859 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
1860 EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
1861 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
1862 priv_ep->wa1_set = 0;
1863 priv_ep->enqueue = 0;
1864 priv_ep->dequeue = 0;
1865 reg = readl(&priv_dev->regs->ep_sts);
1866 priv_ep->pcs = !!EP_STS_CCS(reg);
1867 priv_ep->ccs = !!EP_STS_CCS(reg);
1868 /* one TRB is reserved for link TRB used in DMULT mode*/
1869 priv_ep->free_trbs = priv_ep->num_trbs - 1;
1870exit:
1871 spin_unlock_irqrestore(&priv_dev->lock, flags);
1872
1873 return ret;
1874}
1875
1876/**
1877 * cdns3_gadget_ep_disable Disable endpoint
1878 * @ep: endpoint object
1879 *
1880 * Returns 0 on success, error code elsewhere
1881 */
1882static int cdns3_gadget_ep_disable(struct usb_ep *ep)
1883{
1884 struct cdns3_endpoint *priv_ep;
1885 struct cdns3_request *priv_req;
1886 struct cdns3_device *priv_dev;
1887 struct usb_request *request;
1888 unsigned long flags;
1889 int ret = 0;
1890 u32 ep_cfg;
1891 int val;
1892
1893 if (!ep) {
1894 pr_err("usbss: invalid parameters\n");
1895 return -EINVAL;
1896 }
1897
1898 priv_ep = ep_to_cdns3_ep(ep);
1899 priv_dev = priv_ep->cdns3_dev;
1900
1901 if (WARN_ON(!(priv_ep->flags & EP_ENABLED)))
1902 return 0;
1903
1904 spin_lock_irqsave(&priv_dev->lock, flags);
1905
1906 trace_cdns3_gadget_ep_disable(priv_ep);
1907
1908 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
1909
1910 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1911 ep_cfg &= ~EP_CFG_ENABLE;
1912 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1913
1914 /**
1915 * Driver needs some time before resetting endpoint.
1916 * It need waits for clearing DBUSY bit or for timeout expired.
1917 * 10us is enough time for controller to stop transfer.
1918 */
1919 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
1920 !(val & EP_STS_DBUSY), 10);
1921 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1922
1923 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1924 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1925 1000);
1926 if (unlikely(ret))
1927 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
1928 priv_ep->name);
1929
1930 while (!list_empty(&priv_ep->pending_req_list)) {
1931 request = cdns3_next_request(&priv_ep->pending_req_list);
1932
1933 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1934 -ESHUTDOWN);
1935 }
1936
1937 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
1938 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
1939
1940 kfree(priv_req->request.buf);
1941 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
1942 &priv_req->request);
1943 list_del_init(&priv_req->list);
1944 --priv_ep->wa2_counter;
1945 }
1946
1947 while (!list_empty(&priv_ep->deferred_req_list)) {
1948 request = cdns3_next_request(&priv_ep->deferred_req_list);
1949
1950 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1951 -ESHUTDOWN);
1952 }
1953
1954 priv_ep->descmis_req = NULL;
1955
1956 ep->desc = NULL;
1957 priv_ep->flags &= ~EP_ENABLED;
1958
1959 spin_unlock_irqrestore(&priv_dev->lock, flags);
1960
1961 return ret;
1962}
1963
1964/**
1965 * cdns3_gadget_ep_queue Transfer data on endpoint
1966 * @ep: endpoint object
1967 * @request: request object
1968 * @gfp_flags: gfp flags
1969 *
1970 * Returns 0 on success, error code elsewhere
1971 */
1972static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
1973 struct usb_request *request,
1974 gfp_t gfp_flags)
1975{
1976 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1977 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1978 struct cdns3_request *priv_req;
1979 int ret = 0;
1980
1981 request->actual = 0;
1982 request->status = -EINPROGRESS;
1983 priv_req = to_cdns3_request(request);
1984 trace_cdns3_ep_queue(priv_req);
1985
1986 if (priv_dev->dev_ver < DEV_VER_V2) {
1987 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
1988 priv_req);
1989
1990 if (ret == EINPROGRESS)
1991 return 0;
1992 }
1993
1994 ret = cdns3_prepare_aligned_request_buf(priv_req);
1995 if (ret < 0)
1996 return ret;
1997
1998 ret = usb_gadget_map_request(&priv_dev->gadget, request,
1999 usb_endpoint_dir_in(ep->desc));
2000 if (ret)
2001 return ret;
2002
2003 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2004
2005 /*
2006 * If hardware endpoint configuration has not been set yet then
2007 * just queue request in deferred list. Transfer will be started in
2008 * cdns3_set_hw_configuration.
2009 */
2010 if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
2011 !(priv_ep->flags & EP_STALL_PENDING))
2012 cdns3_start_all_request(priv_dev, priv_ep);
2013
2014 return 0;
2015}
2016
2017static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2018 gfp_t gfp_flags)
2019{
2020 struct usb_request *zlp_request;
2021 struct cdns3_endpoint *priv_ep;
2022 struct cdns3_device *priv_dev;
2023 unsigned long flags;
2024 int ret;
2025
2026 if (!request || !ep)
2027 return -EINVAL;
2028
2029 priv_ep = ep_to_cdns3_ep(ep);
2030 priv_dev = priv_ep->cdns3_dev;
2031
2032 spin_lock_irqsave(&priv_dev->lock, flags);
2033
2034 ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2035
2036 if (ret == 0 && request->zero && request->length &&
2037 (request->length % ep->maxpacket == 0)) {
2038 struct cdns3_request *priv_req;
2039
2040 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2041 zlp_request->buf = priv_dev->zlp_buf;
2042 zlp_request->length = 0;
2043
2044 priv_req = to_cdns3_request(zlp_request);
2045 priv_req->flags |= REQUEST_ZLP;
2046
2047 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2048 priv_ep->name);
2049 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2050 }
2051
2052 spin_unlock_irqrestore(&priv_dev->lock, flags);
2053 return ret;
2054}
2055
2056/**
2057 * cdns3_gadget_ep_dequeue Remove request from transfer queue
2058 * @ep: endpoint object associated with request
2059 * @request: request object
2060 *
2061 * Returns 0 on success, error code elsewhere
2062 */
2063int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2064 struct usb_request *request)
2065{
2066 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2067 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2068 struct usb_request *req, *req_temp;
2069 struct cdns3_request *priv_req;
2070 struct cdns3_trb *link_trb;
2071 unsigned long flags;
2072 int ret = 0;
2073
2074 if (!ep || !request || !ep->desc)
2075 return -EINVAL;
2076
2077 spin_lock_irqsave(&priv_dev->lock, flags);
2078
2079 priv_req = to_cdns3_request(request);
2080
2081 trace_cdns3_ep_dequeue(priv_req);
2082
2083 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2084
2085 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2086 list) {
2087 if (request == req)
2088 goto found;
2089 }
2090
2091 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2092 list) {
2093 if (request == req)
2094 goto found;
2095 }
2096
2097 goto not_found;
2098
2099found:
2100
2101 if (priv_ep->wa1_trb == priv_req->trb)
2102 cdns3_wa1_restore_cycle_bit(priv_ep);
2103
2104 link_trb = priv_req->trb;
2105 cdns3_move_deq_to_next_trb(priv_req);
2106 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2107
2108 /* Update ring */
2109 request = cdns3_next_request(&priv_ep->deferred_req_list);
2110 if (request) {
2111 priv_req = to_cdns3_request(request);
2112
2113 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
2114 (priv_req->start_trb * TRB_SIZE));
2115 link_trb->control = (link_trb->control & TRB_CYCLE) |
2116 TRB_TYPE(TRB_LINK) | TRB_CHAIN | TRB_TOGGLE;
2117 } else {
2118 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
2119 }
2120
2121not_found:
2122 spin_unlock_irqrestore(&priv_dev->lock, flags);
2123 return ret;
2124}
2125
2126/**
2127 * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
2128 * Should be called after acquiring spin_lock and selecting ep
2129 * @ep: endpoint object to set stall on.
2130 */
2131void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2132{
2133 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2134
2135 trace_cdns3_halt(priv_ep, 1, 0);
2136
2137 if (!(priv_ep->flags & EP_STALLED)) {
2138 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2139
2140 if (!(ep_sts_reg & EP_STS_DBUSY))
2141 cdns3_ep_stall_flush(priv_ep);
2142 else
2143 priv_ep->flags |= EP_STALL_PENDING;
2144 }
2145}
2146
2147/**
2148 * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
2149 * Should be called after acquiring spin_lock and selecting ep
2150 * @ep: endpoint object to clear stall on
2151 */
2152int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2153{
2154 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2155 struct usb_request *request;
2156 int ret = 0;
2157 int val;
2158
2159 trace_cdns3_halt(priv_ep, 0, 0);
2160
2161 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2162
2163 /* wait for EPRST cleared */
2164 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2165 !(val & EP_CMD_EPRST), 100);
2166 if (ret)
2167 return -EINVAL;
2168
2169 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2170
2171 request = cdns3_next_request(&priv_ep->pending_req_list);
2172
2173 if (request)
2174 cdns3_rearm_transfer(priv_ep, 1);
2175
2176 cdns3_start_all_request(priv_dev, priv_ep);
2177 return ret;
2178}
2179
2180/**
2181 * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
2182 * @ep: endpoint object to set/clear stall on
2183 * @value: 1 for set stall, 0 for clear stall
2184 *
2185 * Returns 0 on success, error code elsewhere
2186 */
2187int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2188{
2189 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2190 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2191 unsigned long flags;
2192 int ret = 0;
2193
2194 if (!(priv_ep->flags & EP_ENABLED))
2195 return -EPERM;
2196
2197 spin_lock_irqsave(&priv_dev->lock, flags);
2198
2199 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2200
2201 if (!value) {
2202 priv_ep->flags &= ~EP_WEDGE;
2203 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2204 } else {
2205 __cdns3_gadget_ep_set_halt(priv_ep);
2206 }
2207
2208 spin_unlock_irqrestore(&priv_dev->lock, flags);
2209
2210 return ret;
2211}
2212
2213extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2214
2215static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2216 .enable = cdns3_gadget_ep_enable,
2217 .disable = cdns3_gadget_ep_disable,
2218 .alloc_request = cdns3_gadget_ep_alloc_request,
2219 .free_request = cdns3_gadget_ep_free_request,
2220 .queue = cdns3_gadget_ep_queue,
2221 .dequeue = cdns3_gadget_ep_dequeue,
2222 .set_halt = cdns3_gadget_ep_set_halt,
2223 .set_wedge = cdns3_gadget_ep_set_wedge,
2224};
2225
2226/**
2227 * cdns3_gadget_get_frame Returns number of actual ITP frame
2228 * @gadget: gadget object
2229 *
2230 * Returns number of actual ITP frame
2231 */
2232static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2233{
2234 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2235
2236 return readl(&priv_dev->regs->usb_itpn);
2237}
2238
2239int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2240{
2241 enum usb_device_speed speed;
2242
2243 speed = cdns3_get_speed(priv_dev);
2244
2245 if (speed >= USB_SPEED_SUPER)
2246 return 0;
2247
2248 /* Start driving resume signaling to indicate remote wakeup. */
2249 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2250
2251 return 0;
2252}
2253
2254static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2255{
2256 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2257 unsigned long flags;
2258 int ret = 0;
2259
2260 spin_lock_irqsave(&priv_dev->lock, flags);
2261 ret = __cdns3_gadget_wakeup(priv_dev);
2262 spin_unlock_irqrestore(&priv_dev->lock, flags);
2263 return ret;
2264}
2265
2266static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2267 int is_selfpowered)
2268{
2269 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2270 unsigned long flags;
2271
2272 spin_lock_irqsave(&priv_dev->lock, flags);
2273 priv_dev->is_selfpowered = !!is_selfpowered;
2274 spin_unlock_irqrestore(&priv_dev->lock, flags);
2275 return 0;
2276}
2277
2278static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2279{
2280 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2281
2282 if (is_on)
2283 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2284 else
2285 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2286
2287 return 0;
2288}
2289
2290static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2291{
2292 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2293 u32 reg;
2294
2295 cdns3_ep0_config(priv_dev);
2296
2297 /* enable interrupts for endpoint 0 (in and out) */
2298 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2299
2300 /*
2301 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2302 * revision of controller.
2303 */
2304 if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2305 reg = readl(&regs->dbg_link1);
2306
2307 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2308 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2309 DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2310 writel(reg, &regs->dbg_link1);
2311 }
2312
2313 /*
2314 * By default some platforms has set protected access to memory.
2315 * This cause problem with cache, so driver restore non-secure
2316 * access to memory.
2317 */
2318 reg = readl(&regs->dma_axi_ctrl);
2319 reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2320 DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2321 writel(reg, &regs->dma_axi_ctrl);
2322
2323 /* enable generic interrupt*/
2324 writel(USB_IEN_INIT, &regs->usb_ien);
2325 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2326
2327 cdns3_configure_dmult(priv_dev, NULL);
2328
2329 cdns3_gadget_pullup(&priv_dev->gadget, 1);
2330}
2331
2332/**
2333 * cdns3_gadget_udc_start Gadget start
2334 * @gadget: gadget object
2335 * @driver: driver which operates on this gadget
2336 *
2337 * Returns 0 on success, error code elsewhere
2338 */
2339static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2340 struct usb_gadget_driver *driver)
2341{
2342 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2343 unsigned long flags;
2344
2345 spin_lock_irqsave(&priv_dev->lock, flags);
2346 priv_dev->gadget_driver = driver;
Ravi Gunasekarane74da732023-07-19 14:29:08 +05302347 cdns3_gadget_udc_set_speed(gadget, gadget->max_speed);
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +05302348 cdns3_gadget_config(priv_dev);
2349 spin_unlock_irqrestore(&priv_dev->lock, flags);
2350 return 0;
2351}
2352
2353/**
2354 * cdns3_gadget_udc_stop Stops gadget
2355 * @gadget: gadget object
2356 *
2357 * Returns 0
2358 */
2359static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2360{
2361 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2362 struct cdns3_endpoint *priv_ep;
2363 u32 bEndpointAddress;
2364 struct usb_ep *ep;
2365 int ret = 0;
2366 int val;
2367
2368 priv_dev->gadget_driver = NULL;
2369
2370 priv_dev->onchip_used_size = 0;
2371 priv_dev->out_mem_is_allocated = 0;
2372 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2373
2374 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2375 priv_ep = ep_to_cdns3_ep(ep);
2376 bEndpointAddress = priv_ep->num | priv_ep->dir;
2377 cdns3_select_ep(priv_dev, bEndpointAddress);
2378 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2379 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2380 !(val & EP_CMD_EPRST), 100);
2381 }
2382
2383 /* disable interrupt for device */
2384 writel(0, &priv_dev->regs->usb_ien);
2385 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2386
2387 return ret;
2388}
2389
Vignesh Raghavendrac13e14f2019-10-01 17:26:34 +05302390static void cdns3_gadget_udc_set_speed(struct usb_gadget *gadget,
2391 enum usb_device_speed speed)
2392{
2393 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2394
2395 switch (speed) {
2396 case USB_SPEED_FULL:
2397 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2398 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2399 break;
2400 case USB_SPEED_HIGH:
2401 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2402 break;
2403 case USB_SPEED_SUPER:
2404 break;
2405 default:
Sean Andersonccf123b2020-09-15 10:45:14 -04002406 dev_err(priv_dev->dev, "invalid speed parameter %d\n", speed);
Vignesh Raghavendrac13e14f2019-10-01 17:26:34 +05302407 }
2408
2409 priv_dev->gadget.speed = speed;
2410}
2411
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +05302412static const struct usb_gadget_ops cdns3_gadget_ops = {
2413 .get_frame = cdns3_gadget_get_frame,
2414 .wakeup = cdns3_gadget_wakeup,
2415 .set_selfpowered = cdns3_gadget_set_selfpowered,
2416 .pullup = cdns3_gadget_pullup,
2417 .udc_start = cdns3_gadget_udc_start,
2418 .udc_stop = cdns3_gadget_udc_stop,
2419 .match_ep = cdns3_gadget_match_ep,
Vignesh Raghavendrac13e14f2019-10-01 17:26:34 +05302420 .udc_set_speed = cdns3_gadget_udc_set_speed,
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +05302421};
2422
2423static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2424{
2425 int i;
2426
2427 /* ep0 OUT point to ep0 IN. */
2428 priv_dev->eps[16] = NULL;
2429
2430 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2431 if (priv_dev->eps[i]) {
2432 cdns3_free_trb_pool(priv_dev->eps[i]);
2433 devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2434 }
2435}
2436
2437/**
2438 * cdns3_init_eps Initializes software endpoints of gadget
2439 * @cdns3: extended gadget object
2440 *
2441 * Returns 0 on success, error code elsewhere
2442 */
2443static int cdns3_init_eps(struct cdns3_device *priv_dev)
2444{
2445 u32 ep_enabled_reg, iso_ep_reg;
2446 struct cdns3_endpoint *priv_ep;
2447 int ep_dir, ep_number;
2448 u32 ep_mask;
2449 int ret = 0;
2450 int i;
2451
2452 /* Read it from USB_CAP3 to USB_CAP5 */
2453 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2454 iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2455
2456 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2457
2458 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2459 ep_dir = i >> 4; /* i div 16 */
2460 ep_number = i & 0xF; /* i % 16 */
2461 ep_mask = BIT(i);
2462
2463 if (!(ep_enabled_reg & ep_mask))
2464 continue;
2465
2466 if (ep_dir && !ep_number) {
2467 priv_dev->eps[i] = priv_dev->eps[0];
2468 continue;
2469 }
2470
2471 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
2472 GFP_KERNEL);
2473 if (!priv_ep) {
2474 ret = -ENOMEM;
2475 goto err;
2476 }
2477
2478 /* set parent of endpoint object */
2479 priv_ep->cdns3_dev = priv_dev;
2480 priv_dev->eps[i] = priv_ep;
2481 priv_ep->num = ep_number;
2482 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
2483
2484 if (!ep_number) {
2485 ret = cdns3_init_ep0(priv_dev, priv_ep);
2486 if (ret) {
2487 dev_err(priv_dev->dev, "Failed to init ep0\n");
2488 goto err;
2489 }
2490 } else {
2491 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
2492 ep_number, !!ep_dir ? "in" : "out");
2493 priv_ep->endpoint.name = priv_ep->name;
2494
2495 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
2496 CDNS3_EP_MAX_PACKET_LIMIT);
2497 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
2498 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
2499 if (ep_dir)
2500 priv_ep->endpoint.caps.dir_in = 1;
2501 else
2502 priv_ep->endpoint.caps.dir_out = 1;
2503
2504 if (iso_ep_reg & ep_mask)
2505 priv_ep->endpoint.caps.type_iso = 1;
2506
2507 priv_ep->endpoint.caps.type_bulk = 1;
2508 priv_ep->endpoint.caps.type_int = 1;
2509
2510 list_add_tail(&priv_ep->endpoint.ep_list,
2511 &priv_dev->gadget.ep_list);
2512 }
2513
2514 priv_ep->flags = 0;
2515
2516 dev_info(priv_dev->dev, "Initialized %s support: %s %s\n",
2517 priv_ep->name,
2518 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
2519 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
2520
2521 INIT_LIST_HEAD(&priv_ep->pending_req_list);
2522 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
2523 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
2524 }
2525
2526 return 0;
2527err:
2528 cdns3_free_all_eps(priv_dev);
2529 return -ENOMEM;
2530}
2531
2532void cdns3_gadget_exit(struct cdns3 *cdns)
2533{
2534 struct cdns3_device *priv_dev;
2535
2536 priv_dev = cdns->gadget_dev;
2537
2538 usb_del_gadget_udc(&priv_dev->gadget);
2539
2540 cdns3_free_all_eps(priv_dev);
2541
2542 while (!list_empty(&priv_dev->aligned_buf_list)) {
2543 struct cdns3_aligned_buf *buf;
2544
2545 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
2546 dma_free_coherent(buf->buf);
2547
2548 list_del(&buf->list);
2549 kfree(buf);
2550 }
2551
2552 dma_free_coherent(priv_dev->setup_buf);
2553
2554 kfree(priv_dev->zlp_buf);
2555 kfree(priv_dev);
2556 cdns->gadget_dev = NULL;
2557 cdns3_drd_switch_gadget(cdns, 0);
2558}
2559
2560static int cdns3_gadget_start(struct cdns3 *cdns)
2561{
2562 struct cdns3_device *priv_dev;
2563 u32 max_speed;
2564 int ret;
2565
2566 priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
2567 if (!priv_dev)
2568 return -ENOMEM;
2569
2570 cdns->gadget_dev = priv_dev;
2571 priv_dev->sysdev = cdns->dev;
2572 priv_dev->dev = cdns->dev;
2573 priv_dev->regs = cdns->dev_regs;
2574
2575 dev_read_u32(priv_dev->dev, "cdns,on-chip-buff-size",
2576 &priv_dev->onchip_buffers);
2577
2578 if (priv_dev->onchip_buffers <= 0) {
2579 u32 reg = readl(&priv_dev->regs->usb_cap2);
2580
2581 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
2582 }
2583
2584 if (!priv_dev->onchip_buffers)
2585 priv_dev->onchip_buffers = 256;
2586
Kever Yang1b807052020-03-04 08:59:50 +08002587 max_speed = usb_get_maximum_speed(dev_ofnode(cdns->dev));
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +05302588
2589 /* Check the maximum_speed parameter */
2590 switch (max_speed) {
2591 case USB_SPEED_FULL:
Vignesh Raghavendrac13e14f2019-10-01 17:26:34 +05302592 /* fall through */
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +05302593 case USB_SPEED_HIGH:
Vignesh Raghavendrac13e14f2019-10-01 17:26:34 +05302594 /* fall through */
Vignesh Raghavendrab1a49282019-10-01 17:26:33 +05302595 case USB_SPEED_SUPER:
2596 break;
2597 default:
2598 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
2599 max_speed);
2600 /* fall through */
2601 case USB_SPEED_UNKNOWN:
2602 /* default to superspeed */
2603 max_speed = USB_SPEED_SUPER;
2604 break;
2605 }
2606
2607 /* fill gadget fields */
2608 priv_dev->gadget.max_speed = max_speed;
2609 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2610 priv_dev->gadget.ops = &cdns3_gadget_ops;
2611 priv_dev->gadget.name = "cdns3-gadget";
2612#ifndef __UBOOT__
2613 priv_dev->gadget.name = "usb-ss-gadget";
2614 priv_dev->gadget.sg_supported = 1;
2615 priv_dev->gadget.quirk_avoids_skb_reserve = 1;
2616#endif
2617
2618 spin_lock_init(&priv_dev->lock);
2619 INIT_WORK(&priv_dev->pending_status_wq,
2620 cdns3_pending_setup_status_handler);
2621
2622 /* initialize endpoint container */
2623 INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
2624 INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
2625
2626 ret = cdns3_init_eps(priv_dev);
2627 if (ret) {
2628 dev_err(priv_dev->dev, "Failed to create endpoints\n");
2629 goto err1;
2630 }
2631
2632 /* allocate memory for setup packet buffer */
2633 priv_dev->setup_buf =
2634 dma_alloc_coherent(8, (unsigned long *)&priv_dev->setup_dma);
2635 if (!priv_dev->setup_buf) {
2636 ret = -ENOMEM;
2637 goto err2;
2638 }
2639
2640 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
2641
2642 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
2643 readl(&priv_dev->regs->usb_cap6));
2644 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
2645 readl(&priv_dev->regs->usb_cap1));
2646 dev_dbg(priv_dev->dev, "On-Chip memory cnfiguration: %08x\n",
2647 readl(&priv_dev->regs->usb_cap2));
2648
2649 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
2650
2651 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2652 if (!priv_dev->zlp_buf) {
2653 ret = -ENOMEM;
2654 goto err3;
2655 }
2656
2657 /* add USB gadget device */
2658 ret = usb_add_gadget_udc((struct device *)priv_dev->dev,
2659 &priv_dev->gadget);
2660 if (ret < 0) {
2661 dev_err(priv_dev->dev,
2662 "Failed to register USB device controller\n");
2663 goto err4;
2664 }
2665
2666 return 0;
2667err4:
2668 kfree(priv_dev->zlp_buf);
2669err3:
2670 dma_free_coherent(priv_dev->setup_buf);
2671err2:
2672 cdns3_free_all_eps(priv_dev);
2673err1:
2674 cdns->gadget_dev = NULL;
2675 return ret;
2676}
2677
2678static int __cdns3_gadget_init(struct cdns3 *cdns)
2679{
2680 int ret = 0;
2681
2682 cdns3_drd_switch_gadget(cdns, 1);
2683
2684 ret = cdns3_gadget_start(cdns);
2685 if (ret)
2686 return ret;
2687
2688 return 0;
2689}
2690
2691static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
2692{
2693 struct cdns3_device *priv_dev = cdns->gadget_dev;
2694
2695 cdns3_disconnect_gadget(priv_dev);
2696
2697 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2698 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
2699 cdns3_hw_reset_eps_config(priv_dev);
2700
2701 /* disable interrupt for device */
2702 writel(0, &priv_dev->regs->usb_ien);
2703
2704 cdns3_gadget_pullup(&priv_dev->gadget, 0);
2705
2706 return 0;
2707}
2708
2709static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
2710{
2711 struct cdns3_device *priv_dev = cdns->gadget_dev;
2712
2713 if (!priv_dev->gadget_driver)
2714 return 0;
2715
2716 cdns3_gadget_config(priv_dev);
2717
2718 return 0;
2719}
2720
2721/**
2722 * cdns3_gadget_init - initialize device structure
2723 *
2724 * cdns: cdns3 instance
2725 *
2726 * This function initializes the gadget.
2727 */
2728int cdns3_gadget_init(struct cdns3 *cdns)
2729{
2730 struct cdns3_role_driver *rdrv;
2731
2732 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2733 if (!rdrv)
2734 return -ENOMEM;
2735
2736 rdrv->start = __cdns3_gadget_init;
2737 rdrv->stop = cdns3_gadget_exit;
2738 rdrv->suspend = cdns3_gadget_suspend;
2739 rdrv->resume = cdns3_gadget_resume;
2740 rdrv->state = CDNS3_ROLE_STATE_INACTIVE;
2741 rdrv->name = "gadget";
2742 cdns->roles[USB_ROLE_DEVICE] = rdrv;
2743
2744 return 0;
2745}
2746
2747/**
2748 * cdns3_gadget_uboot_handle_interrupt - handle cdns3 gadget interrupt
2749 * @cdns: pointer to struct cdns3
2750 *
2751 * Handles ep0 and gadget interrupt
2752 */
2753static void cdns3_gadget_uboot_handle_interrupt(struct cdns3 *cdns)
2754{
2755 int ret = cdns3_device_irq_handler(0, cdns);
2756
2757 if (ret == IRQ_WAKE_THREAD)
2758 cdns3_device_thread_irq_handler(0, cdns);
2759}
2760
2761int dm_usb_gadget_handle_interrupts(struct udevice *dev)
2762{
2763 struct cdns3 *cdns = dev_get_priv(dev);
2764
2765 cdns3_gadget_uboot_handle_interrupt(cdns);
2766
2767 return 0;
2768}