1 // SPDX-License-Identifier: GPL-2.0+
3 * Copyright (C) 2018 Álvaro Fernández Rojas <noltari@gmail.com>
5 * Derived from linux/drivers/dma/bcm63xx-iudma.c:
6 * Copyright (C) 2015 Simon Arlott <simon@fire.lp0.eu>
8 * Derived from linux/drivers/net/ethernet/broadcom/bcm63xx_enet.c:
9 * Copyright (C) 2008 Maxime Bizon <mbizon@freebox.fr>
11 * Derived from bcm963xx_4.12L.06B_consumer/shared/opensource/include/bcm963xx/63268_map_part.h:
12 * Copyright (C) 2000-2010 Broadcom Corporation
14 * Derived from bcm963xx_4.12L.06B_consumer/bcmdrivers/opensource/net/enet/impl4/bcmenet.c:
15 * Copyright (C) 2010 Broadcom Corporation
22 #include <dma-uclass.h>
29 #include <linux/bitops.h>
30 #include <linux/delay.h>
36 #define DMA_CHAN_FLOWC(x) ((x) >> 1)
37 #define DMA_CHAN_MAX 16
38 #define DMA_CHAN_SIZE 0x10
39 #define DMA_CHAN_TOUT 500
41 /* DMA Global Configuration register */
42 #define DMA_CFG_REG 0x00
43 #define DMA_CFG_ENABLE_SHIFT 0
44 #define DMA_CFG_ENABLE_MASK (1 << DMA_CFG_ENABLE_SHIFT)
45 #define DMA_CFG_FLOWC_ENABLE(x) BIT(DMA_CHAN_FLOWC(x) + 1)
46 #define DMA_CFG_NCHANS_SHIFT 24
47 #define DMA_CFG_NCHANS_MASK (0xf << DMA_CFG_NCHANS_SHIFT)
49 /* DMA Global Flow Control registers */
50 #define DMA_FLOWC_THR_LO_REG(x) (0x04 + DMA_CHAN_FLOWC(x) * 0x0c)
51 #define DMA_FLOWC_THR_HI_REG(x) (0x08 + DMA_CHAN_FLOWC(x) * 0x0c)
52 #define DMA_FLOWC_ALLOC_REG(x) (0x0c + DMA_CHAN_FLOWC(x) * 0x0c)
53 #define DMA_FLOWC_ALLOC_FORCE_SHIFT 31
54 #define DMA_FLOWC_ALLOC_FORCE_MASK (1 << DMA_FLOWC_ALLOC_FORCE_SHIFT)
56 /* DMA Global Reset register */
57 #define DMA_RST_REG 0x34
58 #define DMA_RST_CHAN_SHIFT 0
59 #define DMA_RST_CHAN_MASK(x) (1 << x)
61 /* DMA Channel Configuration register */
62 #define DMAC_CFG_REG(x) (DMA_CHAN_SIZE * (x) + 0x00)
63 #define DMAC_CFG_ENABLE_SHIFT 0
64 #define DMAC_CFG_ENABLE_MASK (1 << DMAC_CFG_ENABLE_SHIFT)
65 #define DMAC_CFG_PKT_HALT_SHIFT 1
66 #define DMAC_CFG_PKT_HALT_MASK (1 << DMAC_CFG_PKT_HALT_SHIFT)
67 #define DMAC_CFG_BRST_HALT_SHIFT 2
68 #define DMAC_CFG_BRST_HALT_MASK (1 << DMAC_CFG_BRST_HALT_SHIFT)
70 /* DMA Channel Max Burst Length register */
71 #define DMAC_BURST_REG(x) (DMA_CHAN_SIZE * (x) + 0x0c)
73 /* DMA SRAM Descriptor Ring Start register */
74 #define DMAS_RSTART_REG(x) (DMA_CHAN_SIZE * (x) + 0x00)
76 /* DMA SRAM State/Bytes done/ring offset register */
77 #define DMAS_STATE_DATA_REG(x) (DMA_CHAN_SIZE * (x) + 0x04)
79 /* DMA SRAM Buffer Descriptor status and length register */
80 #define DMAS_DESC_LEN_STATUS_REG(x) (DMA_CHAN_SIZE * (x) + 0x08)
82 /* DMA SRAM Buffer Descriptor status and length register */
83 #define DMAS_DESC_BASE_BUFPTR_REG(x) (DMA_CHAN_SIZE * (x) + 0x0c)
85 /* DMA Descriptor Status */
86 #define DMAD_ST_CRC_SHIFT 8
87 #define DMAD_ST_CRC_MASK (1 << DMAD_ST_CRC_SHIFT)
88 #define DMAD_ST_WRAP_SHIFT 12
89 #define DMAD_ST_WRAP_MASK (1 << DMAD_ST_WRAP_SHIFT)
90 #define DMAD_ST_SOP_SHIFT 13
91 #define DMAD_ST_SOP_MASK (1 << DMAD_ST_SOP_SHIFT)
92 #define DMAD_ST_EOP_SHIFT 14
93 #define DMAD_ST_EOP_MASK (1 << DMAD_ST_EOP_SHIFT)
94 #define DMAD_ST_OWN_SHIFT 15
95 #define DMAD_ST_OWN_MASK (1 << DMAD_ST_OWN_SHIFT)
97 #define DMAD6348_ST_OV_ERR_SHIFT 0
98 #define DMAD6348_ST_OV_ERR_MASK (1 << DMAD6348_ST_OV_ERR_SHIFT)
99 #define DMAD6348_ST_CRC_ERR_SHIFT 1
100 #define DMAD6348_ST_CRC_ERR_MASK (1 << DMAD6348_ST_CRC_ERR_SHIFT)
101 #define DMAD6348_ST_RX_ERR_SHIFT 2
102 #define DMAD6348_ST_RX_ERR_MASK (1 << DMAD6348_ST_RX_ERR_SHIFT)
103 #define DMAD6348_ST_OS_ERR_SHIFT 4
104 #define DMAD6348_ST_OS_ERR_MASK (1 << DMAD6348_ST_OS_ERR_SHIFT)
105 #define DMAD6348_ST_UN_ERR_SHIFT 9
106 #define DMAD6348_ST_UN_ERR_MASK (1 << DMAD6348_ST_UN_ERR_SHIFT)
108 struct bcm6348_dma_desc
{
114 struct bcm6348_chan_priv
{
115 void __iomem
*dma_ring
;
116 uint8_t dma_ring_size
;
123 struct bcm6348_iudma_hw
{
127 struct bcm6348_iudma_priv
{
128 const struct bcm6348_iudma_hw
*hw
;
132 struct bcm6348_chan_priv
**ch_priv
;
136 static inline bool bcm6348_iudma_chan_is_rx(uint8_t ch
)
141 static inline void bcm6348_iudma_fdc(void *ptr
, ulong size
)
143 ulong start
= (ulong
) ptr
;
145 flush_dcache_range(start
, start
+ size
);
148 static inline void bcm6348_iudma_idc(void *ptr
, ulong size
)
150 ulong start
= (ulong
) ptr
;
152 invalidate_dcache_range(start
, start
+ size
);
155 static void bcm6348_iudma_chan_stop(struct bcm6348_iudma_priv
*priv
,
158 unsigned int timeout
= DMA_CHAN_TOUT
;
163 if (timeout
> DMA_CHAN_TOUT
/ 2)
164 halt
= DMAC_CFG_PKT_HALT_MASK
;
166 halt
= DMAC_CFG_BRST_HALT_MASK
;
168 /* try to stop dma channel */
169 writel_be(halt
, priv
->chan
+ DMAC_CFG_REG(ch
));
172 /* check if channel was stopped */
173 cfg
= readl_be(priv
->chan
+ DMAC_CFG_REG(ch
));
174 if (!(cfg
& DMAC_CFG_ENABLE_MASK
))
181 pr_err("unable to stop channel %u\n", ch
);
183 /* reset dma channel */
184 setbits_be32(priv
->base
+ DMA_RST_REG
, DMA_RST_CHAN_MASK(ch
));
186 clrbits_be32(priv
->base
+ DMA_RST_REG
, DMA_RST_CHAN_MASK(ch
));
189 static int bcm6348_iudma_disable(struct dma
*dma
)
191 struct bcm6348_iudma_priv
*priv
= dev_get_priv(dma
->dev
);
192 struct bcm6348_chan_priv
*ch_priv
= priv
->ch_priv
[dma
->id
];
194 /* stop dma channel */
195 bcm6348_iudma_chan_stop(priv
, dma
->id
);
197 /* dma flow control */
198 if (bcm6348_iudma_chan_is_rx(dma
->id
))
199 writel_be(DMA_FLOWC_ALLOC_FORCE_MASK
,
200 DMA_FLOWC_ALLOC_REG(dma
->id
));
202 /* init channel config */
203 ch_priv
->running
= false;
204 ch_priv
->desc_id
= 0;
205 if (bcm6348_iudma_chan_is_rx(dma
->id
))
206 ch_priv
->desc_cnt
= 0;
208 ch_priv
->desc_cnt
= ch_priv
->dma_ring_size
;
213 static int bcm6348_iudma_enable(struct dma
*dma
)
215 const struct bcm6348_iudma_priv
*priv
= dev_get_priv(dma
->dev
);
216 struct bcm6348_chan_priv
*ch_priv
= priv
->ch_priv
[dma
->id
];
217 struct bcm6348_dma_desc
*dma_desc
= ch_priv
->dma_ring
;
221 for (i
= 0; i
< ch_priv
->desc_cnt
; i
++) {
222 if (bcm6348_iudma_chan_is_rx(dma
->id
)) {
223 ch_priv
->busy_desc
[i
] = false;
224 dma_desc
->status
|= DMAD_ST_OWN_MASK
;
226 dma_desc
->status
= 0;
227 dma_desc
->length
= 0;
228 dma_desc
->address
= 0;
231 if (i
== ch_priv
->desc_cnt
- 1)
232 dma_desc
->status
|= DMAD_ST_WRAP_MASK
;
237 /* init to first descriptor */
238 ch_priv
->desc_id
= 0;
240 /* force cache writeback */
241 bcm6348_iudma_fdc(ch_priv
->dma_ring
,
242 sizeof(*dma_desc
) * ch_priv
->desc_cnt
);
245 writel_be(0, priv
->sram
+ DMAS_STATE_DATA_REG(dma
->id
));
246 writel_be(0, priv
->sram
+ DMAS_DESC_LEN_STATUS_REG(dma
->id
));
247 writel_be(0, priv
->sram
+ DMAS_DESC_BASE_BUFPTR_REG(dma
->id
));
249 /* set dma ring start */
250 writel_be(virt_to_phys(ch_priv
->dma_ring
),
251 priv
->sram
+ DMAS_RSTART_REG(dma
->id
));
253 /* set flow control */
254 if (bcm6348_iudma_chan_is_rx(dma
->id
)) {
257 setbits_be32(priv
->base
+ DMA_CFG_REG
,
258 DMA_CFG_FLOWC_ENABLE(dma
->id
));
260 val
= ch_priv
->desc_cnt
/ 3;
261 writel_be(val
, priv
->base
+ DMA_FLOWC_THR_LO_REG(dma
->id
));
263 val
= (ch_priv
->desc_cnt
* 2) / 3;
264 writel_be(val
, priv
->base
+ DMA_FLOWC_THR_HI_REG(dma
->id
));
266 writel_be(0, priv
->base
+ DMA_FLOWC_ALLOC_REG(dma
->id
));
269 /* set dma max burst */
270 writel_be(ch_priv
->desc_cnt
,
271 priv
->chan
+ DMAC_BURST_REG(dma
->id
));
273 /* kick rx dma channel */
274 if (bcm6348_iudma_chan_is_rx(dma
->id
))
275 setbits_be32(priv
->chan
+ DMAC_CFG_REG(dma
->id
),
276 DMAC_CFG_ENABLE_MASK
);
278 /* channel is now enabled */
279 ch_priv
->running
= true;
284 static int bcm6348_iudma_request(struct dma
*dma
)
286 const struct bcm6348_iudma_priv
*priv
= dev_get_priv(dma
->dev
);
287 struct bcm6348_chan_priv
*ch_priv
;
289 /* check if channel is valid */
290 if (dma
->id
>= priv
->n_channels
)
293 /* alloc channel private data */
294 priv
->ch_priv
[dma
->id
] = calloc(1, sizeof(struct bcm6348_chan_priv
));
295 if (!priv
->ch_priv
[dma
->id
])
297 ch_priv
= priv
->ch_priv
[dma
->id
];
300 if (bcm6348_iudma_chan_is_rx(dma
->id
))
301 ch_priv
->dma_ring_size
= DMA_RX_DESC
;
303 ch_priv
->dma_ring_size
= DMA_TX_DESC
;
306 malloc_cache_aligned(sizeof(struct bcm6348_dma_desc
) *
307 ch_priv
->dma_ring_size
);
308 if (!ch_priv
->dma_ring
)
311 /* init channel config */
312 ch_priv
->running
= false;
313 ch_priv
->desc_id
= 0;
314 if (bcm6348_iudma_chan_is_rx(dma
->id
)) {
315 ch_priv
->desc_cnt
= 0;
316 ch_priv
->busy_desc
= NULL
;
318 ch_priv
->desc_cnt
= ch_priv
->dma_ring_size
;
319 ch_priv
->busy_desc
= calloc(ch_priv
->desc_cnt
, sizeof(bool));
325 static int bcm6348_iudma_receive(struct dma
*dma
, void **dst
, void *metadata
)
327 const struct bcm6348_iudma_priv
*priv
= dev_get_priv(dma
->dev
);
328 const struct bcm6348_iudma_hw
*hw
= priv
->hw
;
329 struct bcm6348_chan_priv
*ch_priv
= priv
->ch_priv
[dma
->id
];
330 struct bcm6348_dma_desc
*dma_desc
= dma_desc
= ch_priv
->dma_ring
;
333 if (!ch_priv
->running
)
336 /* get dma ring descriptor address */
337 dma_desc
+= ch_priv
->desc_id
;
339 /* invalidate cache data */
340 bcm6348_iudma_idc(dma_desc
, sizeof(*dma_desc
));
343 if (dma_desc
->status
& DMAD_ST_OWN_MASK
)
347 if (!(dma_desc
->status
& DMAD_ST_EOP_MASK
) ||
348 !(dma_desc
->status
& DMAD_ST_SOP_MASK
) ||
349 (dma_desc
->status
& hw
->err_mask
)) {
350 pr_err("invalid pkt received (ch=%ld desc=%u) (st=%04x)\n",
351 dma
->id
, ch_priv
->desc_id
, dma_desc
->status
);
354 /* set dma buffer address */
355 *dst
= phys_to_virt(dma_desc
->address
);
357 /* invalidate cache data */
358 bcm6348_iudma_idc(*dst
, dma_desc
->length
);
360 /* return packet length */
361 ret
= dma_desc
->length
;
364 /* busy dma descriptor */
365 ch_priv
->busy_desc
[ch_priv
->desc_id
] = true;
367 /* increment dma descriptor */
368 ch_priv
->desc_id
= (ch_priv
->desc_id
+ 1) % ch_priv
->desc_cnt
;
373 static int bcm6348_iudma_send(struct dma
*dma
, void *src
, size_t len
,
376 const struct bcm6348_iudma_priv
*priv
= dev_get_priv(dma
->dev
);
377 struct bcm6348_chan_priv
*ch_priv
= priv
->ch_priv
[dma
->id
];
378 struct bcm6348_dma_desc
*dma_desc
;
381 if (!ch_priv
->running
)
385 bcm6348_iudma_fdc(src
, len
);
387 /* get dma ring descriptor address */
388 dma_desc
= ch_priv
->dma_ring
;
389 dma_desc
+= ch_priv
->desc_id
;
391 /* config dma descriptor */
392 status
= (DMAD_ST_OWN_MASK
|
396 if (ch_priv
->desc_id
== ch_priv
->desc_cnt
- 1)
397 status
|= DMAD_ST_WRAP_MASK
;
399 /* set dma descriptor */
400 dma_desc
->address
= virt_to_phys(src
);
401 dma_desc
->length
= len
;
402 dma_desc
->status
= status
;
405 bcm6348_iudma_fdc(dma_desc
, sizeof(*dma_desc
));
407 /* kick tx dma channel */
408 setbits_be32(priv
->chan
+ DMAC_CFG_REG(dma
->id
), DMAC_CFG_ENABLE_MASK
);
410 /* poll dma status */
412 /* invalidate cache */
413 bcm6348_iudma_idc(dma_desc
, sizeof(*dma_desc
));
415 if (!(dma_desc
->status
& DMAD_ST_OWN_MASK
))
419 /* increment dma descriptor */
420 ch_priv
->desc_id
= (ch_priv
->desc_id
+ 1) % ch_priv
->desc_cnt
;
425 static int bcm6348_iudma_free_rcv_buf(struct dma
*dma
, void *dst
, size_t size
)
427 const struct bcm6348_iudma_priv
*priv
= dev_get_priv(dma
->dev
);
428 struct bcm6348_chan_priv
*ch_priv
= priv
->ch_priv
[dma
->id
];
429 struct bcm6348_dma_desc
*dma_desc
= ch_priv
->dma_ring
;
434 /* get dirty dma descriptor */
435 for (i
= 0; i
< ch_priv
->desc_cnt
; i
++) {
436 if (phys_to_virt(dma_desc
->address
) == dst
)
442 /* dma descriptor not found */
443 if (i
== ch_priv
->desc_cnt
) {
444 pr_err("dirty dma descriptor not found\n");
448 /* invalidate cache */
449 bcm6348_iudma_idc(ch_priv
->dma_ring
,
450 sizeof(*dma_desc
) * ch_priv
->desc_cnt
);
452 /* free dma descriptor */
453 ch_priv
->busy_desc
[i
] = false;
455 status
= DMAD_ST_OWN_MASK
;
456 if (i
== ch_priv
->desc_cnt
- 1)
457 status
|= DMAD_ST_WRAP_MASK
;
459 dma_desc
->status
|= status
;
460 dma_desc
->length
= PKTSIZE_ALIGN
;
462 /* tell dma we allocated one buffer */
463 writel_be(1, DMA_FLOWC_ALLOC_REG(dma
->id
));
466 bcm6348_iudma_fdc(ch_priv
->dma_ring
,
467 sizeof(*dma_desc
) * ch_priv
->desc_cnt
);
469 /* kick rx dma channel if disabled */
470 cfg
= readl_be(priv
->chan
+ DMAC_CFG_REG(dma
->id
));
471 if (!(cfg
& DMAC_CFG_ENABLE_MASK
))
472 setbits_be32(priv
->chan
+ DMAC_CFG_REG(dma
->id
),
473 DMAC_CFG_ENABLE_MASK
);
478 static int bcm6348_iudma_add_rcv_buf(struct dma
*dma
, void *dst
, size_t size
)
480 const struct bcm6348_iudma_priv
*priv
= dev_get_priv(dma
->dev
);
481 struct bcm6348_chan_priv
*ch_priv
= priv
->ch_priv
[dma
->id
];
482 struct bcm6348_dma_desc
*dma_desc
= ch_priv
->dma_ring
;
484 /* no more dma descriptors available */
485 if (ch_priv
->desc_cnt
== ch_priv
->dma_ring_size
) {
486 pr_err("max number of buffers reached\n");
490 /* get next dma descriptor */
491 dma_desc
+= ch_priv
->desc_cnt
;
493 /* init dma descriptor */
494 dma_desc
->address
= virt_to_phys(dst
);
495 dma_desc
->length
= size
;
496 dma_desc
->status
= 0;
499 bcm6348_iudma_fdc(dma_desc
, sizeof(*dma_desc
));
501 /* increment dma descriptors */
507 static int bcm6348_iudma_prepare_rcv_buf(struct dma
*dma
, void *dst
,
510 const struct bcm6348_iudma_priv
*priv
= dev_get_priv(dma
->dev
);
511 struct bcm6348_chan_priv
*ch_priv
= priv
->ch_priv
[dma
->id
];
513 /* only add new rx buffers if channel isn't running */
514 if (ch_priv
->running
)
515 return bcm6348_iudma_free_rcv_buf(dma
, dst
, size
);
517 return bcm6348_iudma_add_rcv_buf(dma
, dst
, size
);
520 static const struct dma_ops bcm6348_iudma_ops
= {
521 .disable
= bcm6348_iudma_disable
,
522 .enable
= bcm6348_iudma_enable
,
523 .prepare_rcv_buf
= bcm6348_iudma_prepare_rcv_buf
,
524 .request
= bcm6348_iudma_request
,
525 .receive
= bcm6348_iudma_receive
,
526 .send
= bcm6348_iudma_send
,
529 static const struct bcm6348_iudma_hw bcm6348_hw
= {
530 .err_mask
= (DMAD6348_ST_OV_ERR_MASK
|
531 DMAD6348_ST_CRC_ERR_MASK
|
532 DMAD6348_ST_RX_ERR_MASK
|
533 DMAD6348_ST_OS_ERR_MASK
|
534 DMAD6348_ST_UN_ERR_MASK
),
537 static const struct bcm6348_iudma_hw bcm6368_hw
= {
541 static const struct udevice_id bcm6348_iudma_ids
[] = {
543 .compatible
= "brcm,bcm6348-iudma",
544 .data
= (ulong
)&bcm6348_hw
,
546 .compatible
= "brcm,bcm6368-iudma",
547 .data
= (ulong
)&bcm6368_hw
,
548 }, { /* sentinel */ }
551 static int bcm6348_iudma_probe(struct udevice
*dev
)
553 struct dma_dev_priv
*uc_priv
= dev_get_uclass_priv(dev
);
554 struct bcm6348_iudma_priv
*priv
= dev_get_priv(dev
);
555 const struct bcm6348_iudma_hw
*hw
=
556 (const struct bcm6348_iudma_hw
*)dev_get_driver_data(dev
);
560 uc_priv
->supported
= (DMA_SUPPORTS_DEV_TO_MEM
|
561 DMA_SUPPORTS_MEM_TO_DEV
);
564 /* dma global base address */
565 priv
->base
= dev_remap_addr_name(dev
, "dma");
569 /* dma channels base address */
570 priv
->chan
= dev_remap_addr_name(dev
, "dma-channels");
574 /* dma sram base address */
575 priv
->sram
= dev_remap_addr_name(dev
, "dma-sram");
579 /* get number of channels */
580 priv
->n_channels
= dev_read_u32_default(dev
, "dma-channels", 8);
581 if (priv
->n_channels
> DMA_CHAN_MAX
)
584 /* try to enable clocks */
589 ret
= clk_get_by_index(dev
, i
, &clk
);
593 ret
= clk_enable(&clk
);
595 pr_err("error enabling clock %d\n", i
);
602 /* try to perform resets */
604 struct reset_ctl reset
;
607 ret
= reset_get_by_index(dev
, i
, &reset
);
611 ret
= reset_deassert(&reset
);
613 pr_err("error deasserting reset %d\n", i
);
617 ret
= reset_free(&reset
);
619 pr_err("error freeing reset %d\n", i
);
624 /* disable dma controller */
625 clrbits_be32(priv
->base
+ DMA_CFG_REG
, DMA_CFG_ENABLE_MASK
);
627 /* alloc channel private data pointers */
628 priv
->ch_priv
= calloc(priv
->n_channels
,
629 sizeof(struct bcm6348_chan_priv
*));
633 /* stop dma channels */
634 for (ch
= 0; ch
< priv
->n_channels
; ch
++)
635 bcm6348_iudma_chan_stop(priv
, ch
);
637 /* enable dma controller */
638 setbits_be32(priv
->base
+ DMA_CFG_REG
, DMA_CFG_ENABLE_MASK
);
643 U_BOOT_DRIVER(bcm6348_iudma
) = {
644 .name
= "bcm6348_iudma",
646 .of_match
= bcm6348_iudma_ids
,
647 .ops
= &bcm6348_iudma_ops
,
648 .priv_auto
= sizeof(struct bcm6348_iudma_priv
),
649 .probe
= bcm6348_iudma_probe
,