1 // SPDX-License-Identifier: GPL-2.0+
3 * Freescale i.MX28 APBH DMA driver
5 * Copyright (C) 2011 Marek Vasut <marek.vasut@gmail.com>
6 * on behalf of DENX Software Engineering GmbH
8 * Based on code from LTIB:
9 * Copyright (C) 2010 Freescale Semiconductor, Inc. All Rights Reserved.
15 #include <asm/cache.h>
16 #include <linux/list.h>
20 #include <linux/errno.h>
22 #include <asm/arch/clock.h>
23 #include <asm/arch/imx-regs.h>
24 #include <asm/arch/sys_proto.h>
25 #include <asm/mach-imx/dma.h>
26 #include <asm/mach-imx/regs-apbh.h>
28 static struct mxs_dma_chan mxs_dma_channels
[MXS_MAX_DMA_CHANNELS
];
31 * Test is the DMA channel is valid channel
33 int mxs_dma_validate_chan(int channel
)
35 struct mxs_dma_chan
*pchan
;
37 if ((channel
< 0) || (channel
>= MXS_MAX_DMA_CHANNELS
))
40 pchan
= mxs_dma_channels
+ channel
;
41 if (!(pchan
->flags
& MXS_DMA_FLAGS_ALLOCATED
))
48 * Return the address of the command within a descriptor.
50 static unsigned int mxs_dma_cmd_address(struct mxs_dma_desc
*desc
)
52 return desc
->address
+ offsetof(struct mxs_dma_desc
, cmd
);
56 * Read a DMA channel's hardware semaphore.
58 * As used by the MXS platform's DMA software, the DMA channel's hardware
59 * semaphore reflects the number of DMA commands the hardware will process, but
60 * has not yet finished. This is a volatile value read directly from hardware,
61 * so it must be be viewed as immediately stale.
63 * If the channel is not marked busy, or has finished processing all its
64 * commands, this value should be zero.
66 * See mxs_dma_append() for details on how DMA command blocks must be configured
67 * to maintain the expected behavior of the semaphore's value.
69 static int mxs_dma_read_semaphore(int channel
)
71 struct mxs_apbh_regs
*apbh_regs
=
72 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
76 ret
= mxs_dma_validate_chan(channel
);
80 tmp
= readl(&apbh_regs
->ch
[channel
].hw_apbh_ch_sema
);
82 tmp
&= APBH_CHn_SEMA_PHORE_MASK
;
83 tmp
>>= APBH_CHn_SEMA_PHORE_OFFSET
;
88 #if !CONFIG_IS_ENABLED(SYS_DCACHE_OFF)
89 void mxs_dma_flush_desc(struct mxs_dma_desc
*desc
)
94 addr
= (uintptr_t)desc
;
95 size
= roundup(sizeof(struct mxs_dma_desc
), MXS_DMA_ALIGNMENT
);
97 flush_dcache_range(addr
, addr
+ size
);
100 inline void mxs_dma_flush_desc(struct mxs_dma_desc
*desc
) {}
104 * Enable a DMA channel.
106 * If the given channel has any DMA descriptors on its active list, this
107 * function causes the DMA hardware to begin processing them.
109 * This function marks the DMA channel as "busy," whether or not there are any
110 * descriptors to process.
112 static int mxs_dma_enable(int channel
)
114 struct mxs_apbh_regs
*apbh_regs
=
115 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
117 struct mxs_dma_chan
*pchan
;
118 struct mxs_dma_desc
*pdesc
;
121 ret
= mxs_dma_validate_chan(channel
);
125 pchan
= mxs_dma_channels
+ channel
;
127 if (pchan
->pending_num
== 0) {
128 pchan
->flags
|= MXS_DMA_FLAGS_BUSY
;
132 pdesc
= list_first_entry(&pchan
->active
, struct mxs_dma_desc
, node
);
136 if (pchan
->flags
& MXS_DMA_FLAGS_BUSY
) {
137 if (!(pdesc
->cmd
.data
& MXS_DMA_DESC_CHAIN
))
140 sem
= mxs_dma_read_semaphore(channel
);
145 pdesc
= list_entry(pdesc
->node
.next
,
146 struct mxs_dma_desc
, node
);
147 writel(mxs_dma_cmd_address(pdesc
),
148 &apbh_regs
->ch
[channel
].hw_apbh_ch_nxtcmdar
);
150 writel(pchan
->pending_num
,
151 &apbh_regs
->ch
[channel
].hw_apbh_ch_sema
);
152 pchan
->active_num
+= pchan
->pending_num
;
153 pchan
->pending_num
= 0;
155 pchan
->active_num
+= pchan
->pending_num
;
156 pchan
->pending_num
= 0;
157 writel(mxs_dma_cmd_address(pdesc
),
158 &apbh_regs
->ch
[channel
].hw_apbh_ch_nxtcmdar
);
159 writel(pchan
->active_num
,
160 &apbh_regs
->ch
[channel
].hw_apbh_ch_sema
);
161 writel(1 << (channel
+ APBH_CTRL0_CLKGATE_CHANNEL_OFFSET
),
162 &apbh_regs
->hw_apbh_ctrl0_clr
);
165 pchan
->flags
|= MXS_DMA_FLAGS_BUSY
;
170 * Disable a DMA channel.
172 * This function shuts down a DMA channel and marks it as "not busy." Any
173 * descriptors on the active list are immediately moved to the head of the
174 * "done" list, whether or not they have actually been processed by the
175 * hardware. The "ready" flags of these descriptors are NOT cleared, so they
176 * still appear to be active.
178 * This function immediately shuts down a DMA channel's hardware, aborting any
179 * I/O that may be in progress, potentially leaving I/O hardware in an undefined
180 * state. It is unwise to call this function if there is ANY chance the hardware
181 * is still processing a command.
183 static int mxs_dma_disable(int channel
)
185 struct mxs_dma_chan
*pchan
;
186 struct mxs_apbh_regs
*apbh_regs
=
187 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
190 ret
= mxs_dma_validate_chan(channel
);
194 pchan
= mxs_dma_channels
+ channel
;
196 if (!(pchan
->flags
& MXS_DMA_FLAGS_BUSY
))
199 writel(1 << (channel
+ APBH_CTRL0_CLKGATE_CHANNEL_OFFSET
),
200 &apbh_regs
->hw_apbh_ctrl0_set
);
202 pchan
->flags
&= ~MXS_DMA_FLAGS_BUSY
;
203 pchan
->active_num
= 0;
204 pchan
->pending_num
= 0;
205 list_splice_init(&pchan
->active
, &pchan
->done
);
211 * Resets the DMA channel hardware.
213 static int mxs_dma_reset(int channel
)
215 struct mxs_apbh_regs
*apbh_regs
=
216 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
218 #if defined(CONFIG_MX23)
219 uint32_t setreg
= (uint32_t)(&apbh_regs
->hw_apbh_ctrl0_set
);
220 uint32_t offset
= APBH_CTRL0_RESET_CHANNEL_OFFSET
;
221 #elif defined(CONFIG_MX28) || defined(CONFIG_MX6) || defined(CONFIG_MX7) || \
222 defined(CONFIG_IMX8) || defined(CONFIG_IMX8M)
223 u32 setreg
= (uintptr_t)(&apbh_regs
->hw_apbh_channel_ctrl_set
);
224 u32 offset
= APBH_CHANNEL_CTRL_RESET_CHANNEL_OFFSET
;
227 ret
= mxs_dma_validate_chan(channel
);
231 writel(1 << (channel
+ offset
), (uintptr_t)setreg
);
237 * Enable or disable DMA interrupt.
239 * This function enables the given DMA channel to interrupt the CPU.
241 static int mxs_dma_enable_irq(int channel
, int enable
)
243 struct mxs_apbh_regs
*apbh_regs
=
244 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
247 ret
= mxs_dma_validate_chan(channel
);
252 writel(1 << (channel
+ APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET
),
253 &apbh_regs
->hw_apbh_ctrl1_set
);
255 writel(1 << (channel
+ APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET
),
256 &apbh_regs
->hw_apbh_ctrl1_clr
);
262 * Clear DMA interrupt.
264 * The software that is using the DMA channel must register to receive its
265 * interrupts and, when they arrive, must call this function to clear them.
267 static int mxs_dma_ack_irq(int channel
)
269 struct mxs_apbh_regs
*apbh_regs
=
270 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
273 ret
= mxs_dma_validate_chan(channel
);
277 writel(1 << channel
, &apbh_regs
->hw_apbh_ctrl1_clr
);
278 writel(1 << channel
, &apbh_regs
->hw_apbh_ctrl2_clr
);
284 * Request to reserve a DMA channel
286 static int mxs_dma_request(int channel
)
288 struct mxs_dma_chan
*pchan
;
290 if ((channel
< 0) || (channel
>= MXS_MAX_DMA_CHANNELS
))
293 pchan
= mxs_dma_channels
+ channel
;
294 if ((pchan
->flags
& MXS_DMA_FLAGS_VALID
) != MXS_DMA_FLAGS_VALID
)
297 if (pchan
->flags
& MXS_DMA_FLAGS_ALLOCATED
)
300 pchan
->flags
|= MXS_DMA_FLAGS_ALLOCATED
;
301 pchan
->active_num
= 0;
302 pchan
->pending_num
= 0;
304 INIT_LIST_HEAD(&pchan
->active
);
305 INIT_LIST_HEAD(&pchan
->done
);
311 * Release a DMA channel.
313 * This function releases a DMA channel from its current owner.
315 * The channel will NOT be released if it's marked "busy" (see
318 int mxs_dma_release(int channel
)
320 struct mxs_dma_chan
*pchan
;
323 ret
= mxs_dma_validate_chan(channel
);
327 pchan
= mxs_dma_channels
+ channel
;
329 if (pchan
->flags
& MXS_DMA_FLAGS_BUSY
)
333 pchan
->active_num
= 0;
334 pchan
->pending_num
= 0;
335 pchan
->flags
&= ~MXS_DMA_FLAGS_ALLOCATED
;
341 * Allocate DMA descriptor
343 struct mxs_dma_desc
*mxs_dma_desc_alloc(void)
345 struct mxs_dma_desc
*pdesc
;
348 size
= roundup(sizeof(struct mxs_dma_desc
), MXS_DMA_ALIGNMENT
);
349 pdesc
= memalign(MXS_DMA_ALIGNMENT
, size
);
354 memset(pdesc
, 0, sizeof(*pdesc
));
355 pdesc
->address
= (dma_addr_t
)pdesc
;
361 * Free DMA descriptor
363 void mxs_dma_desc_free(struct mxs_dma_desc
*pdesc
)
372 * Add a DMA descriptor to a channel.
374 * If the descriptor list for this channel is not empty, this function sets the
375 * CHAIN bit and the NEXTCMD_ADDR fields in the last descriptor's DMA command so
376 * it will chain to the new descriptor's command.
378 * Then, this function marks the new descriptor as "ready," adds it to the end
379 * of the active descriptor list, and increments the count of pending
382 * The MXS platform DMA software imposes some rules on DMA commands to maintain
383 * important invariants. These rules are NOT checked, but they must be carefully
384 * applied by software that uses MXS DMA channels.
387 * The DMA channel's hardware semaphore must reflect the number of DMA
388 * commands the hardware will process, but has not yet finished.
391 * A DMA channel begins processing commands when its hardware semaphore is
392 * written with a value greater than zero, and it stops processing commands
393 * when the semaphore returns to zero.
395 * When a channel finishes a DMA command, it will decrement its semaphore if
396 * the DECREMENT_SEMAPHORE bit is set in that command's flags bits.
398 * In principle, it's not necessary for the DECREMENT_SEMAPHORE to be set,
399 * unless it suits the purposes of the software. For example, one could
400 * construct a series of five DMA commands, with the DECREMENT_SEMAPHORE
401 * bit set only in the last one. Then, setting the DMA channel's hardware
402 * semaphore to one would cause the entire series of five commands to be
403 * processed. However, this example would violate the invariant given above.
406 * ALL DMA commands MUST have the DECREMENT_SEMAPHORE bit set so that the DMA
407 * channel's hardware semaphore will be decremented EVERY time a command is
410 int mxs_dma_desc_append(int channel
, struct mxs_dma_desc
*pdesc
)
412 struct mxs_dma_chan
*pchan
;
413 struct mxs_dma_desc
*last
;
416 ret
= mxs_dma_validate_chan(channel
);
420 pchan
= mxs_dma_channels
+ channel
;
422 pdesc
->cmd
.next
= mxs_dma_cmd_address(pdesc
);
423 pdesc
->flags
|= MXS_DMA_DESC_FIRST
| MXS_DMA_DESC_LAST
;
425 if (!list_empty(&pchan
->active
)) {
426 last
= list_entry(pchan
->active
.prev
, struct mxs_dma_desc
,
429 pdesc
->flags
&= ~MXS_DMA_DESC_FIRST
;
430 last
->flags
&= ~MXS_DMA_DESC_LAST
;
432 last
->cmd
.next
= mxs_dma_cmd_address(pdesc
);
433 last
->cmd
.data
|= MXS_DMA_DESC_CHAIN
;
435 mxs_dma_flush_desc(last
);
437 pdesc
->flags
|= MXS_DMA_DESC_READY
;
438 if (pdesc
->flags
& MXS_DMA_DESC_FIRST
)
439 pchan
->pending_num
++;
440 list_add_tail(&pdesc
->node
, &pchan
->active
);
442 mxs_dma_flush_desc(pdesc
);
448 * Clean up processed DMA descriptors.
450 * This function removes processed DMA descriptors from the "active" list. Pass
451 * in a non-NULL list head to get the descriptors moved to your list. Pass NULL
452 * to get the descriptors moved to the channel's "done" list. Descriptors on
453 * the "done" list can be retrieved with mxs_dma_get_finished().
455 * This function marks the DMA channel as "not busy" if no unprocessed
456 * descriptors remain on the "active" list.
458 static int mxs_dma_finish(int channel
, struct list_head
*head
)
461 struct mxs_dma_chan
*pchan
;
462 struct list_head
*p
, *q
;
463 struct mxs_dma_desc
*pdesc
;
466 ret
= mxs_dma_validate_chan(channel
);
470 pchan
= mxs_dma_channels
+ channel
;
472 sem
= mxs_dma_read_semaphore(channel
);
476 if (sem
== pchan
->active_num
)
479 list_for_each_safe(p
, q
, &pchan
->active
) {
480 if ((pchan
->active_num
) <= sem
)
483 pdesc
= list_entry(p
, struct mxs_dma_desc
, node
);
484 pdesc
->flags
&= ~MXS_DMA_DESC_READY
;
487 list_move_tail(p
, head
);
489 list_move_tail(p
, &pchan
->done
);
491 if (pdesc
->flags
& MXS_DMA_DESC_LAST
)
496 pchan
->flags
&= ~MXS_DMA_FLAGS_BUSY
;
502 * Wait for DMA channel to complete
504 static int mxs_dma_wait_complete(uint32_t timeout
, unsigned int chan
)
506 struct mxs_apbh_regs
*apbh_regs
=
507 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
510 ret
= mxs_dma_validate_chan(chan
);
514 if (mxs_wait_mask_set(&apbh_regs
->hw_apbh_ctrl1_reg
,
515 1 << chan
, timeout
)) {
524 * Execute the DMA channel
526 int mxs_dma_go(int chan
)
528 uint32_t timeout
= 10000000;
531 LIST_HEAD(tmp_desc_list
);
533 mxs_dma_enable_irq(chan
, 1);
534 mxs_dma_enable(chan
);
536 /* Wait for DMA to finish. */
537 ret
= mxs_dma_wait_complete(timeout
, chan
);
539 /* Clear out the descriptors we just ran. */
540 mxs_dma_finish(chan
, &tmp_desc_list
);
542 /* Shut the DMA channel down. */
543 mxs_dma_ack_irq(chan
);
545 mxs_dma_enable_irq(chan
, 0);
546 mxs_dma_disable(chan
);
552 * Execute a continuously running circular DMA descriptor.
553 * NOTE: This is not intended for general use, but rather
554 * for the LCD driver in Smart-LCD mode. It allows
555 * continuous triggering of the RUN bit there.
557 void mxs_dma_circ_start(int chan
, struct mxs_dma_desc
*pdesc
)
559 struct mxs_apbh_regs
*apbh_regs
=
560 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
562 mxs_dma_flush_desc(pdesc
);
564 mxs_dma_enable_irq(chan
, 1);
566 writel(mxs_dma_cmd_address(pdesc
),
567 &apbh_regs
->ch
[chan
].hw_apbh_ch_nxtcmdar
);
568 writel(1, &apbh_regs
->ch
[chan
].hw_apbh_ch_sema
);
569 writel(1 << (chan
+ APBH_CTRL0_CLKGATE_CHANNEL_OFFSET
),
570 &apbh_regs
->hw_apbh_ctrl0_clr
);
574 * Initialize the DMA hardware
576 void mxs_dma_init(void)
578 struct mxs_apbh_regs
*apbh_regs
=
579 (struct mxs_apbh_regs
*)MXS_APBH_BASE
;
581 mxs_reset_block(&apbh_regs
->hw_apbh_ctrl0_reg
);
583 #ifdef CONFIG_APBH_DMA_BURST8
584 writel(APBH_CTRL0_AHB_BURST8_EN
,
585 &apbh_regs
->hw_apbh_ctrl0_set
);
587 writel(APBH_CTRL0_AHB_BURST8_EN
,
588 &apbh_regs
->hw_apbh_ctrl0_clr
);
591 #ifdef CONFIG_APBH_DMA_BURST
592 writel(APBH_CTRL0_APB_BURST_EN
,
593 &apbh_regs
->hw_apbh_ctrl0_set
);
595 writel(APBH_CTRL0_APB_BURST_EN
,
596 &apbh_regs
->hw_apbh_ctrl0_clr
);
600 int mxs_dma_init_channel(int channel
)
602 struct mxs_dma_chan
*pchan
;
605 pchan
= mxs_dma_channels
+ channel
;
606 pchan
->flags
= MXS_DMA_FLAGS_VALID
;
608 ret
= mxs_dma_request(channel
);
611 printf("MXS DMA: Can't acquire DMA channel %i\n",
616 mxs_dma_reset(channel
);
617 mxs_dma_ack_irq(channel
);