1 // SPDX-License-Identifier: GPL-2.0-only
3 * ADMA driver for Nvidia's Tegra210 ADMA controller.
5 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
9 #include <linux/iopoll.h>
10 #include <linux/module.h>
11 #include <linux/of_device.h>
12 #include <linux/of_dma.h>
13 #include <linux/of_irq.h>
14 #include <linux/pm_runtime.h>
15 #include <linux/slab.h>
19 #define ADMA_CH_CMD 0x00
20 #define ADMA_CH_STATUS 0x0c
21 #define ADMA_CH_STATUS_XFER_EN BIT(0)
22 #define ADMA_CH_STATUS_XFER_PAUSED BIT(1)
24 #define ADMA_CH_INT_STATUS 0x10
25 #define ADMA_CH_INT_STATUS_XFER_DONE BIT(0)
27 #define ADMA_CH_INT_CLEAR 0x1c
28 #define ADMA_CH_CTRL 0x24
29 #define ADMA_CH_CTRL_DIR(val) (((val) & 0xf) << 12)
30 #define ADMA_CH_CTRL_DIR_AHUB2MEM 2
31 #define ADMA_CH_CTRL_DIR_MEM2AHUB 4
32 #define ADMA_CH_CTRL_MODE_CONTINUOUS (2 << 8)
33 #define ADMA_CH_CTRL_FLOWCTRL_EN BIT(1)
34 #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT 0
36 #define ADMA_CH_CONFIG 0x28
37 #define ADMA_CH_CONFIG_SRC_BUF(val) (((val) & 0x7) << 28)
38 #define ADMA_CH_CONFIG_TRG_BUF(val) (((val) & 0x7) << 24)
39 #define ADMA_CH_CONFIG_BURST_SIZE_SHIFT 20
40 #define ADMA_CH_CONFIG_MAX_BURST_SIZE 16
41 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0xf)
42 #define ADMA_CH_CONFIG_MAX_BUFS 8
44 #define ADMA_CH_FIFO_CTRL 0x2c
45 #define TEGRA210_ADMA_CH_FIFO_CTRL_OFLWTHRES(val) (((val) & 0xf) << 24)
46 #define TEGRA210_ADMA_CH_FIFO_CTRL_STRVTHRES(val) (((val) & 0xf) << 16)
47 #define TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0xf) << 8)
48 #define TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0xf)
49 #define TEGRA186_ADMA_CH_FIFO_CTRL_OFLWTHRES(val) (((val) & 0x1f) << 24)
50 #define TEGRA186_ADMA_CH_FIFO_CTRL_STRVTHRES(val) (((val) & 0x1f) << 16)
51 #define TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0x1f) << 8)
52 #define TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0x1f)
54 #define ADMA_CH_LOWER_SRC_ADDR 0x34
55 #define ADMA_CH_LOWER_TRG_ADDR 0x3c
56 #define ADMA_CH_TC 0x44
57 #define ADMA_CH_TC_COUNT_MASK 0x3ffffffc
59 #define ADMA_CH_XFER_STATUS 0x54
60 #define ADMA_CH_XFER_STATUS_COUNT_MASK 0xffff
62 #define ADMA_GLOBAL_CMD 0x00
63 #define ADMA_GLOBAL_SOFT_RESET 0x04
65 #define TEGRA_ADMA_BURST_COMPLETE_TIME 20
67 #define TEGRA210_FIFO_CTRL_DEFAULT (TEGRA210_ADMA_CH_FIFO_CTRL_OFLWTHRES(1) | \
68 TEGRA210_ADMA_CH_FIFO_CTRL_STRVTHRES(1) | \
69 TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
70 TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(3))
72 #define TEGRA186_FIFO_CTRL_DEFAULT (TEGRA186_ADMA_CH_FIFO_CTRL_OFLWTHRES(1) | \
73 TEGRA186_ADMA_CH_FIFO_CTRL_STRVTHRES(1) | \
74 TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
75 TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(3))
77 #define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift)
82 * struct tegra_adma_chip_data - Tegra chip specific data
83 * @global_reg_offset: Register offset of DMA global register.
84 * @global_int_clear: Register offset of DMA global interrupt clear.
85 * @ch_req_tx_shift: Register offset for AHUB transmit channel select.
86 * @ch_req_rx_shift: Register offset for AHUB receive channel select.
87 * @ch_base_offset: Register offset of DMA channel registers.
88 * @ch_fifo_ctrl: Default value for channel FIFO CTRL register.
89 * @ch_req_mask: Mask for Tx or Rx channel select.
90 * @ch_req_max: Maximum number of Tx or Rx channels available.
91 * @ch_reg_size: Size of DMA channel register space.
92 * @nr_channels: Number of DMA channels available.
94 struct tegra_adma_chip_data
{
95 unsigned int (*adma_get_burst_config
)(unsigned int burst_size
);
96 unsigned int global_reg_offset
;
97 unsigned int global_int_clear
;
98 unsigned int ch_req_tx_shift
;
99 unsigned int ch_req_rx_shift
;
100 unsigned int ch_base_offset
;
101 unsigned int ch_fifo_ctrl
;
102 unsigned int ch_req_mask
;
103 unsigned int ch_req_max
;
104 unsigned int ch_reg_size
;
105 unsigned int nr_channels
;
109 * struct tegra_adma_chan_regs - Tegra ADMA channel registers
111 struct tegra_adma_chan_regs
{
114 unsigned int src_addr
;
115 unsigned int trg_addr
;
116 unsigned int fifo_ctrl
;
122 * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests.
124 struct tegra_adma_desc
{
125 struct virt_dma_desc vd
;
126 struct tegra_adma_chan_regs ch_regs
;
133 * struct tegra_adma_chan - Tegra ADMA channel information
135 struct tegra_adma_chan
{
136 struct virt_dma_chan vc
;
137 struct tegra_adma_desc
*desc
;
138 struct tegra_adma
*tdma
;
140 void __iomem
*chan_addr
;
142 /* Slave channel configuration info */
143 struct dma_slave_config sconfig
;
144 enum dma_transfer_direction sreq_dir
;
145 unsigned int sreq_index
;
147 struct tegra_adma_chan_regs ch_regs
;
149 /* Transfer count and position info */
150 unsigned int tx_buf_count
;
151 unsigned int tx_buf_pos
;
155 * struct tegra_adma - Tegra ADMA controller information
158 struct dma_device dma_dev
;
160 void __iomem
*base_addr
;
161 struct clk
*ahub_clk
;
162 unsigned int nr_channels
;
163 unsigned long rx_requests_reserved
;
164 unsigned long tx_requests_reserved
;
166 /* Used to store global command register state when suspending */
167 unsigned int global_cmd
;
169 const struct tegra_adma_chip_data
*cdata
;
171 /* Last member of the structure */
172 struct tegra_adma_chan channels
[0];
175 static inline void tdma_write(struct tegra_adma
*tdma
, u32 reg
, u32 val
)
177 writel(val
, tdma
->base_addr
+ tdma
->cdata
->global_reg_offset
+ reg
);
180 static inline u32
tdma_read(struct tegra_adma
*tdma
, u32 reg
)
182 return readl(tdma
->base_addr
+ tdma
->cdata
->global_reg_offset
+ reg
);
185 static inline void tdma_ch_write(struct tegra_adma_chan
*tdc
, u32 reg
, u32 val
)
187 writel(val
, tdc
->chan_addr
+ reg
);
190 static inline u32
tdma_ch_read(struct tegra_adma_chan
*tdc
, u32 reg
)
192 return readl(tdc
->chan_addr
+ reg
);
195 static inline struct tegra_adma_chan
*to_tegra_adma_chan(struct dma_chan
*dc
)
197 return container_of(dc
, struct tegra_adma_chan
, vc
.chan
);
200 static inline struct tegra_adma_desc
*to_tegra_adma_desc(
201 struct dma_async_tx_descriptor
*td
)
203 return container_of(td
, struct tegra_adma_desc
, vd
.tx
);
206 static inline struct device
*tdc2dev(struct tegra_adma_chan
*tdc
)
208 return tdc
->tdma
->dev
;
211 static void tegra_adma_desc_free(struct virt_dma_desc
*vd
)
213 kfree(container_of(vd
, struct tegra_adma_desc
, vd
));
216 static int tegra_adma_slave_config(struct dma_chan
*dc
,
217 struct dma_slave_config
*sconfig
)
219 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
221 memcpy(&tdc
->sconfig
, sconfig
, sizeof(*sconfig
));
226 static int tegra_adma_init(struct tegra_adma
*tdma
)
231 /* Clear any interrupts */
232 tdma_write(tdma
, tdma
->cdata
->global_int_clear
, 0x1);
234 /* Assert soft reset */
235 tdma_write(tdma
, ADMA_GLOBAL_SOFT_RESET
, 0x1);
237 /* Wait for reset to clear */
238 ret
= readx_poll_timeout(readl
,
240 tdma
->cdata
->global_reg_offset
+
241 ADMA_GLOBAL_SOFT_RESET
,
242 status
, status
== 0, 20, 10000);
246 /* Enable global ADMA registers */
247 tdma_write(tdma
, ADMA_GLOBAL_CMD
, 1);
252 static int tegra_adma_request_alloc(struct tegra_adma_chan
*tdc
,
253 enum dma_transfer_direction direction
)
255 struct tegra_adma
*tdma
= tdc
->tdma
;
256 unsigned int sreq_index
= tdc
->sreq_index
;
258 if (tdc
->sreq_reserved
)
259 return tdc
->sreq_dir
== direction
? 0 : -EINVAL
;
261 if (sreq_index
> tdma
->cdata
->ch_req_max
) {
262 dev_err(tdma
->dev
, "invalid DMA request\n");
268 if (test_and_set_bit(sreq_index
, &tdma
->tx_requests_reserved
)) {
269 dev_err(tdma
->dev
, "DMA request reserved\n");
275 if (test_and_set_bit(sreq_index
, &tdma
->rx_requests_reserved
)) {
276 dev_err(tdma
->dev
, "DMA request reserved\n");
282 dev_WARN(tdma
->dev
, "channel %s has invalid transfer type\n",
283 dma_chan_name(&tdc
->vc
.chan
));
287 tdc
->sreq_dir
= direction
;
288 tdc
->sreq_reserved
= true;
293 static void tegra_adma_request_free(struct tegra_adma_chan
*tdc
)
295 struct tegra_adma
*tdma
= tdc
->tdma
;
297 if (!tdc
->sreq_reserved
)
300 switch (tdc
->sreq_dir
) {
302 clear_bit(tdc
->sreq_index
, &tdma
->tx_requests_reserved
);
306 clear_bit(tdc
->sreq_index
, &tdma
->rx_requests_reserved
);
310 dev_WARN(tdma
->dev
, "channel %s has invalid transfer type\n",
311 dma_chan_name(&tdc
->vc
.chan
));
315 tdc
->sreq_reserved
= false;
318 static u32
tegra_adma_irq_status(struct tegra_adma_chan
*tdc
)
320 u32 status
= tdma_ch_read(tdc
, ADMA_CH_INT_STATUS
);
322 return status
& ADMA_CH_INT_STATUS_XFER_DONE
;
325 static u32
tegra_adma_irq_clear(struct tegra_adma_chan
*tdc
)
327 u32 status
= tegra_adma_irq_status(tdc
);
330 tdma_ch_write(tdc
, ADMA_CH_INT_CLEAR
, status
);
335 static void tegra_adma_stop(struct tegra_adma_chan
*tdc
)
340 tdma_ch_write(tdc
, ADMA_CH_CMD
, 0);
342 /* Clear interrupt status */
343 tegra_adma_irq_clear(tdc
);
345 if (readx_poll_timeout_atomic(readl
, tdc
->chan_addr
+ ADMA_CH_STATUS
,
346 status
, !(status
& ADMA_CH_STATUS_XFER_EN
),
348 dev_err(tdc2dev(tdc
), "unable to stop DMA channel\n");
356 static void tegra_adma_start(struct tegra_adma_chan
*tdc
)
358 struct virt_dma_desc
*vd
= vchan_next_desc(&tdc
->vc
);
359 struct tegra_adma_chan_regs
*ch_regs
;
360 struct tegra_adma_desc
*desc
;
367 desc
= to_tegra_adma_desc(&vd
->tx
);
370 dev_warn(tdc2dev(tdc
), "unable to start DMA, no descriptor\n");
374 ch_regs
= &desc
->ch_regs
;
377 tdc
->tx_buf_count
= 0;
378 tdma_ch_write(tdc
, ADMA_CH_TC
, ch_regs
->tc
);
379 tdma_ch_write(tdc
, ADMA_CH_CTRL
, ch_regs
->ctrl
);
380 tdma_ch_write(tdc
, ADMA_CH_LOWER_SRC_ADDR
, ch_regs
->src_addr
);
381 tdma_ch_write(tdc
, ADMA_CH_LOWER_TRG_ADDR
, ch_regs
->trg_addr
);
382 tdma_ch_write(tdc
, ADMA_CH_FIFO_CTRL
, ch_regs
->fifo_ctrl
);
383 tdma_ch_write(tdc
, ADMA_CH_CONFIG
, ch_regs
->config
);
386 tdma_ch_write(tdc
, ADMA_CH_CMD
, 1);
391 static unsigned int tegra_adma_get_residue(struct tegra_adma_chan
*tdc
)
393 struct tegra_adma_desc
*desc
= tdc
->desc
;
394 unsigned int max
= ADMA_CH_XFER_STATUS_COUNT_MASK
+ 1;
395 unsigned int pos
= tdma_ch_read(tdc
, ADMA_CH_XFER_STATUS
);
396 unsigned int periods_remaining
;
399 * Handle wrap around of buffer count register
401 if (pos
< tdc
->tx_buf_pos
)
402 tdc
->tx_buf_count
+= pos
+ (max
- tdc
->tx_buf_pos
);
404 tdc
->tx_buf_count
+= pos
- tdc
->tx_buf_pos
;
406 periods_remaining
= tdc
->tx_buf_count
% desc
->num_periods
;
407 tdc
->tx_buf_pos
= pos
;
409 return desc
->buf_len
- (periods_remaining
* desc
->period_len
);
412 static irqreturn_t
tegra_adma_isr(int irq
, void *dev_id
)
414 struct tegra_adma_chan
*tdc
= dev_id
;
415 unsigned long status
;
418 spin_lock_irqsave(&tdc
->vc
.lock
, flags
);
420 status
= tegra_adma_irq_clear(tdc
);
421 if (status
== 0 || !tdc
->desc
) {
422 spin_unlock_irqrestore(&tdc
->vc
.lock
, flags
);
426 vchan_cyclic_callback(&tdc
->desc
->vd
);
428 spin_unlock_irqrestore(&tdc
->vc
.lock
, flags
);
433 static void tegra_adma_issue_pending(struct dma_chan
*dc
)
435 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
438 spin_lock_irqsave(&tdc
->vc
.lock
, flags
);
440 if (vchan_issue_pending(&tdc
->vc
)) {
442 tegra_adma_start(tdc
);
445 spin_unlock_irqrestore(&tdc
->vc
.lock
, flags
);
448 static bool tegra_adma_is_paused(struct tegra_adma_chan
*tdc
)
452 csts
= tdma_ch_read(tdc
, ADMA_CH_STATUS
);
453 csts
&= ADMA_CH_STATUS_XFER_PAUSED
;
455 return csts
? true : false;
458 static int tegra_adma_pause(struct dma_chan
*dc
)
460 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
461 struct tegra_adma_desc
*desc
= tdc
->desc
;
462 struct tegra_adma_chan_regs
*ch_regs
= &desc
->ch_regs
;
465 ch_regs
->ctrl
= tdma_ch_read(tdc
, ADMA_CH_CTRL
);
466 ch_regs
->ctrl
|= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT
);
467 tdma_ch_write(tdc
, ADMA_CH_CTRL
, ch_regs
->ctrl
);
469 while (dcnt
-- && !tegra_adma_is_paused(tdc
))
470 udelay(TEGRA_ADMA_BURST_COMPLETE_TIME
);
473 dev_err(tdc2dev(tdc
), "unable to pause DMA channel\n");
480 static int tegra_adma_resume(struct dma_chan
*dc
)
482 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
483 struct tegra_adma_desc
*desc
= tdc
->desc
;
484 struct tegra_adma_chan_regs
*ch_regs
= &desc
->ch_regs
;
486 ch_regs
->ctrl
= tdma_ch_read(tdc
, ADMA_CH_CTRL
);
487 ch_regs
->ctrl
&= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT
);
488 tdma_ch_write(tdc
, ADMA_CH_CTRL
, ch_regs
->ctrl
);
493 static int tegra_adma_terminate_all(struct dma_chan
*dc
)
495 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
499 spin_lock_irqsave(&tdc
->vc
.lock
, flags
);
502 tegra_adma_stop(tdc
);
504 tegra_adma_request_free(tdc
);
505 vchan_get_all_descriptors(&tdc
->vc
, &head
);
506 spin_unlock_irqrestore(&tdc
->vc
.lock
, flags
);
507 vchan_dma_desc_free_list(&tdc
->vc
, &head
);
512 static enum dma_status
tegra_adma_tx_status(struct dma_chan
*dc
,
514 struct dma_tx_state
*txstate
)
516 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
517 struct tegra_adma_desc
*desc
;
518 struct virt_dma_desc
*vd
;
521 unsigned int residual
;
523 ret
= dma_cookie_status(dc
, cookie
, txstate
);
524 if (ret
== DMA_COMPLETE
|| !txstate
)
527 spin_lock_irqsave(&tdc
->vc
.lock
, flags
);
529 vd
= vchan_find_desc(&tdc
->vc
, cookie
);
531 desc
= to_tegra_adma_desc(&vd
->tx
);
532 residual
= desc
->ch_regs
.tc
;
533 } else if (tdc
->desc
&& tdc
->desc
->vd
.tx
.cookie
== cookie
) {
534 residual
= tegra_adma_get_residue(tdc
);
539 spin_unlock_irqrestore(&tdc
->vc
.lock
, flags
);
541 dma_set_residue(txstate
, residual
);
546 static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size
)
548 if (!burst_size
|| burst_size
> ADMA_CH_CONFIG_MAX_BURST_SIZE
)
549 burst_size
= ADMA_CH_CONFIG_MAX_BURST_SIZE
;
551 return fls(burst_size
) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT
;
554 static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size
)
556 if (!burst_size
|| burst_size
> ADMA_CH_CONFIG_MAX_BURST_SIZE
)
557 burst_size
= ADMA_CH_CONFIG_MAX_BURST_SIZE
;
559 return (burst_size
- 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT
;
562 static int tegra_adma_set_xfer_params(struct tegra_adma_chan
*tdc
,
563 struct tegra_adma_desc
*desc
,
565 enum dma_transfer_direction direction
)
567 struct tegra_adma_chan_regs
*ch_regs
= &desc
->ch_regs
;
568 const struct tegra_adma_chip_data
*cdata
= tdc
->tdma
->cdata
;
569 unsigned int burst_size
, adma_dir
;
571 if (desc
->num_periods
> ADMA_CH_CONFIG_MAX_BUFS
)
576 adma_dir
= ADMA_CH_CTRL_DIR_MEM2AHUB
;
577 burst_size
= tdc
->sconfig
.dst_maxburst
;
578 ch_regs
->config
= ADMA_CH_CONFIG_SRC_BUF(desc
->num_periods
- 1);
579 ch_regs
->ctrl
= ADMA_CH_REG_FIELD_VAL(tdc
->sreq_index
,
581 cdata
->ch_req_tx_shift
);
582 ch_regs
->src_addr
= buf_addr
;
586 adma_dir
= ADMA_CH_CTRL_DIR_AHUB2MEM
;
587 burst_size
= tdc
->sconfig
.src_maxburst
;
588 ch_regs
->config
= ADMA_CH_CONFIG_TRG_BUF(desc
->num_periods
- 1);
589 ch_regs
->ctrl
= ADMA_CH_REG_FIELD_VAL(tdc
->sreq_index
,
591 cdata
->ch_req_rx_shift
);
592 ch_regs
->trg_addr
= buf_addr
;
596 dev_err(tdc2dev(tdc
), "DMA direction is not supported\n");
600 ch_regs
->ctrl
|= ADMA_CH_CTRL_DIR(adma_dir
) |
601 ADMA_CH_CTRL_MODE_CONTINUOUS
|
602 ADMA_CH_CTRL_FLOWCTRL_EN
;
603 ch_regs
->config
|= cdata
->adma_get_burst_config(burst_size
);
604 ch_regs
->config
|= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1);
605 ch_regs
->fifo_ctrl
= cdata
->ch_fifo_ctrl
;
606 ch_regs
->tc
= desc
->period_len
& ADMA_CH_TC_COUNT_MASK
;
608 return tegra_adma_request_alloc(tdc
, direction
);
611 static struct dma_async_tx_descriptor
*tegra_adma_prep_dma_cyclic(
612 struct dma_chan
*dc
, dma_addr_t buf_addr
, size_t buf_len
,
613 size_t period_len
, enum dma_transfer_direction direction
,
616 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
617 struct tegra_adma_desc
*desc
= NULL
;
619 if (!buf_len
|| !period_len
|| period_len
> ADMA_CH_TC_COUNT_MASK
) {
620 dev_err(tdc2dev(tdc
), "invalid buffer/period len\n");
624 if (buf_len
% period_len
) {
625 dev_err(tdc2dev(tdc
), "buf_len not a multiple of period_len\n");
629 if (!IS_ALIGNED(buf_addr
, 4)) {
630 dev_err(tdc2dev(tdc
), "invalid buffer alignment\n");
634 desc
= kzalloc(sizeof(*desc
), GFP_NOWAIT
);
638 desc
->buf_len
= buf_len
;
639 desc
->period_len
= period_len
;
640 desc
->num_periods
= buf_len
/ period_len
;
642 if (tegra_adma_set_xfer_params(tdc
, desc
, buf_addr
, direction
)) {
647 return vchan_tx_prep(&tdc
->vc
, &desc
->vd
, flags
);
650 static int tegra_adma_alloc_chan_resources(struct dma_chan
*dc
)
652 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
655 ret
= request_irq(tdc
->irq
, tegra_adma_isr
, 0, dma_chan_name(dc
), tdc
);
657 dev_err(tdc2dev(tdc
), "failed to get interrupt for %s\n",
662 ret
= pm_runtime_get_sync(tdc2dev(tdc
));
664 free_irq(tdc
->irq
, tdc
);
668 dma_cookie_init(&tdc
->vc
.chan
);
673 static void tegra_adma_free_chan_resources(struct dma_chan
*dc
)
675 struct tegra_adma_chan
*tdc
= to_tegra_adma_chan(dc
);
677 tegra_adma_terminate_all(dc
);
678 vchan_free_chan_resources(&tdc
->vc
);
679 tasklet_kill(&tdc
->vc
.task
);
680 free_irq(tdc
->irq
, tdc
);
681 pm_runtime_put(tdc2dev(tdc
));
684 tdc
->sreq_dir
= DMA_TRANS_NONE
;
687 static struct dma_chan
*tegra_dma_of_xlate(struct of_phandle_args
*dma_spec
,
688 struct of_dma
*ofdma
)
690 struct tegra_adma
*tdma
= ofdma
->of_dma_data
;
691 struct tegra_adma_chan
*tdc
;
692 struct dma_chan
*chan
;
693 unsigned int sreq_index
;
695 if (dma_spec
->args_count
!= 1)
698 sreq_index
= dma_spec
->args
[0];
700 if (sreq_index
== 0) {
701 dev_err(tdma
->dev
, "DMA request must not be 0\n");
705 chan
= dma_get_any_slave_channel(&tdma
->dma_dev
);
709 tdc
= to_tegra_adma_chan(chan
);
710 tdc
->sreq_index
= sreq_index
;
715 static int __maybe_unused
tegra_adma_runtime_suspend(struct device
*dev
)
717 struct tegra_adma
*tdma
= dev_get_drvdata(dev
);
718 struct tegra_adma_chan_regs
*ch_reg
;
719 struct tegra_adma_chan
*tdc
;
722 tdma
->global_cmd
= tdma_read(tdma
, ADMA_GLOBAL_CMD
);
723 if (!tdma
->global_cmd
)
726 for (i
= 0; i
< tdma
->nr_channels
; i
++) {
727 tdc
= &tdma
->channels
[i
];
728 ch_reg
= &tdc
->ch_regs
;
729 ch_reg
->cmd
= tdma_ch_read(tdc
, ADMA_CH_CMD
);
730 /* skip if channel is not active */
733 ch_reg
->tc
= tdma_ch_read(tdc
, ADMA_CH_TC
);
734 ch_reg
->src_addr
= tdma_ch_read(tdc
, ADMA_CH_LOWER_SRC_ADDR
);
735 ch_reg
->trg_addr
= tdma_ch_read(tdc
, ADMA_CH_LOWER_TRG_ADDR
);
736 ch_reg
->ctrl
= tdma_ch_read(tdc
, ADMA_CH_CTRL
);
737 ch_reg
->fifo_ctrl
= tdma_ch_read(tdc
, ADMA_CH_FIFO_CTRL
);
738 ch_reg
->config
= tdma_ch_read(tdc
, ADMA_CH_CONFIG
);
742 clk_disable_unprepare(tdma
->ahub_clk
);
747 static int __maybe_unused
tegra_adma_runtime_resume(struct device
*dev
)
749 struct tegra_adma
*tdma
= dev_get_drvdata(dev
);
750 struct tegra_adma_chan_regs
*ch_reg
;
751 struct tegra_adma_chan
*tdc
;
754 ret
= clk_prepare_enable(tdma
->ahub_clk
);
756 dev_err(dev
, "ahub clk_enable failed: %d\n", ret
);
759 tdma_write(tdma
, ADMA_GLOBAL_CMD
, tdma
->global_cmd
);
761 if (!tdma
->global_cmd
)
764 for (i
= 0; i
< tdma
->nr_channels
; i
++) {
765 tdc
= &tdma
->channels
[i
];
766 ch_reg
= &tdc
->ch_regs
;
767 /* skip if channel was not active earlier */
770 tdma_ch_write(tdc
, ADMA_CH_TC
, ch_reg
->tc
);
771 tdma_ch_write(tdc
, ADMA_CH_LOWER_SRC_ADDR
, ch_reg
->src_addr
);
772 tdma_ch_write(tdc
, ADMA_CH_LOWER_TRG_ADDR
, ch_reg
->trg_addr
);
773 tdma_ch_write(tdc
, ADMA_CH_CTRL
, ch_reg
->ctrl
);
774 tdma_ch_write(tdc
, ADMA_CH_FIFO_CTRL
, ch_reg
->fifo_ctrl
);
775 tdma_ch_write(tdc
, ADMA_CH_CONFIG
, ch_reg
->config
);
776 tdma_ch_write(tdc
, ADMA_CH_CMD
, ch_reg
->cmd
);
782 static const struct tegra_adma_chip_data tegra210_chip_data
= {
783 .adma_get_burst_config
= tegra210_adma_get_burst_config
,
784 .global_reg_offset
= 0xc00,
785 .global_int_clear
= 0x20,
786 .ch_req_tx_shift
= 28,
787 .ch_req_rx_shift
= 24,
789 .ch_fifo_ctrl
= TEGRA210_FIFO_CTRL_DEFAULT
,
796 static const struct tegra_adma_chip_data tegra186_chip_data
= {
797 .adma_get_burst_config
= tegra186_adma_get_burst_config
,
798 .global_reg_offset
= 0,
799 .global_int_clear
= 0x402c,
800 .ch_req_tx_shift
= 27,
801 .ch_req_rx_shift
= 22,
802 .ch_base_offset
= 0x10000,
803 .ch_fifo_ctrl
= TEGRA186_FIFO_CTRL_DEFAULT
,
806 .ch_reg_size
= 0x100,
810 static const struct of_device_id tegra_adma_of_match
[] = {
811 { .compatible
= "nvidia,tegra210-adma", .data
= &tegra210_chip_data
},
812 { .compatible
= "nvidia,tegra186-adma", .data
= &tegra186_chip_data
},
815 MODULE_DEVICE_TABLE(of
, tegra_adma_of_match
);
817 static int tegra_adma_probe(struct platform_device
*pdev
)
819 const struct tegra_adma_chip_data
*cdata
;
820 struct tegra_adma
*tdma
;
821 struct resource
*res
;
824 cdata
= of_device_get_match_data(&pdev
->dev
);
826 dev_err(&pdev
->dev
, "device match data not found\n");
830 tdma
= devm_kzalloc(&pdev
->dev
,
831 struct_size(tdma
, channels
, cdata
->nr_channels
),
836 tdma
->dev
= &pdev
->dev
;
838 tdma
->nr_channels
= cdata
->nr_channels
;
839 platform_set_drvdata(pdev
, tdma
);
841 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
842 tdma
->base_addr
= devm_ioremap_resource(&pdev
->dev
, res
);
843 if (IS_ERR(tdma
->base_addr
))
844 return PTR_ERR(tdma
->base_addr
);
846 tdma
->ahub_clk
= devm_clk_get(&pdev
->dev
, "d_audio");
847 if (IS_ERR(tdma
->ahub_clk
)) {
848 dev_err(&pdev
->dev
, "Error: Missing ahub controller clock\n");
849 return PTR_ERR(tdma
->ahub_clk
);
852 INIT_LIST_HEAD(&tdma
->dma_dev
.channels
);
853 for (i
= 0; i
< tdma
->nr_channels
; i
++) {
854 struct tegra_adma_chan
*tdc
= &tdma
->channels
[i
];
856 tdc
->chan_addr
= tdma
->base_addr
+ cdata
->ch_base_offset
857 + (cdata
->ch_reg_size
* i
);
859 tdc
->irq
= of_irq_get(pdev
->dev
.of_node
, i
);
861 ret
= tdc
->irq
?: -ENXIO
;
865 vchan_init(&tdc
->vc
, &tdma
->dma_dev
);
866 tdc
->vc
.desc_free
= tegra_adma_desc_free
;
870 pm_runtime_enable(&pdev
->dev
);
872 ret
= pm_runtime_get_sync(&pdev
->dev
);
876 ret
= tegra_adma_init(tdma
);
880 dma_cap_set(DMA_SLAVE
, tdma
->dma_dev
.cap_mask
);
881 dma_cap_set(DMA_PRIVATE
, tdma
->dma_dev
.cap_mask
);
882 dma_cap_set(DMA_CYCLIC
, tdma
->dma_dev
.cap_mask
);
884 tdma
->dma_dev
.dev
= &pdev
->dev
;
885 tdma
->dma_dev
.device_alloc_chan_resources
=
886 tegra_adma_alloc_chan_resources
;
887 tdma
->dma_dev
.device_free_chan_resources
=
888 tegra_adma_free_chan_resources
;
889 tdma
->dma_dev
.device_issue_pending
= tegra_adma_issue_pending
;
890 tdma
->dma_dev
.device_prep_dma_cyclic
= tegra_adma_prep_dma_cyclic
;
891 tdma
->dma_dev
.device_config
= tegra_adma_slave_config
;
892 tdma
->dma_dev
.device_tx_status
= tegra_adma_tx_status
;
893 tdma
->dma_dev
.device_terminate_all
= tegra_adma_terminate_all
;
894 tdma
->dma_dev
.src_addr_widths
= BIT(DMA_SLAVE_BUSWIDTH_4_BYTES
);
895 tdma
->dma_dev
.dst_addr_widths
= BIT(DMA_SLAVE_BUSWIDTH_4_BYTES
);
896 tdma
->dma_dev
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
897 tdma
->dma_dev
.residue_granularity
= DMA_RESIDUE_GRANULARITY_SEGMENT
;
898 tdma
->dma_dev
.device_pause
= tegra_adma_pause
;
899 tdma
->dma_dev
.device_resume
= tegra_adma_resume
;
901 ret
= dma_async_device_register(&tdma
->dma_dev
);
903 dev_err(&pdev
->dev
, "ADMA registration failed: %d\n", ret
);
907 ret
= of_dma_controller_register(pdev
->dev
.of_node
,
908 tegra_dma_of_xlate
, tdma
);
910 dev_err(&pdev
->dev
, "ADMA OF registration failed %d\n", ret
);
914 pm_runtime_put(&pdev
->dev
);
916 dev_info(&pdev
->dev
, "Tegra210 ADMA driver registered %d channels\n",
922 dma_async_device_unregister(&tdma
->dma_dev
);
924 pm_runtime_put_sync(&pdev
->dev
);
926 pm_runtime_disable(&pdev
->dev
);
929 irq_dispose_mapping(tdma
->channels
[i
].irq
);
934 static int tegra_adma_remove(struct platform_device
*pdev
)
936 struct tegra_adma
*tdma
= platform_get_drvdata(pdev
);
939 of_dma_controller_free(pdev
->dev
.of_node
);
940 dma_async_device_unregister(&tdma
->dma_dev
);
942 for (i
= 0; i
< tdma
->nr_channels
; ++i
)
943 irq_dispose_mapping(tdma
->channels
[i
].irq
);
945 pm_runtime_put_sync(&pdev
->dev
);
946 pm_runtime_disable(&pdev
->dev
);
951 static const struct dev_pm_ops tegra_adma_dev_pm_ops
= {
952 SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend
,
953 tegra_adma_runtime_resume
, NULL
)
954 SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend
,
955 pm_runtime_force_resume
)
958 static struct platform_driver tegra_admac_driver
= {
960 .name
= "tegra-adma",
961 .pm
= &tegra_adma_dev_pm_ops
,
962 .of_match_table
= tegra_adma_of_match
,
964 .probe
= tegra_adma_probe
,
965 .remove
= tegra_adma_remove
,
968 module_platform_driver(tegra_admac_driver
);
970 MODULE_ALIAS("platform:tegra210-adma");
971 MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver");
972 MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>");
973 MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>");
974 MODULE_LICENSE("GPL v2");