]> git.ipfire.org Git - thirdparty/linux.git/blob - drivers/dma/owl-dma.c
projects / thirdparty / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
io_uring: reset -EBUSY error when io sq thread is waken up
[thirdparty/linux.git] / drivers / dma / owl-dma.c
1 // SPDX-License-Identifier: GPL-2.0+
2 //
3 // Actions Semi Owl SoCs DMA driver
4 //
5 // Copyright (c) 2014 Actions Semi Inc.
6 // Author: David Liu <liuwei@actions-semi.com>
7 //
8 // Copyright (c) 2018 Linaro Ltd.
9 // Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
10
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/dmaengine.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dmapool.h>
17 #include <linux/err.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
20 #include <linux/io.h>
21 #include <linux/mm.h>
22 #include <linux/module.h>
23 #include <linux/of_device.h>
24 #include <linux/of_dma.h>
25 #include <linux/slab.h>
26 #include "virt-dma.h"
27
28 #define OWL_DMA_FRAME_MAX_LENGTH 0xfffff
29
30 /* Global DMA Controller Registers */
31 #define OWL_DMA_IRQ_PD0 0x00
32 #define OWL_DMA_IRQ_PD1 0x04
33 #define OWL_DMA_IRQ_PD2 0x08
34 #define OWL_DMA_IRQ_PD3 0x0C
35 #define OWL_DMA_IRQ_EN0 0x10
36 #define OWL_DMA_IRQ_EN1 0x14
37 #define OWL_DMA_IRQ_EN2 0x18
38 #define OWL_DMA_IRQ_EN3 0x1C
39 #define OWL_DMA_SECURE_ACCESS_CTL 0x20
40 #define OWL_DMA_NIC_QOS 0x24
41 #define OWL_DMA_DBGSEL 0x28
42 #define OWL_DMA_IDLE_STAT 0x2C
43
44 /* Channel Registers */
45 #define OWL_DMA_CHAN_BASE(i) (0x100 + (i) * 0x100)
46 #define OWL_DMAX_MODE 0x00
47 #define OWL_DMAX_SOURCE 0x04
48 #define OWL_DMAX_DESTINATION 0x08
49 #define OWL_DMAX_FRAME_LEN 0x0C
50 #define OWL_DMAX_FRAME_CNT 0x10
51 #define OWL_DMAX_REMAIN_FRAME_CNT 0x14
52 #define OWL_DMAX_REMAIN_CNT 0x18
53 #define OWL_DMAX_SOURCE_STRIDE 0x1C
54 #define OWL_DMAX_DESTINATION_STRIDE 0x20
55 #define OWL_DMAX_START 0x24
56 #define OWL_DMAX_PAUSE 0x28
57 #define OWL_DMAX_CHAINED_CTL 0x2C
58 #define OWL_DMAX_CONSTANT 0x30
59 #define OWL_DMAX_LINKLIST_CTL 0x34
60 #define OWL_DMAX_NEXT_DESCRIPTOR 0x38
61 #define OWL_DMAX_CURRENT_DESCRIPTOR_NUM 0x3C
62 #define OWL_DMAX_INT_CTL 0x40
63 #define OWL_DMAX_INT_STATUS 0x44
64 #define OWL_DMAX_CURRENT_SOURCE_POINTER 0x48
65 #define OWL_DMAX_CURRENT_DESTINATION_POINTER 0x4C
66
67 /* OWL_DMAX_MODE Bits */
68 #define OWL_DMA_MODE_TS(x) (((x) & GENMASK(5, 0)) << 0)
69 #define OWL_DMA_MODE_ST(x) (((x) & GENMASK(1, 0)) << 8)
70 #define OWL_DMA_MODE_ST_DEV OWL_DMA_MODE_ST(0)
71 #define OWL_DMA_MODE_ST_DCU OWL_DMA_MODE_ST(2)
72 #define OWL_DMA_MODE_ST_SRAM OWL_DMA_MODE_ST(3)
73 #define OWL_DMA_MODE_DT(x) (((x) & GENMASK(1, 0)) << 10)
74 #define OWL_DMA_MODE_DT_DEV OWL_DMA_MODE_DT(0)
75 #define OWL_DMA_MODE_DT_DCU OWL_DMA_MODE_DT(2)
76 #define OWL_DMA_MODE_DT_SRAM OWL_DMA_MODE_DT(3)
77 #define OWL_DMA_MODE_SAM(x) (((x) & GENMASK(1, 0)) << 16)
78 #define OWL_DMA_MODE_SAM_CONST OWL_DMA_MODE_SAM(0)
79 #define OWL_DMA_MODE_SAM_INC OWL_DMA_MODE_SAM(1)
80 #define OWL_DMA_MODE_SAM_STRIDE OWL_DMA_MODE_SAM(2)
81 #define OWL_DMA_MODE_DAM(x) (((x) & GENMASK(1, 0)) << 18)
82 #define OWL_DMA_MODE_DAM_CONST OWL_DMA_MODE_DAM(0)
83 #define OWL_DMA_MODE_DAM_INC OWL_DMA_MODE_DAM(1)
84 #define OWL_DMA_MODE_DAM_STRIDE OWL_DMA_MODE_DAM(2)
85 #define OWL_DMA_MODE_PW(x) (((x) & GENMASK(2, 0)) << 20)
86 #define OWL_DMA_MODE_CB BIT(23)
87 #define OWL_DMA_MODE_NDDBW(x) (((x) & 0x1) << 28)
88 #define OWL_DMA_MODE_NDDBW_32BIT OWL_DMA_MODE_NDDBW(0)
89 #define OWL_DMA_MODE_NDDBW_8BIT OWL_DMA_MODE_NDDBW(1)
90 #define OWL_DMA_MODE_CFE BIT(29)
91 #define OWL_DMA_MODE_LME BIT(30)
92 #define OWL_DMA_MODE_CME BIT(31)
93
94 /* OWL_DMAX_LINKLIST_CTL Bits */
95 #define OWL_DMA_LLC_SAV(x) (((x) & GENMASK(1, 0)) << 8)
96 #define OWL_DMA_LLC_SAV_INC OWL_DMA_LLC_SAV(0)
97 #define OWL_DMA_LLC_SAV_LOAD_NEXT OWL_DMA_LLC_SAV(1)
98 #define OWL_DMA_LLC_SAV_LOAD_PREV OWL_DMA_LLC_SAV(2)
99 #define OWL_DMA_LLC_DAV(x) (((x) & GENMASK(1, 0)) << 10)
100 #define OWL_DMA_LLC_DAV_INC OWL_DMA_LLC_DAV(0)
101 #define OWL_DMA_LLC_DAV_LOAD_NEXT OWL_DMA_LLC_DAV(1)
102 #define OWL_DMA_LLC_DAV_LOAD_PREV OWL_DMA_LLC_DAV(2)
103 #define OWL_DMA_LLC_SUSPEND BIT(16)
104
105 /* OWL_DMAX_INT_CTL Bits */
106 #define OWL_DMA_INTCTL_BLOCK BIT(0)
107 #define OWL_DMA_INTCTL_SUPER_BLOCK BIT(1)
108 #define OWL_DMA_INTCTL_FRAME BIT(2)
109 #define OWL_DMA_INTCTL_HALF_FRAME BIT(3)
110 #define OWL_DMA_INTCTL_LAST_FRAME BIT(4)
111
112 /* OWL_DMAX_INT_STATUS Bits */
113 #define OWL_DMA_INTSTAT_BLOCK BIT(0)
114 #define OWL_DMA_INTSTAT_SUPER_BLOCK BIT(1)
115 #define OWL_DMA_INTSTAT_FRAME BIT(2)
116 #define OWL_DMA_INTSTAT_HALF_FRAME BIT(3)
117 #define OWL_DMA_INTSTAT_LAST_FRAME BIT(4)
118
119 /* Pack shift and newshift in a single word */
120 #define BIT_FIELD(val, width, shift, newshift) \
121 ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))
122
123 /**
124 * struct owl_dma_lli_hw - Hardware link list for dma transfer
125 * @next_lli: physical address of the next link list
126 * @saddr: source physical address
127 * @daddr: destination physical address
128 * @flen: frame length
129 * @fcnt: frame count
130 * @src_stride: source stride
131 * @dst_stride: destination stride
132 * @ctrla: dma_mode and linklist ctrl config
133 * @ctrlb: interrupt config
134 * @const_num: data for constant fill
135 */
136 struct owl_dma_lli_hw {
137 u32 next_lli;
138 u32 saddr;
139 u32 daddr;
140 u32 flen:20;
141 u32 fcnt:12;
142 u32 src_stride;
143 u32 dst_stride;
144 u32 ctrla;
145 u32 ctrlb;
146 u32 const_num;
147 };
148
149 /**
150 * struct owl_dma_lli - Link list for dma transfer
151 * @hw: hardware link list
152 * @phys: physical address of hardware link list
153 * @node: node for txd's lli_list
154 */
155 struct owl_dma_lli {
156 struct owl_dma_lli_hw hw;
157 dma_addr_t phys;
158 struct list_head node;
159 };
160
161 /**
162 * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor
163 * @vd: virtual DMA descriptor
164 * @lli_list: link list of lli nodes
165 * @cyclic: flag to indicate cyclic transfers
166 */
167 struct owl_dma_txd {
168 struct virt_dma_desc vd;
169 struct list_head lli_list;
170 bool cyclic;
171 };
172
173 /**
174 * struct owl_dma_pchan - Holder for the physical channels
175 * @id: physical index to this channel
176 * @base: virtual memory base for the dma channel
177 * @vchan: the virtual channel currently being served by this physical channel
178 * @lock: a lock to use when altering an instance of this struct
179 */
180 struct owl_dma_pchan {
181 u32 id;
182 void __iomem *base;
183 struct owl_dma_vchan *vchan;
184 spinlock_t lock;
185 };
186
187 /**
188 * struct owl_dma_pchan - Wrapper for DMA ENGINE channel
189 * @vc: wrappped virtual channel
190 * @pchan: the physical channel utilized by this channel
191 * @txd: active transaction on this channel
192 * @cfg: slave configuration for this channel
193 * @drq: physical DMA request ID for this channel
194 */
195 struct owl_dma_vchan {
196 struct virt_dma_chan vc;
197 struct owl_dma_pchan *pchan;
198 struct owl_dma_txd *txd;
199 struct dma_slave_config cfg;
200 u8 drq;
201 };
202
203 /**
204 * struct owl_dma - Holder for the Owl DMA controller
205 * @dma: dma engine for this instance
206 * @base: virtual memory base for the DMA controller
207 * @clk: clock for the DMA controller
208 * @lock: a lock to use when change DMA controller global register
209 * @lli_pool: a pool for the LLI descriptors
210 * @irq: interrupt ID for the DMA controller
211 * @nr_pchans: the number of physical channels
212 * @pchans: array of data for the physical channels
213 * @nr_vchans: the number of physical channels
214 * @vchans: array of data for the physical channels
215 */
216 struct owl_dma {
217 struct dma_device dma;
218 void __iomem *base;
219 struct clk *clk;
220 spinlock_t lock;
221 struct dma_pool *lli_pool;
222 int irq;
223
224 unsigned int nr_pchans;
225 struct owl_dma_pchan *pchans;
226
227 unsigned int nr_vchans;
228 struct owl_dma_vchan *vchans;
229 };
230
231 static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,
232 u32 val, bool state)
233 {
234 u32 regval;
235
236 regval = readl(pchan->base + reg);
237
238 if (state)
239 regval |= val;
240 else
241 regval &= ~val;
242
243 writel(val, pchan->base + reg);
244 }
245
246 static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)
247 {
248 writel(data, pchan->base + reg);
249 }
250
251 static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)
252 {
253 return readl(pchan->base + reg);
254 }
255
256 static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)
257 {
258 u32 regval;
259
260 regval = readl(od->base + reg);
261
262 if (state)
263 regval |= val;
264 else
265 regval &= ~val;
266
267 writel(val, od->base + reg);
268 }
269
270 static void dma_writel(struct owl_dma *od, u32 reg, u32 data)
271 {
272 writel(data, od->base + reg);
273 }
274
275 static u32 dma_readl(struct owl_dma *od, u32 reg)
276 {
277 return readl(od->base + reg);
278 }
279
280 static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
281 {
282 return container_of(dd, struct owl_dma, dma);
283 }
284
285 static struct device *chan2dev(struct dma_chan *chan)
286 {
287 return &chan->dev->device;
288 }
289
290 static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
291 {
292 return container_of(chan, struct owl_dma_vchan, vc.chan);
293 }
294
295 static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)
296 {
297 return container_of(tx, struct owl_dma_txd, vd.tx);
298 }
299
300 static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)
301 {
302 u32 ctl;
303
304 ctl = BIT_FIELD(mode, 4, 28, 28) |
305 BIT_FIELD(mode, 8, 16, 20) |
306 BIT_FIELD(mode, 4, 8, 16) |
307 BIT_FIELD(mode, 6, 0, 10) |
308 BIT_FIELD(llc_ctl, 2, 10, 8) |
309 BIT_FIELD(llc_ctl, 2, 8, 6);
310
311 return ctl;
312 }
313
314 static inline u32 llc_hw_ctrlb(u32 int_ctl)
315 {
316 u32 ctl;
317
318 ctl = BIT_FIELD(int_ctl, 7, 0, 18);
319
320 return ctl;
321 }
322
323 static void owl_dma_free_lli(struct owl_dma *od,
324 struct owl_dma_lli *lli)
325 {
326 list_del(&lli->node);
327 dma_pool_free(od->lli_pool, lli, lli->phys);
328 }
329
330 static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)
331 {
332 struct owl_dma_lli *lli;
333 dma_addr_t phys;
334
335 lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);
336 if (!lli)
337 return NULL;
338
339 INIT_LIST_HEAD(&lli->node);
340 lli->phys = phys;
341
342 return lli;
343 }
344
345 static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,
346 struct owl_dma_lli *prev,
347 struct owl_dma_lli *next,
348 bool is_cyclic)
349 {
350 if (!is_cyclic)
351 list_add_tail(&next->node, &txd->lli_list);
352
353 if (prev) {
354 prev->hw.next_lli = next->phys;
355 prev->hw.ctrla |= llc_hw_ctrla(OWL_DMA_MODE_LME, 0);
356 }
357
358 return next;
359 }
360
361 static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,
362 struct owl_dma_lli *lli,
363 dma_addr_t src, dma_addr_t dst,
364 u32 len, enum dma_transfer_direction dir,
365 struct dma_slave_config *sconfig,
366 bool is_cyclic)
367 {
368 struct owl_dma_lli_hw *hw = &lli->hw;
369 u32 mode;
370
371 mode = OWL_DMA_MODE_PW(0);
372
373 switch (dir) {
374 case DMA_MEM_TO_MEM:
375 mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |
376 OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |
377 OWL_DMA_MODE_DAM_INC;
378
379 break;
380 case DMA_MEM_TO_DEV:
381 mode |= OWL_DMA_MODE_TS(vchan->drq)
382 | OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV
383 | OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST;
384
385 /*
386 * Hardware only supports 32bit and 8bit buswidth. Since the
387 * default is 32bit, select 8bit only when requested.
388 */
389 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
390 mode |= OWL_DMA_MODE_NDDBW_8BIT;
391
392 break;
393 case DMA_DEV_TO_MEM:
394 mode |= OWL_DMA_MODE_TS(vchan->drq)
395 | OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU
396 | OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC;
397
398 /*
399 * Hardware only supports 32bit and 8bit buswidth. Since the
400 * default is 32bit, select 8bit only when requested.
401 */
402 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
403 mode |= OWL_DMA_MODE_NDDBW_8BIT;
404
405 break;
406 default:
407 return -EINVAL;
408 }
409
410 hw->next_lli = 0; /* One link list by default */
411 hw->saddr = src;
412 hw->daddr = dst;
413
414 hw->fcnt = 1; /* Frame count fixed as 1 */
415 hw->flen = len; /* Max frame length is 1MB */
416 hw->src_stride = 0;
417 hw->dst_stride = 0;
418 hw->ctrla = llc_hw_ctrla(mode,
419 OWL_DMA_LLC_SAV_LOAD_NEXT |
420 OWL_DMA_LLC_DAV_LOAD_NEXT);
421
422 if (is_cyclic)
423 hw->ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK);
424 else
425 hw->ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);
426
427 return 0;
428 }
429
430 static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
431 struct owl_dma_vchan *vchan)
432 {
433 struct owl_dma_pchan *pchan = NULL;
434 unsigned long flags;
435 int i;
436
437 for (i = 0; i < od->nr_pchans; i++) {
438 pchan = &od->pchans[i];
439
440 spin_lock_irqsave(&pchan->lock, flags);
441 if (!pchan->vchan) {
442 pchan->vchan = vchan;
443 spin_unlock_irqrestore(&pchan->lock, flags);
444 break;
445 }
446
447 spin_unlock_irqrestore(&pchan->lock, flags);
448 }
449
450 return pchan;
451 }
452
453 static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)
454 {
455 unsigned int val;
456
457 val = dma_readl(od, OWL_DMA_IDLE_STAT);
458
459 return !(val & (1 << pchan->id));
460 }
461
462 static void owl_dma_terminate_pchan(struct owl_dma *od,
463 struct owl_dma_pchan *pchan)
464 {
465 unsigned long flags;
466 u32 irq_pd;
467
468 pchan_writel(pchan, OWL_DMAX_START, 0);
469 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
470
471 spin_lock_irqsave(&od->lock, flags);
472 dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);
473
474 irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);
475 if (irq_pd & (1 << pchan->id)) {
476 dev_warn(od->dma.dev,
477 "terminating pchan %d that still has pending irq\n",
478 pchan->id);
479 dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));
480 }
481
482 pchan->vchan = NULL;
483
484 spin_unlock_irqrestore(&od->lock, flags);
485 }
486
487 static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan)
488 {
489 pchan_writel(pchan, 1, OWL_DMAX_PAUSE);
490 }
491
492 static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan)
493 {
494 pchan_writel(pchan, 0, OWL_DMAX_PAUSE);
495 }
496
497 static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
498 {
499 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
500 struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);
501 struct owl_dma_pchan *pchan = vchan->pchan;
502 struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
503 struct owl_dma_lli *lli;
504 unsigned long flags;
505 u32 int_ctl;
506
507 list_del(&vd->node);
508
509 vchan->txd = txd;
510
511 /* Wait for channel inactive */
512 while (owl_dma_pchan_busy(od, pchan))
513 cpu_relax();
514
515 lli = list_first_entry(&txd->lli_list,
516 struct owl_dma_lli, node);
517
518 if (txd->cyclic)
519 int_ctl = OWL_DMA_INTCTL_BLOCK;
520 else
521 int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;
522
523 pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);
524 pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,
525 OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);
526 pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);
527 pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);
528
529 /* Clear IRQ status for this pchan */
530 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
531
532 spin_lock_irqsave(&od->lock, flags);
533
534 dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);
535
536 spin_unlock_irqrestore(&od->lock, flags);
537
538 dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);
539
540 /* Start DMA transfer for this pchan */
541 pchan_writel(pchan, OWL_DMAX_START, 0x1);
542
543 return 0;
544 }
545
546 static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)
547 {
548 /* Ensure that the physical channel is stopped */
549 owl_dma_terminate_pchan(od, vchan->pchan);
550
551 vchan->pchan = NULL;
552 }
553
554 static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)
555 {
556 struct owl_dma *od = dev_id;
557 struct owl_dma_vchan *vchan;
558 struct owl_dma_pchan *pchan;
559 unsigned long pending;
560 int i;
561 unsigned int global_irq_pending, chan_irq_pending;
562
563 spin_lock(&od->lock);
564
565 pending = dma_readl(od, OWL_DMA_IRQ_PD0);
566
567 /* Clear IRQ status for each pchan */
568 for_each_set_bit(i, &pending, od->nr_pchans) {
569 pchan = &od->pchans[i];
570 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
571 }
572
573 /* Clear pending IRQ */
574 dma_writel(od, OWL_DMA_IRQ_PD0, pending);
575
576 /* Check missed pending IRQ */
577 for (i = 0; i < od->nr_pchans; i++) {
578 pchan = &od->pchans[i];
579 chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &
580 pchan_readl(pchan, OWL_DMAX_INT_STATUS);
581
582 /* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */
583 dma_readl(od, OWL_DMA_IRQ_PD0);
584
585 global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);
586
587 if (chan_irq_pending && !(global_irq_pending & BIT(i))) {
588 dev_dbg(od->dma.dev,
589 "global and channel IRQ pending match err\n");
590
591 /* Clear IRQ status for this pchan */
592 pchan_update(pchan, OWL_DMAX_INT_STATUS,
593 0xff, false);
594
595 /* Update global IRQ pending */
596 pending |= BIT(i);
597 }
598 }
599
600 spin_unlock(&od->lock);
601
602 for_each_set_bit(i, &pending, od->nr_pchans) {
603 struct owl_dma_txd *txd;
604
605 pchan = &od->pchans[i];
606
607 vchan = pchan->vchan;
608 if (!vchan) {
609 dev_warn(od->dma.dev, "no vchan attached on pchan %d\n",
610 pchan->id);
611 continue;
612 }
613
614 spin_lock(&vchan->vc.lock);
615
616 txd = vchan->txd;
617 if (txd) {
618 vchan->txd = NULL;
619
620 vchan_cookie_complete(&txd->vd);
621
622 /*
623 * Start the next descriptor (if any),
624 * otherwise free this channel.
625 */
626 if (vchan_next_desc(&vchan->vc))
627 owl_dma_start_next_txd(vchan);
628 else
629 owl_dma_phy_free(od, vchan);
630 }
631
632 spin_unlock(&vchan->vc.lock);
633 }
634
635 return IRQ_HANDLED;
636 }
637
638 static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)
639 {
640 struct owl_dma_lli *lli, *_lli;
641
642 if (unlikely(!txd))
643 return;
644
645 list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)
646 owl_dma_free_lli(od, lli);
647
648 kfree(txd);
649 }
650
651 static void owl_dma_desc_free(struct virt_dma_desc *vd)
652 {
653 struct owl_dma *od = to_owl_dma(vd->tx.chan->device);
654 struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
655
656 owl_dma_free_txd(od, txd);
657 }
658
659 static int owl_dma_terminate_all(struct dma_chan *chan)
660 {
661 struct owl_dma *od = to_owl_dma(chan->device);
662 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
663 unsigned long flags;
664 LIST_HEAD(head);
665
666 spin_lock_irqsave(&vchan->vc.lock, flags);
667
668 if (vchan->pchan)
669 owl_dma_phy_free(od, vchan);
670
671 if (vchan->txd) {
672 owl_dma_desc_free(&vchan->txd->vd);
673 vchan->txd = NULL;
674 }
675
676 vchan_get_all_descriptors(&vchan->vc, &head);
677
678 spin_unlock_irqrestore(&vchan->vc.lock, flags);
679
680 vchan_dma_desc_free_list(&vchan->vc, &head);
681
682 return 0;
683 }
684
685 static int owl_dma_config(struct dma_chan *chan,
686 struct dma_slave_config *config)
687 {
688 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
689
690 /* Reject definitely invalid configurations */
691 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
692 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
693 return -EINVAL;
694
695 memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config));
696
697 return 0;
698 }
699
700 static int owl_dma_pause(struct dma_chan *chan)
701 {
702 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
703 unsigned long flags;
704
705 spin_lock_irqsave(&vchan->vc.lock, flags);
706
707 owl_dma_pause_pchan(vchan->pchan);
708
709 spin_unlock_irqrestore(&vchan->vc.lock, flags);
710
711 return 0;
712 }
713
714 static int owl_dma_resume(struct dma_chan *chan)
715 {
716 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
717 unsigned long flags;
718
719 if (!vchan->pchan && !vchan->txd)
720 return 0;
721
722 dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
723
724 spin_lock_irqsave(&vchan->vc.lock, flags);
725
726 owl_dma_resume_pchan(vchan->pchan);
727
728 spin_unlock_irqrestore(&vchan->vc.lock, flags);
729
730 return 0;
731 }
732
733 static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)
734 {
735 struct owl_dma_pchan *pchan;
736 struct owl_dma_txd *txd;
737 struct owl_dma_lli *lli;
738 unsigned int next_lli_phy;
739 size_t bytes;
740
741 pchan = vchan->pchan;
742 txd = vchan->txd;
743
744 if (!pchan || !txd)
745 return 0;
746
747 /* Get remain count of current node in link list */
748 bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);
749
750 /* Loop through the preceding nodes to get total remaining bytes */
751 if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {
752 next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);
753 list_for_each_entry(lli, &txd->lli_list, node) {
754 /* Start from the next active node */
755 if (lli->phys == next_lli_phy) {
756 list_for_each_entry(lli, &txd->lli_list, node)
757 bytes += lli->hw.flen;
758 break;
759 }
760 }
761 }
762
763 return bytes;
764 }
765
766 static enum dma_status owl_dma_tx_status(struct dma_chan *chan,
767 dma_cookie_t cookie,
768 struct dma_tx_state *state)
769 {
770 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
771 struct owl_dma_lli *lli;
772 struct virt_dma_desc *vd;
773 struct owl_dma_txd *txd;
774 enum dma_status ret;
775 unsigned long flags;
776 size_t bytes = 0;
777
778 ret = dma_cookie_status(chan, cookie, state);
779 if (ret == DMA_COMPLETE || !state)
780 return ret;
781
782 spin_lock_irqsave(&vchan->vc.lock, flags);
783
784 vd = vchan_find_desc(&vchan->vc, cookie);
785 if (vd) {
786 txd = to_owl_txd(&vd->tx);
787 list_for_each_entry(lli, &txd->lli_list, node)
788 bytes += lli->hw.flen;
789 } else {
790 bytes = owl_dma_getbytes_chan(vchan);
791 }
792
793 spin_unlock_irqrestore(&vchan->vc.lock, flags);
794
795 dma_set_residue(state, bytes);
796
797 return ret;
798 }
799
800 static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)
801 {
802 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
803 struct owl_dma_pchan *pchan;
804
805 pchan = owl_dma_get_pchan(od, vchan);
806 if (!pchan)
807 return;
808
809 dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id);
810
811 vchan->pchan = pchan;
812 owl_dma_start_next_txd(vchan);
813 }
814
815 static void owl_dma_issue_pending(struct dma_chan *chan)
816 {
817 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
818 unsigned long flags;
819
820 spin_lock_irqsave(&vchan->vc.lock, flags);
821 if (vchan_issue_pending(&vchan->vc)) {
822 if (!vchan->pchan)
823 owl_dma_phy_alloc_and_start(vchan);
824 }
825 spin_unlock_irqrestore(&vchan->vc.lock, flags);
826 }
827
828 static struct dma_async_tx_descriptor
829 *owl_dma_prep_memcpy(struct dma_chan *chan,
830 dma_addr_t dst, dma_addr_t src,
831 size_t len, unsigned long flags)
832 {
833 struct owl_dma *od = to_owl_dma(chan->device);
834 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
835 struct owl_dma_txd *txd;
836 struct owl_dma_lli *lli, *prev = NULL;
837 size_t offset, bytes;
838 int ret;
839
840 if (!len)
841 return NULL;
842
843 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
844 if (!txd)
845 return NULL;
846
847 INIT_LIST_HEAD(&txd->lli_list);
848
849 /* Process the transfer as frame by frame */
850 for (offset = 0; offset < len; offset += bytes) {
851 lli = owl_dma_alloc_lli(od);
852 if (!lli) {
853 dev_warn(chan2dev(chan), "failed to allocate lli\n");
854 goto err_txd_free;
855 }
856
857 bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);
858
859 ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,
860 bytes, DMA_MEM_TO_MEM,
861 &vchan->cfg, txd->cyclic);
862 if (ret) {
863 dev_warn(chan2dev(chan), "failed to config lli\n");
864 goto err_txd_free;
865 }
866
867 prev = owl_dma_add_lli(txd, prev, lli, false);
868 }
869
870 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
871
872 err_txd_free:
873 owl_dma_free_txd(od, txd);
874 return NULL;
875 }
876
877 static struct dma_async_tx_descriptor
878 *owl_dma_prep_slave_sg(struct dma_chan *chan,
879 struct scatterlist *sgl,
880 unsigned int sg_len,
881 enum dma_transfer_direction dir,
882 unsigned long flags, void *context)
883 {
884 struct owl_dma *od = to_owl_dma(chan->device);
885 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
886 struct dma_slave_config *sconfig = &vchan->cfg;
887 struct owl_dma_txd *txd;
888 struct owl_dma_lli *lli, *prev = NULL;
889 struct scatterlist *sg;
890 dma_addr_t addr, src = 0, dst = 0;
891 size_t len;
892 int ret, i;
893
894 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
895 if (!txd)
896 return NULL;
897
898 INIT_LIST_HEAD(&txd->lli_list);
899
900 for_each_sg(sgl, sg, sg_len, i) {
901 addr = sg_dma_address(sg);
902 len = sg_dma_len(sg);
903
904 if (len > OWL_DMA_FRAME_MAX_LENGTH) {
905 dev_err(od->dma.dev,
906 "frame length exceeds max supported length");
907 goto err_txd_free;
908 }
909
910 lli = owl_dma_alloc_lli(od);
911 if (!lli) {
912 dev_err(chan2dev(chan), "failed to allocate lli");
913 goto err_txd_free;
914 }
915
916 if (dir == DMA_MEM_TO_DEV) {
917 src = addr;
918 dst = sconfig->dst_addr;
919 } else {
920 src = sconfig->src_addr;
921 dst = addr;
922 }
923
924 ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig,
925 txd->cyclic);
926 if (ret) {
927 dev_warn(chan2dev(chan), "failed to config lli");
928 goto err_txd_free;
929 }
930
931 prev = owl_dma_add_lli(txd, prev, lli, false);
932 }
933
934 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
935
936 err_txd_free:
937 owl_dma_free_txd(od, txd);
938
939 return NULL;
940 }
941
942 static struct dma_async_tx_descriptor
943 *owl_prep_dma_cyclic(struct dma_chan *chan,
944 dma_addr_t buf_addr, size_t buf_len,
945 size_t period_len,
946 enum dma_transfer_direction dir,
947 unsigned long flags)
948 {
949 struct owl_dma *od = to_owl_dma(chan->device);
950 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
951 struct dma_slave_config *sconfig = &vchan->cfg;
952 struct owl_dma_txd *txd;
953 struct owl_dma_lli *lli, *prev = NULL, *first = NULL;
954 dma_addr_t src = 0, dst = 0;
955 unsigned int periods = buf_len / period_len;
956 int ret, i;
957
958 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
959 if (!txd)
960 return NULL;
961
962 INIT_LIST_HEAD(&txd->lli_list);
963 txd->cyclic = true;
964
965 for (i = 0; i < periods; i++) {
966 lli = owl_dma_alloc_lli(od);
967 if (!lli) {
968 dev_warn(chan2dev(chan), "failed to allocate lli");
969 goto err_txd_free;
970 }
971
972 if (dir == DMA_MEM_TO_DEV) {
973 src = buf_addr + (period_len * i);
974 dst = sconfig->dst_addr;
975 } else if (dir == DMA_DEV_TO_MEM) {
976 src = sconfig->src_addr;
977 dst = buf_addr + (period_len * i);
978 }
979
980 ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len,
981 dir, sconfig, txd->cyclic);
982 if (ret) {
983 dev_warn(chan2dev(chan), "failed to config lli");
984 goto err_txd_free;
985 }
986
987 if (!first)
988 first = lli;
989
990 prev = owl_dma_add_lli(txd, prev, lli, false);
991 }
992
993 /* close the cyclic list */
994 owl_dma_add_lli(txd, prev, first, true);
995
996 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
997
998 err_txd_free:
999 owl_dma_free_txd(od, txd);
1000
1001 return NULL;
1002 }
1003
1004 static void owl_dma_free_chan_resources(struct dma_chan *chan)
1005 {
1006 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
1007
1008 /* Ensure all queued descriptors are freed */
1009 vchan_free_chan_resources(&vchan->vc);
1010 }
1011
1012 static inline void owl_dma_free(struct owl_dma *od)
1013 {
1014 struct owl_dma_vchan *vchan = NULL;
1015 struct owl_dma_vchan *next;
1016
1017 list_for_each_entry_safe(vchan,
1018 next, &od->dma.channels, vc.chan.device_node) {
1019 list_del(&vchan->vc.chan.device_node);
1020 tasklet_kill(&vchan->vc.task);
1021 }
1022 }
1023
1024 static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec,
1025 struct of_dma *ofdma)
1026 {
1027 struct owl_dma *od = ofdma->of_dma_data;
1028 struct owl_dma_vchan *vchan;
1029 struct dma_chan *chan;
1030 u8 drq = dma_spec->args[0];
1031
1032 if (drq > od->nr_vchans)
1033 return NULL;
1034
1035 chan = dma_get_any_slave_channel(&od->dma);
1036 if (!chan)
1037 return NULL;
1038
1039 vchan = to_owl_vchan(chan);
1040 vchan->drq = drq;
1041
1042 return chan;
1043 }
1044
1045 static int owl_dma_probe(struct platform_device *pdev)
1046 {
1047 struct device_node *np = pdev->dev.of_node;
1048 struct owl_dma *od;
1049 int ret, i, nr_channels, nr_requests;
1050
1051 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1052 if (!od)
1053 return -ENOMEM;
1054
1055 od->base = devm_platform_ioremap_resource(pdev, 0);
1056 if (IS_ERR(od->base))
1057 return PTR_ERR(od->base);
1058
1059 ret = of_property_read_u32(np, "dma-channels", &nr_channels);
1060 if (ret) {
1061 dev_err(&pdev->dev, "can't get dma-channels\n");
1062 return ret;
1063 }
1064
1065 ret = of_property_read_u32(np, "dma-requests", &nr_requests);
1066 if (ret) {
1067 dev_err(&pdev->dev, "can't get dma-requests\n");
1068 return ret;
1069 }
1070
1071 dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n",
1072 nr_channels, nr_requests);
1073
1074 od->nr_pchans = nr_channels;
1075 od->nr_vchans = nr_requests;
1076
1077 pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
1078
1079 platform_set_drvdata(pdev, od);
1080 spin_lock_init(&od->lock);
1081
1082 dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);
1083 dma_cap_set(DMA_SLAVE, od->dma.cap_mask);
1084 dma_cap_set(DMA_CYCLIC, od->dma.cap_mask);
1085
1086 od->dma.dev = &pdev->dev;
1087 od->dma.device_free_chan_resources = owl_dma_free_chan_resources;
1088 od->dma.device_tx_status = owl_dma_tx_status;
1089 od->dma.device_issue_pending = owl_dma_issue_pending;
1090 od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;
1091 od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg;
1092 od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic;
1093 od->dma.device_config = owl_dma_config;
1094 od->dma.device_pause = owl_dma_pause;
1095 od->dma.device_resume = owl_dma_resume;
1096 od->dma.device_terminate_all = owl_dma_terminate_all;
1097 od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1098 od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1099 od->dma.directions = BIT(DMA_MEM_TO_MEM);
1100 od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1101
1102 INIT_LIST_HEAD(&od->dma.channels);
1103
1104 od->clk = devm_clk_get(&pdev->dev, NULL);
1105 if (IS_ERR(od->clk)) {
1106 dev_err(&pdev->dev, "unable to get clock\n");
1107 return PTR_ERR(od->clk);
1108 }
1109
1110 /*
1111 * Eventhough the DMA controller is capable of generating 4
1112 * IRQ's for DMA priority feature, we only use 1 IRQ for
1113 * simplification.
1114 */
1115 od->irq = platform_get_irq(pdev, 0);
1116 ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,
1117 dev_name(&pdev->dev), od);
1118 if (ret) {
1119 dev_err(&pdev->dev, "unable to request IRQ\n");
1120 return ret;
1121 }
1122
1123 /* Init physical channel */
1124 od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,
1125 sizeof(struct owl_dma_pchan), GFP_KERNEL);
1126 if (!od->pchans)
1127 return -ENOMEM;
1128
1129 for (i = 0; i < od->nr_pchans; i++) {
1130 struct owl_dma_pchan *pchan = &od->pchans[i];
1131
1132 pchan->id = i;
1133 pchan->base = od->base + OWL_DMA_CHAN_BASE(i);
1134 }
1135
1136 /* Init virtual channel */
1137 od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,
1138 sizeof(struct owl_dma_vchan), GFP_KERNEL);
1139 if (!od->vchans)
1140 return -ENOMEM;
1141
1142 for (i = 0; i < od->nr_vchans; i++) {
1143 struct owl_dma_vchan *vchan = &od->vchans[i];
1144
1145 vchan->vc.desc_free = owl_dma_desc_free;
1146 vchan_init(&vchan->vc, &od->dma);
1147 }
1148
1149 /* Create a pool of consistent memory blocks for hardware descriptors */
1150 od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,
1151 sizeof(struct owl_dma_lli),
1152 __alignof__(struct owl_dma_lli),
1153 0);
1154 if (!od->lli_pool) {
1155 dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n");
1156 return -ENOMEM;
1157 }
1158
1159 clk_prepare_enable(od->clk);
1160
1161 ret = dma_async_device_register(&od->dma);
1162 if (ret) {
1163 dev_err(&pdev->dev, "failed to register DMA engine device\n");
1164 goto err_pool_free;
1165 }
1166
1167 /* Device-tree DMA controller registration */
1168 ret = of_dma_controller_register(pdev->dev.of_node,
1169 owl_dma_of_xlate, od);
1170 if (ret) {
1171 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1172 goto err_dma_unregister;
1173 }
1174
1175 return 0;
1176
1177 err_dma_unregister:
1178 dma_async_device_unregister(&od->dma);
1179 err_pool_free:
1180 clk_disable_unprepare(od->clk);
1181 dma_pool_destroy(od->lli_pool);
1182
1183 return ret;
1184 }
1185
1186 static int owl_dma_remove(struct platform_device *pdev)
1187 {
1188 struct owl_dma *od = platform_get_drvdata(pdev);
1189
1190 of_dma_controller_free(pdev->dev.of_node);
1191 dma_async_device_unregister(&od->dma);
1192
1193 /* Mask all interrupts for this execution environment */
1194 dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);
1195
1196 /* Make sure we won't have any further interrupts */
1197 devm_free_irq(od->dma.dev, od->irq, od);
1198
1199 owl_dma_free(od);
1200
1201 clk_disable_unprepare(od->clk);
1202
1203 return 0;
1204 }
1205
1206 static const struct of_device_id owl_dma_match[] = {
1207 { .compatible = "actions,s900-dma", },
1208 { /* sentinel */ }
1209 };
1210 MODULE_DEVICE_TABLE(of, owl_dma_match);
1211
1212 static struct platform_driver owl_dma_driver = {
1213 .probe = owl_dma_probe,
1214 .remove = owl_dma_remove,
1215 .driver = {
1216 .name = "dma-owl",
1217 .of_match_table = of_match_ptr(owl_dma_match),
1218 },
1219 };
1220
1221 static int owl_dma_init(void)
1222 {
1223 return platform_driver_register(&owl_dma_driver);
1224 }
1225 subsys_initcall(owl_dma_init);
1226
1227 static void __exit owl_dma_exit(void)
1228 {
1229 platform_driver_unregister(&owl_dma_driver);
1230 }
1231 module_exit(owl_dma_exit);
1232
1233 MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
1234 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
1235 MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
1236 MODULE_LICENSE("GPL");
A mirror of Linus' kernel repository