drivers/dma/img-mdc-dma.c

   1 /*
   2  * IMG Multi-threaded DMA Controller (MDC)
   3  *
   4  * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
   5  * Copyright (C) 2014 Google, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2, as published by the Free Software Foundation.
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/dmapool.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/io.h>
  18 #include <linux/irq.h>
  19 #include <linux/kernel.h>
  20 #include <linux/mfd/syscon.h>
  21 #include <linux/module.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/of_dma.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/pm_runtime.h>
  27 #include <linux/regmap.h>
  28 #include <linux/slab.h>
  29 #include <linux/spinlock.h>
  30
  31 #include "dmaengine.h"
  32 #include "virt-dma.h"
  33
  34 #define MDC_MAX_DMA_CHANNELS                    32
  35
  36 #define MDC_GENERAL_CONFIG                      0x000
  37 #define MDC_GENERAL_CONFIG_LIST_IEN             BIT(31)
  38 #define MDC_GENERAL_CONFIG_IEN                  BIT(29)
  39 #define MDC_GENERAL_CONFIG_LEVEL_INT            BIT(28)
  40 #define MDC_GENERAL_CONFIG_INC_W                BIT(12)
  41 #define MDC_GENERAL_CONFIG_INC_R                BIT(8)
  42 #define MDC_GENERAL_CONFIG_PHYSICAL_W           BIT(7)
  43 #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT        4
  44 #define MDC_GENERAL_CONFIG_WIDTH_W_MASK         0x7
  45 #define MDC_GENERAL_CONFIG_PHYSICAL_R           BIT(3)
  46 #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT        0
  47 #define MDC_GENERAL_CONFIG_WIDTH_R_MASK         0x7
  48
  49 #define MDC_READ_PORT_CONFIG                    0x004
  50 #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT      28
  51 #define MDC_READ_PORT_CONFIG_STHREAD_MASK       0xf
  52 #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT      24
  53 #define MDC_READ_PORT_CONFIG_RTHREAD_MASK       0xf
  54 #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT      16
  55 #define MDC_READ_PORT_CONFIG_WTHREAD_MASK       0xf
  56 #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT   4
  57 #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK    0xff
  58 #define MDC_READ_PORT_CONFIG_DREQ_ENABLE        BIT(1)
  59
  60 #define MDC_READ_ADDRESS                        0x008
  61
  62 #define MDC_WRITE_ADDRESS                       0x00c
  63
  64 #define MDC_TRANSFER_SIZE                       0x010
  65 #define MDC_TRANSFER_SIZE_MASK                  0xffffff
  66
  67 #define MDC_LIST_NODE_ADDRESS                   0x014
  68
  69 #define MDC_CMDS_PROCESSED                      0x018
  70 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
  71 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK  0x3f
  72 #define MDC_CMDS_PROCESSED_INT_ACTIVE           BIT(8)
  73 #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT      0
  74 #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK       0x3f
  75
  76 #define MDC_CONTROL_AND_STATUS                  0x01c
  77 #define MDC_CONTROL_AND_STATUS_CANCEL           BIT(20)
  78 #define MDC_CONTROL_AND_STATUS_LIST_EN          BIT(4)
  79 #define MDC_CONTROL_AND_STATUS_EN               BIT(0)
  80
  81 #define MDC_ACTIVE_TRANSFER_SIZE                0x030
  82
  83 #define MDC_GLOBAL_CONFIG_A                             0x900
  84 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT       16
  85 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK        0xff
  86 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT          8
  87 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK           0xff
  88 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT         0
  89 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK          0xff
  90
  91 struct mdc_hw_list_desc {
  92         u32 gen_conf;
  93         u32 readport_conf;
  94         u32 read_addr;
  95         u32 write_addr;
  96         u32 xfer_size;
  97         u32 node_addr;
  98         u32 cmds_done;
  99         u32 ctrl_status;
 100         /*
 101          * Not part of the list descriptor, but instead used by the CPU to
 102          * traverse the list.
 103          */
 104         struct mdc_hw_list_desc *next_desc;
 105 };
 106
 107 struct mdc_tx_desc {
 108         struct mdc_chan *chan;
 109         struct virt_dma_desc vd;
 110         dma_addr_t list_phys;
 111         struct mdc_hw_list_desc *list;
 112         bool cyclic;
 113         bool cmd_loaded;
 114         unsigned int list_len;
 115         unsigned int list_period_len;
 116         size_t list_xfer_size;
 117         unsigned int list_cmds_done;
 118 };
 119
 120 struct mdc_chan {
 121         struct mdc_dma *mdma;
 122         struct virt_dma_chan vc;
 123         struct dma_slave_config config;
 124         struct mdc_tx_desc *desc;
 125         int irq;
 126         unsigned int periph;
 127         unsigned int thread;
 128         unsigned int chan_nr;
 129 };
 130
 131 struct mdc_dma_soc_data {
 132         void (*enable_chan)(struct mdc_chan *mchan);
 133         void (*disable_chan)(struct mdc_chan *mchan);
 134 };
 135
 136 struct mdc_dma {
 137         struct dma_device dma_dev;
 138         void __iomem *regs;
 139         struct clk *clk;
 140         struct dma_pool *desc_pool;
 141         struct regmap *periph_regs;
 142         spinlock_t lock;
 143         unsigned int nr_threads;
 144         unsigned int nr_channels;
 145         unsigned int bus_width;
 146         unsigned int max_burst_mult;
 147         unsigned int max_xfer_size;
 148         const struct mdc_dma_soc_data *soc;
 149         struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
 150 };
 151
 152 static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
 153 {
 154         return readl(mdma->regs + reg);
 155 }
 156
 157 static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
 158 {
 159         writel(val, mdma->regs + reg);
 160 }
 161
 162 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
 163 {
 164         return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
 165 }
 166
 167 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
 168 {
 169         mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
 170 }
 171
 172 static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
 173 {
 174         return container_of(to_virt_chan(c), struct mdc_chan, vc);
 175 }
 176
 177 static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
 178 {
 179         struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
 180
 181         return container_of(vdesc, struct mdc_tx_desc, vd);
 182 }
 183
 184 static inline struct device *mdma2dev(struct mdc_dma *mdma)
 185 {
 186         return mdma->dma_dev.dev;
 187 }
 188
 189 static inline unsigned int to_mdc_width(unsigned int bytes)
 190 {
 191         return ffs(bytes) - 1;
 192 }
 193
 194 static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
 195                                       unsigned int bytes)
 196 {
 197         ldesc->gen_conf |= to_mdc_width(bytes) <<
 198                 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
 199 }
 200
 201 static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
 202                                        unsigned int bytes)
 203 {
 204         ldesc->gen_conf |= to_mdc_width(bytes) <<
 205                 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
 206 }
 207
 208 static void mdc_list_desc_config(struct mdc_chan *mchan,
 209                                  struct mdc_hw_list_desc *ldesc,
 210                                  enum dma_transfer_direction dir,
 211                                  dma_addr_t src, dma_addr_t dst, size_t len)
 212 {
 213         struct mdc_dma *mdma = mchan->mdma;
 214         unsigned int max_burst, burst_size;
 215
 216         ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
 217                 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
 218                 MDC_GENERAL_CONFIG_PHYSICAL_R;
 219         ldesc->readport_conf =
 220                 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
 221                 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
 222                 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
 223         ldesc->read_addr = src;
 224         ldesc->write_addr = dst;
 225         ldesc->xfer_size = len - 1;
 226         ldesc->node_addr = 0;
 227         ldesc->cmds_done = 0;
 228         ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
 229                 MDC_CONTROL_AND_STATUS_EN;
 230         ldesc->next_desc = NULL;
 231
 232         if (IS_ALIGNED(dst, mdma->bus_width) &&
 233             IS_ALIGNED(src, mdma->bus_width))
 234                 max_burst = mdma->bus_width * mdma->max_burst_mult;
 235         else
 236                 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
 237
 238         if (dir == DMA_MEM_TO_DEV) {
 239                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
 240                 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
 241                 mdc_set_read_width(ldesc, mdma->bus_width);
 242                 mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
 243                 burst_size = min(max_burst, mchan->config.dst_maxburst *
 244                                  mchan->config.dst_addr_width);
 245         } else if (dir == DMA_DEV_TO_MEM) {
 246                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
 247                 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
 248                 mdc_set_read_width(ldesc, mchan->config.src_addr_width);
 249                 mdc_set_write_width(ldesc, mdma->bus_width);
 250                 burst_size = min(max_burst, mchan->config.src_maxburst *
 251                                  mchan->config.src_addr_width);
 252         } else {
 253                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
 254                         MDC_GENERAL_CONFIG_INC_W;
 255                 mdc_set_read_width(ldesc, mdma->bus_width);
 256                 mdc_set_write_width(ldesc, mdma->bus_width);
 257                 burst_size = max_burst;
 258         }
 259         ldesc->readport_conf |= (burst_size - 1) <<
 260                 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
 261 }
 262
 263 static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
 264 {
 265         struct mdc_dma *mdma = mdesc->chan->mdma;
 266         struct mdc_hw_list_desc *curr, *next;
 267         dma_addr_t curr_phys, next_phys;
 268
 269         curr = mdesc->list;
 270         curr_phys = mdesc->list_phys;
 271         while (curr) {
 272                 next = curr->next_desc;
 273                 next_phys = curr->node_addr;
 274                 dma_pool_free(mdma->desc_pool, curr, curr_phys);
 275                 curr = next;
 276                 curr_phys = next_phys;
 277         }
 278 }
 279
 280 static void mdc_desc_free(struct virt_dma_desc *vd)
 281 {
 282         struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
 283
 284         mdc_list_desc_free(mdesc);
 285         kfree(mdesc);
 286 }
 287
 288 static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
 289         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
 290         unsigned long flags)
 291 {
 292         struct mdc_chan *mchan = to_mdc_chan(chan);
 293         struct mdc_dma *mdma = mchan->mdma;
 294         struct mdc_tx_desc *mdesc;
 295         struct mdc_hw_list_desc *curr, *prev = NULL;
 296         dma_addr_t curr_phys;
 297
 298         if (!len)
 299                 return NULL;
 300
 301         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 302         if (!mdesc)
 303                 return NULL;
 304         mdesc->chan = mchan;
 305         mdesc->list_xfer_size = len;
 306
 307         while (len > 0) {
 308                 size_t xfer_size;
 309
 310                 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
 311                 if (!curr)
 312                         goto free_desc;
 313
 314                 if (prev) {
 315                         prev->node_addr = curr_phys;
 316                         prev->next_desc = curr;
 317                 } else {
 318                         mdesc->list_phys = curr_phys;
 319                         mdesc->list = curr;
 320                 }
 321
 322                 xfer_size = min_t(size_t, mdma->max_xfer_size, len);
 323
 324                 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
 325                                      xfer_size);
 326
 327                 prev = curr;
 328
 329                 mdesc->list_len++;
 330                 src += xfer_size;
 331                 dest += xfer_size;
 332                 len -= xfer_size;
 333         }
 334
 335         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 336
 337 free_desc:
 338         mdc_desc_free(&mdesc->vd);
 339
 340         return NULL;
 341 }
 342
 343 static int mdc_check_slave_width(struct mdc_chan *mchan,
 344                                  enum dma_transfer_direction dir)
 345 {
 346         enum dma_slave_buswidth width;
 347
 348         if (dir == DMA_MEM_TO_DEV)
 349                 width = mchan->config.dst_addr_width;
 350         else
 351                 width = mchan->config.src_addr_width;
 352
 353         switch (width) {
 354         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 355         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 356         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 357         case DMA_SLAVE_BUSWIDTH_8_BYTES:
 358                 break;
 359         default:
 360                 return -EINVAL;
 361         }
 362
 363         if (width > mchan->mdma->bus_width)
 364                 return -EINVAL;
 365
 366         return 0;
 367 }
 368
 369 static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
 370         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 371         size_t period_len, enum dma_transfer_direction dir,
 372         unsigned long flags)
 373 {
 374         struct mdc_chan *mchan = to_mdc_chan(chan);
 375         struct mdc_dma *mdma = mchan->mdma;
 376         struct mdc_tx_desc *mdesc;
 377         struct mdc_hw_list_desc *curr, *prev = NULL;
 378         dma_addr_t curr_phys;
 379
 380         if (!buf_len && !period_len)
 381                 return NULL;
 382
 383         if (!is_slave_direction(dir))
 384                 return NULL;
 385
 386         if (mdc_check_slave_width(mchan, dir) < 0)
 387                 return NULL;
 388
 389         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 390         if (!mdesc)
 391                 return NULL;
 392         mdesc->chan = mchan;
 393         mdesc->cyclic = true;
 394         mdesc->list_xfer_size = buf_len;
 395         mdesc->list_period_len = DIV_ROUND_UP(period_len,
 396                                               mdma->max_xfer_size);
 397
 398         while (buf_len > 0) {
 399                 size_t remainder = min(period_len, buf_len);
 400
 401                 while (remainder > 0) {
 402                         size_t xfer_size;
 403
 404                         curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
 405                                               &curr_phys);
 406                         if (!curr)
 407                                 goto free_desc;
 408
 409                         if (!prev) {
 410                                 mdesc->list_phys = curr_phys;
 411                                 mdesc->list = curr;
 412                         } else {
 413                                 prev->node_addr = curr_phys;
 414                                 prev->next_desc = curr;
 415                         }
 416
 417                         xfer_size = min_t(size_t, mdma->max_xfer_size,
 418                                           remainder);
 419
 420                         if (dir == DMA_MEM_TO_DEV) {
 421                                 mdc_list_desc_config(mchan, curr, dir,
 422                                                      buf_addr,
 423                                                      mchan->config.dst_addr,
 424                                                      xfer_size);
 425                         } else {
 426                                 mdc_list_desc_config(mchan, curr, dir,
 427                                                      mchan->config.src_addr,
 428                                                      buf_addr,
 429                                                      xfer_size);
 430                         }
 431
 432                         prev = curr;
 433
 434                         mdesc->list_len++;
 435                         buf_addr += xfer_size;
 436                         buf_len -= xfer_size;
 437                         remainder -= xfer_size;
 438                 }
 439         }
 440         prev->node_addr = mdesc->list_phys;
 441
 442         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 443
 444 free_desc:
 445         mdc_desc_free(&mdesc->vd);
 446
 447         return NULL;
 448 }
 449
 450 static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
 451         struct dma_chan *chan, struct scatterlist *sgl,
 452         unsigned int sg_len, enum dma_transfer_direction dir,
 453         unsigned long flags, void *context)
 454 {
 455         struct mdc_chan *mchan = to_mdc_chan(chan);
 456         struct mdc_dma *mdma = mchan->mdma;
 457         struct mdc_tx_desc *mdesc;
 458         struct scatterlist *sg;
 459         struct mdc_hw_list_desc *curr, *prev = NULL;
 460         dma_addr_t curr_phys;
 461         unsigned int i;
 462
 463         if (!sgl)
 464                 return NULL;
 465
 466         if (!is_slave_direction(dir))
 467                 return NULL;
 468
 469         if (mdc_check_slave_width(mchan, dir) < 0)
 470                 return NULL;
 471
 472         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 473         if (!mdesc)
 474                 return NULL;
 475         mdesc->chan = mchan;
 476
 477         for_each_sg(sgl, sg, sg_len, i) {
 478                 dma_addr_t buf = sg_dma_address(sg);
 479                 size_t buf_len = sg_dma_len(sg);
 480
 481                 while (buf_len > 0) {
 482                         size_t xfer_size;
 483
 484                         curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
 485                                               &curr_phys);
 486                         if (!curr)
 487                                 goto free_desc;
 488
 489                         if (!prev) {
 490                                 mdesc->list_phys = curr_phys;
 491                                 mdesc->list = curr;
 492                         } else {
 493                                 prev->node_addr = curr_phys;
 494                                 prev->next_desc = curr;
 495                         }
 496
 497                         xfer_size = min_t(size_t, mdma->max_xfer_size,
 498                                           buf_len);
 499
 500                         if (dir == DMA_MEM_TO_DEV) {
 501                                 mdc_list_desc_config(mchan, curr, dir, buf,
 502                                                      mchan->config.dst_addr,
 503                                                      xfer_size);
 504                         } else {
 505                                 mdc_list_desc_config(mchan, curr, dir,
 506                                                      mchan->config.src_addr,
 507                                                      buf, xfer_size);
 508                         }
 509
 510                         prev = curr;
 511
 512                         mdesc->list_len++;
 513                         mdesc->list_xfer_size += xfer_size;
 514                         buf += xfer_size;
 515                         buf_len -= xfer_size;
 516                 }
 517         }
 518
 519         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 520
 521 free_desc:
 522         mdc_desc_free(&mdesc->vd);
 523
 524         return NULL;
 525 }
 526
 527 static void mdc_issue_desc(struct mdc_chan *mchan)
 528 {
 529         struct mdc_dma *mdma = mchan->mdma;
 530         struct virt_dma_desc *vd;
 531         struct mdc_tx_desc *mdesc;
 532         u32 val;
 533
 534         vd = vchan_next_desc(&mchan->vc);
 535         if (!vd)
 536                 return;
 537
 538         list_del(&vd->node);
 539
 540         mdesc = to_mdc_desc(&vd->tx);
 541         mchan->desc = mdesc;
 542
 543         dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
 544                 mchan->chan_nr);
 545
 546         mdma->soc->enable_chan(mchan);
 547
 548         val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
 549         val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
 550                 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
 551                 MDC_GENERAL_CONFIG_PHYSICAL_R;
 552         mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
 553         val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
 554                 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
 555                 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
 556         mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
 557         mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
 558         val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
 559         val |= MDC_CONTROL_AND_STATUS_LIST_EN;
 560         mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
 561 }
 562
 563 static void mdc_issue_pending(struct dma_chan *chan)
 564 {
 565         struct mdc_chan *mchan = to_mdc_chan(chan);
 566         unsigned long flags;
 567
 568         spin_lock_irqsave(&mchan->vc.lock, flags);
 569         if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
 570                 mdc_issue_desc(mchan);
 571         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 572 }
 573
 574 static enum dma_status mdc_tx_status(struct dma_chan *chan,
 575         dma_cookie_t cookie, struct dma_tx_state *txstate)
 576 {
 577         struct mdc_chan *mchan = to_mdc_chan(chan);
 578         struct mdc_tx_desc *mdesc;
 579         struct virt_dma_desc *vd;
 580         unsigned long flags;
 581         size_t bytes = 0;
 582         int ret;
 583
 584         ret = dma_cookie_status(chan, cookie, txstate);
 585         if (ret == DMA_COMPLETE)
 586                 return ret;
 587
 588         if (!txstate)
 589                 return ret;
 590
 591         spin_lock_irqsave(&mchan->vc.lock, flags);
 592         vd = vchan_find_desc(&mchan->vc, cookie);
 593         if (vd) {
 594                 mdesc = to_mdc_desc(&vd->tx);
 595                 bytes = mdesc->list_xfer_size;
 596         } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
 597                 struct mdc_hw_list_desc *ldesc;
 598                 u32 val1, val2, done, processed, residue;
 599                 int i, cmds;
 600
 601                 mdesc = mchan->desc;
 602
 603                 /*
 604                  * Determine the number of commands that haven't been
 605                  * processed (handled by the IRQ handler) yet.
 606                  */
 607                 do {
 608                         val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
 609                                 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
 610                         residue = mdc_chan_readl(mchan,
 611                                                  MDC_ACTIVE_TRANSFER_SIZE);
 612                         val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
 613                                 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
 614                 } while (val1 != val2);
 615
 616                 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 617                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 618                 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
 619                         MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
 620                 cmds = (done - processed) %
 621                         (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
 622
 623                 /*
 624                  * If the command loaded event hasn't been processed yet, then
 625                  * the difference above includes an extra command.
 626                  */
 627                 if (!mdesc->cmd_loaded)
 628                         cmds--;
 629                 else
 630                         cmds += mdesc->list_cmds_done;
 631
 632                 bytes = mdesc->list_xfer_size;
 633                 ldesc = mdesc->list;
 634                 for (i = 0; i < cmds; i++) {
 635                         bytes -= ldesc->xfer_size + 1;
 636                         ldesc = ldesc->next_desc;
 637                 }
 638                 if (ldesc) {
 639                         if (residue != MDC_TRANSFER_SIZE_MASK)
 640                                 bytes -= ldesc->xfer_size - residue;
 641                         else
 642                                 bytes -= ldesc->xfer_size + 1;
 643                 }
 644         }
 645         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 646
 647         dma_set_residue(txstate, bytes);
 648
 649         return ret;
 650 }
 651
 652 static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
 653 {
 654         u32 val, processed, done1, done2;
 655         unsigned int ret;
 656
 657         val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 658         processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
 659                                 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
 660         /*
 661          * CMDS_DONE may have incremented between reading CMDS_PROCESSED
 662          * and clearing INT_ACTIVE.  Re-read CMDS_PROCESSED to ensure we
 663          * didn't miss a command completion.
 664          */
 665         do {
 666                 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 667
 668                 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 669                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 670
 671                 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
 672                           MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
 673                          MDC_CMDS_PROCESSED_INT_ACTIVE);
 674
 675                 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
 676
 677                 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
 678
 679                 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 680
 681                 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 682                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 683         } while (done1 != done2);
 684
 685         if (done1 >= processed)
 686                 ret = done1 - processed;
 687         else
 688                 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
 689                         processed) + done1;
 690
 691         return ret;
 692 }
 693
 694 static int mdc_terminate_all(struct dma_chan *chan)
 695 {
 696         struct mdc_chan *mchan = to_mdc_chan(chan);
 697         unsigned long flags;
 698         LIST_HEAD(head);
 699
 700         spin_lock_irqsave(&mchan->vc.lock, flags);
 701
 702         mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
 703                         MDC_CONTROL_AND_STATUS);
 704
 705         if (mchan->desc) {
 706                 vchan_terminate_vdesc(&mchan->desc->vd);
 707                 mchan->desc = NULL;
 708         }
 709         vchan_get_all_descriptors(&mchan->vc, &head);
 710
 711         mdc_get_new_events(mchan);
 712
 713         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 714
 715         vchan_dma_desc_free_list(&mchan->vc, &head);
 716
 717         return 0;
 718 }
 719
 720 static void mdc_synchronize(struct dma_chan *chan)
 721 {
 722         struct mdc_chan *mchan = to_mdc_chan(chan);
 723
 724         vchan_synchronize(&mchan->vc);
 725 }
 726
 727 static int mdc_slave_config(struct dma_chan *chan,
 728                             struct dma_slave_config *config)
 729 {
 730         struct mdc_chan *mchan = to_mdc_chan(chan);
 731         unsigned long flags;
 732
 733         spin_lock_irqsave(&mchan->vc.lock, flags);
 734         mchan->config = *config;
 735         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 736
 737         return 0;
 738 }
 739
 740 static int mdc_alloc_chan_resources(struct dma_chan *chan)
 741 {
 742         struct mdc_chan *mchan = to_mdc_chan(chan);
 743         struct device *dev = mdma2dev(mchan->mdma);
 744
 745         return pm_runtime_get_sync(dev);
 746 }
 747
 748 static void mdc_free_chan_resources(struct dma_chan *chan)
 749 {
 750         struct mdc_chan *mchan = to_mdc_chan(chan);
 751         struct mdc_dma *mdma = mchan->mdma;
 752         struct device *dev = mdma2dev(mdma);
 753
 754         mdc_terminate_all(chan);
 755         mdma->soc->disable_chan(mchan);
 756         pm_runtime_put(dev);
 757 }
 758
 759 static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
 760 {
 761         struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
 762         struct mdc_tx_desc *mdesc;
 763         unsigned int i, new_events;
 764
 765         spin_lock(&mchan->vc.lock);
 766
 767         dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
 768
 769         new_events = mdc_get_new_events(mchan);
 770
 771         if (!new_events)
 772                 goto out;
 773
 774         mdesc = mchan->desc;
 775         if (!mdesc) {
 776                 dev_warn(mdma2dev(mchan->mdma),
 777                          "IRQ with no active descriptor on channel %d\n",
 778                          mchan->chan_nr);
 779                 goto out;
 780         }
 781
 782         for (i = 0; i < new_events; i++) {
 783                 /*
 784                  * The first interrupt in a transfer indicates that the
 785                  * command list has been loaded, not that a command has
 786                  * been completed.
 787                  */
 788                 if (!mdesc->cmd_loaded) {
 789                         mdesc->cmd_loaded = true;
 790                         continue;
 791                 }
 792
 793                 mdesc->list_cmds_done++;
 794                 if (mdesc->cyclic) {
 795                         mdesc->list_cmds_done %= mdesc->list_len;
 796                         if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
 797                                 vchan_cyclic_callback(&mdesc->vd);
 798                 } else if (mdesc->list_cmds_done == mdesc->list_len) {
 799                         mchan->desc = NULL;
 800                         vchan_cookie_complete(&mdesc->vd);
 801                         mdc_issue_desc(mchan);
 802                         break;
 803                 }
 804         }
 805 out:
 806         spin_unlock(&mchan->vc.lock);
 807
 808         return IRQ_HANDLED;
 809 }
 810
 811 static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
 812                                      struct of_dma *ofdma)
 813 {
 814         struct mdc_dma *mdma = ofdma->of_dma_data;
 815         struct dma_chan *chan;
 816
 817         if (dma_spec->args_count != 3)
 818                 return NULL;
 819
 820         list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
 821                 struct mdc_chan *mchan = to_mdc_chan(chan);
 822
 823                 if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
 824                         continue;
 825                 if (dma_get_slave_channel(chan)) {
 826                         mchan->periph = dma_spec->args[0];
 827                         mchan->thread = dma_spec->args[2];
 828                         return chan;
 829                 }
 830         }
 831
 832         return NULL;
 833 }
 834
 835 #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch)       (0x120 + 0x4 * ((ch) / 4))
 836 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
 837 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK      0x3f
 838
 839 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
 840 {
 841         struct mdc_dma *mdma = mchan->mdma;
 842
 843         regmap_update_bits(mdma->periph_regs,
 844                            PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
 845                            PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
 846                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
 847                            mchan->periph <<
 848                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
 849 }
 850
 851 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
 852 {
 853         struct mdc_dma *mdma = mchan->mdma;
 854
 855         regmap_update_bits(mdma->periph_regs,
 856                            PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
 857                            PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
 858                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
 859                            0);
 860 }
 861
 862 static const struct mdc_dma_soc_data pistachio_mdc_data = {
 863         .enable_chan = pistachio_mdc_enable_chan,
 864         .disable_chan = pistachio_mdc_disable_chan,
 865 };
 866
 867 static const struct of_device_id mdc_dma_of_match[] = {
 868         { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
 869         { },
 870 };
 871 MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
 872
 873 static int img_mdc_runtime_suspend(struct device *dev)
 874 {
 875         struct mdc_dma *mdma = dev_get_drvdata(dev);
 876
 877         clk_disable_unprepare(mdma->clk);
 878
 879         return 0;
 880 }
 881
 882 static int img_mdc_runtime_resume(struct device *dev)
 883 {
 884         struct mdc_dma *mdma = dev_get_drvdata(dev);
 885
 886         return clk_prepare_enable(mdma->clk);
 887 }
 888
 889 static int mdc_dma_probe(struct platform_device *pdev)
 890 {
 891         struct mdc_dma *mdma;
 892         struct resource *res;
 893         unsigned int i;
 894         u32 val;
 895         int ret;
 896
 897         mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
 898         if (!mdma)
 899                 return -ENOMEM;
 900         platform_set_drvdata(pdev, mdma);
 901
 902         mdma->soc = of_device_get_match_data(&pdev->dev);
 903
 904         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 905         mdma->regs = devm_ioremap_resource(&pdev->dev, res);
 906         if (IS_ERR(mdma->regs))
 907                 return PTR_ERR(mdma->regs);
 908
 909         mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 910                                                             "img,cr-periph");
 911         if (IS_ERR(mdma->periph_regs))
 912                 return PTR_ERR(mdma->periph_regs);
 913
 914         mdma->clk = devm_clk_get(&pdev->dev, "sys");
 915         if (IS_ERR(mdma->clk))
 916                 return PTR_ERR(mdma->clk);
 917
 918         dma_cap_zero(mdma->dma_dev.cap_mask);
 919         dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
 920         dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
 921         dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
 922         dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
 923
 924         val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
 925         mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
 926                 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
 927         mdma->nr_threads =
 928                 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
 929                       MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
 930         mdma->bus_width =
 931                 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
 932                        MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
 933         /*
 934          * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
 935          * are supported, this makes it possible for the value reported in
 936          * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
 937          * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
 938          * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining.  To eliminate this
 939          * ambiguity, restrict transfer sizes to one bus-width less than the
 940          * actual maximum.
 941          */
 942         mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
 943
 944         of_property_read_u32(pdev->dev.of_node, "dma-channels",
 945                              &mdma->nr_channels);
 946         ret = of_property_read_u32(pdev->dev.of_node,
 947                                    "img,max-burst-multiplier",
 948                                    &mdma->max_burst_mult);
 949         if (ret)
 950                 return ret;
 951
 952         mdma->dma_dev.dev = &pdev->dev;
 953         mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
 954         mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
 955         mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
 956         mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;
 957         mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
 958         mdma->dma_dev.device_tx_status = mdc_tx_status;
 959         mdma->dma_dev.device_issue_pending = mdc_issue_pending;
 960         mdma->dma_dev.device_terminate_all = mdc_terminate_all;
 961         mdma->dma_dev.device_synchronize = mdc_synchronize;
 962         mdma->dma_dev.device_config = mdc_slave_config;
 963
 964         mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 965         mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 966         for (i = 1; i <= mdma->bus_width; i <<= 1) {
 967                 mdma->dma_dev.src_addr_widths |= BIT(i);
 968                 mdma->dma_dev.dst_addr_widths |= BIT(i);
 969         }
 970
 971         INIT_LIST_HEAD(&mdma->dma_dev.channels);
 972         for (i = 0; i < mdma->nr_channels; i++) {
 973                 struct mdc_chan *mchan = &mdma->channels[i];
 974
 975                 mchan->mdma = mdma;
 976                 mchan->chan_nr = i;
 977                 mchan->irq = platform_get_irq(pdev, i);
 978                 if (mchan->irq < 0)
 979                         return mchan->irq;
 980
 981                 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
 982                                        IRQ_TYPE_LEVEL_HIGH,
 983                                        dev_name(&pdev->dev), mchan);
 984                 if (ret < 0)
 985                         return ret;
 986
 987                 mchan->vc.desc_free = mdc_desc_free;
 988                 vchan_init(&mchan->vc, &mdma->dma_dev);
 989         }
 990
 991         mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
 992                                            sizeof(struct mdc_hw_list_desc),
 993                                            4, 0);
 994         if (!mdma->desc_pool)
 995                 return -ENOMEM;
 996
 997         pm_runtime_enable(&pdev->dev);
 998         if (!pm_runtime_enabled(&pdev->dev)) {
 999                 ret = img_mdc_runtime_resume(&pdev->dev);
1000                 if (ret)
1001                         return ret;
1002         }
1003
1004         ret = dma_async_device_register(&mdma->dma_dev);
1005         if (ret)
1006                 goto suspend;
1007
1008         ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
1009         if (ret)
1010                 goto unregister;
1011
1012         dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
1013                  mdma->nr_channels, mdma->nr_threads);
1014
1015         return 0;
1016
1017 unregister:
1018         dma_async_device_unregister(&mdma->dma_dev);
1019 suspend:
1020         if (!pm_runtime_enabled(&pdev->dev))
1021                 img_mdc_runtime_suspend(&pdev->dev);
1022         pm_runtime_disable(&pdev->dev);
1023         return ret;
1024 }
1025
1026 static int mdc_dma_remove(struct platform_device *pdev)
1027 {
1028         struct mdc_dma *mdma = platform_get_drvdata(pdev);
1029         struct mdc_chan *mchan, *next;
1030
1031         of_dma_controller_free(pdev->dev.of_node);
1032         dma_async_device_unregister(&mdma->dma_dev);
1033
1034         list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
1035                                  vc.chan.device_node) {
1036                 list_del(&mchan->vc.chan.device_node);
1037
1038                 devm_free_irq(&pdev->dev, mchan->irq, mchan);
1039
1040                 tasklet_kill(&mchan->vc.task);
1041         }
1042
1043         pm_runtime_disable(&pdev->dev);
1044         if (!pm_runtime_status_suspended(&pdev->dev))
1045                 img_mdc_runtime_suspend(&pdev->dev);
1046
1047         return 0;
1048 }
1049
1050 #ifdef CONFIG_PM_SLEEP
1051 static int img_mdc_suspend_late(struct device *dev)
1052 {
1053         struct mdc_dma *mdma = dev_get_drvdata(dev);
1054         int i;
1055
1056         /* Check that all channels are idle */
1057         for (i = 0; i < mdma->nr_channels; i++) {
1058                 struct mdc_chan *mchan = &mdma->channels[i];
1059
1060                 if (unlikely(mchan->desc))
1061                         return -EBUSY;
1062         }
1063
1064         return pm_runtime_force_suspend(dev);
1065 }
1066
1067 static int img_mdc_resume_early(struct device *dev)
1068 {
1069         return pm_runtime_force_resume(dev);
1070 }
1071 #endif /* CONFIG_PM_SLEEP */
1072
1073 static const struct dev_pm_ops img_mdc_pm_ops = {
1074         SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,
1075                            img_mdc_runtime_resume, NULL)
1076         SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,
1077                                      img_mdc_resume_early)
1078 };
1079
1080 static struct platform_driver mdc_dma_driver = {
1081         .driver = {
1082                 .name = "img-mdc-dma",
1083                 .pm = &img_mdc_pm_ops,
1084                 .of_match_table = of_match_ptr(mdc_dma_of_match),
1085         },
1086         .probe = mdc_dma_probe,
1087         .remove = mdc_dma_remove,
1088 };
1089 module_platform_driver(mdc_dma_driver);
1090
1091 MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
1092 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
1093 MODULE_LICENSE("GPL v2");