int mid_rid;
struct list_head ld_free;
- struct list_head ld_queue;
- struct list_head ld_active;
struct {
struct rz_lmdesc *base;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
u32 nxla;
u32 chctrl;
- u32 chstat;
dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
nxla = rz_dmac_lmdesc_addr(channel, channel->lmdesc.head);
- chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
- if (!(chstat & CHSTAT_EN)) {
- chctrl = (channel->chctrl | CHCTRL_SETEN);
- rz_dmac_ch_writel(channel, nxla, NXLA, 1);
- rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
- rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
- rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
- }
+ chctrl = (channel->chctrl | CHCTRL_SETEN);
+ rz_dmac_ch_writel(channel, nxla, NXLA, 1);
+ rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
+ rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
+ rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
}
static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
channel->chctrl = CHCTRL_SETEN;
}
-static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
+static void rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
{
- struct rz_dmac_desc *d = chan->desc;
struct virt_dma_desc *vd;
vd = vchan_next_desc(&chan->vc);
- if (!vd)
- return 0;
+ if (!vd) {
+ chan->desc = NULL;
+ return;
+ }
list_del(&vd->node);
+ chan->desc = to_rz_dmac_desc(vd);
- switch (d->type) {
+ switch (chan->desc->type) {
case RZ_DMAC_DESC_MEMCPY:
rz_dmac_prepare_desc_for_memcpy(chan);
break;
case RZ_DMAC_DESC_SLAVE_SG:
rz_dmac_prepare_descs_for_slave_sg(chan);
break;
-
- default:
- return -EINVAL;
}
rz_dmac_enable_hw(chan);
-
- return 0;
}
/*
rz_lmdesc_setup(channel, channel->lmdesc.base);
rz_dmac_disable_hw(channel);
- list_splice_tail_init(&channel->ld_active, &channel->ld_free);
- list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
if (channel->mid_rid >= 0) {
clear_bit(channel->mid_rid, dmac->modules);
spin_unlock_irqrestore(&channel->vc.lock, flags);
+ vchan_free_chan_resources(&channel->vc);
+
+ spin_lock_irqsave(&channel->vc.lock, flags);
+
list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
+ list_del(&desc->node);
kfree(desc);
channel->descs_allocated--;
}
INIT_LIST_HEAD(&channel->ld_free);
- vchan_free_chan_resources(&channel->vc);
+
+ spin_unlock_irqrestore(&channel->vc.lock, flags);
}
static struct dma_async_tx_descriptor *
if (!desc)
return NULL;
- desc->type = RZ_DMAC_DESC_MEMCPY;
- desc->src = src;
- desc->dest = dest;
- desc->len = len;
- desc->direction = DMA_MEM_TO_MEM;
-
- list_move_tail(channel->ld_free.next, &channel->ld_queue);
+ list_del(&desc->node);
}
+ desc->type = RZ_DMAC_DESC_MEMCPY;
+ desc->src = src;
+ desc->dest = dest;
+ desc->len = len;
+ desc->direction = DMA_MEM_TO_MEM;
+
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
}
if (!desc)
return NULL;
- for_each_sg(sgl, sg, sg_len, i)
- dma_length += sg_dma_len(sg);
+ list_del(&desc->node);
+ }
- desc->type = RZ_DMAC_DESC_SLAVE_SG;
- desc->sg = sgl;
- desc->sgcount = sg_len;
- desc->len = dma_length;
- desc->direction = direction;
+ for_each_sg(sgl, sg, sg_len, i)
+ dma_length += sg_dma_len(sg);
- if (direction == DMA_DEV_TO_MEM)
- desc->src = channel->src_per_address;
- else
- desc->dest = channel->dst_per_address;
+ desc->type = RZ_DMAC_DESC_SLAVE_SG;
+ desc->sg = sgl;
+ desc->sgcount = sg_len;
+ desc->len = dma_length;
+ desc->direction = direction;
- list_move_tail(channel->ld_free.next, &channel->ld_queue);
- }
+ if (direction == DMA_DEV_TO_MEM)
+ desc->src = channel->src_per_address;
+ else
+ desc->dest = channel->dst_per_address;
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
}
rz_dmac_disable_hw(channel);
rz_lmdesc_setup(channel, channel->lmdesc.base);
- list_splice_tail_init(&channel->ld_active, &channel->ld_free);
- list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
+ if (channel->desc) {
+ vchan_terminate_vdesc(&channel->desc->vd);
+ channel->desc = NULL;
+ }
+
vchan_get_all_descriptors(&channel->vc, &head);
spin_unlock_irqrestore(&channel->vc.lock, flags);
vchan_dma_desc_free_list(&channel->vc, &head);
static void rz_dmac_issue_pending(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
- struct rz_dmac *dmac = to_rz_dmac(chan->device);
- struct rz_dmac_desc *desc;
unsigned long flags;
spin_lock_irqsave(&channel->vc.lock, flags);
- if (!list_empty(&channel->ld_queue)) {
- desc = list_first_entry(&channel->ld_queue,
- struct rz_dmac_desc, node);
- channel->desc = desc;
- if (vchan_issue_pending(&channel->vc)) {
- if (rz_dmac_xfer_desc(channel) < 0)
- dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
- channel->index);
- else
- list_move_tail(channel->ld_queue.next,
- &channel->ld_active);
- }
- }
+ /*
+ * Issue the descriptor. If another transfer is already in progress, the
+ * issued descriptor will be handled after the current transfer finishes.
+ */
+ if (vchan_issue_pending(&channel->vc) && !channel->desc)
+ rz_dmac_xfer_desc(channel);
spin_unlock_irqrestore(&channel->vc.lock, flags);
}
static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
{
- /*
- * Place holder
- * Descriptor allocation is done during alloc_chan_resources and
- * get freed during free_chan_resources.
- * list is used to manage the descriptors and avoid any memory
- * allocation/free during DMA read/write.
- */
+ struct rz_dmac_chan *channel = to_rz_dmac_chan(vd->tx.chan);
+ struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan);
+ struct rz_dmac_desc *desc = to_rz_dmac_desc(vd);
+
+ guard(spinlock_irqsave)(&vc->lock);
+
+ list_add_tail(&desc->node, &channel->ld_free);
}
static void rz_dmac_device_synchronize(struct dma_chan *chan)
u32 chstat;
int ret;
+ vchan_synchronize(&channel->vc);
+
ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),
100, 100000, false, channel, CHSTAT, 1);
if (ret < 0)
static u32 rz_dmac_chan_get_residue(struct rz_dmac_chan *channel,
dma_cookie_t cookie)
{
- struct rz_dmac_desc *current_desc, *desc;
- enum dma_status status;
+ struct rz_dmac_desc *desc = NULL;
+ struct virt_dma_desc *vd;
u32 crla, crtb, i;
- /* Get current processing virtual descriptor */
- current_desc = list_first_entry_or_null(&channel->ld_active,
- struct rz_dmac_desc, node);
- if (!current_desc)
- return 0;
-
- /*
- * If the cookie corresponds to a descriptor that has been completed
- * there is no residue. The same check has already been performed by the
- * caller but without holding the channel lock, so the descriptor could
- * now be complete.
- */
- status = dma_cookie_status(&channel->vc.chan, cookie, NULL);
- if (status == DMA_COMPLETE)
- return 0;
-
- /*
- * If the cookie doesn't correspond to the currently processing virtual
- * descriptor then the descriptor hasn't been processed yet, and the
- * residue is equal to the full descriptor size. Also, a client driver
- * is possible to call this function before rz_dmac_irq_handler_thread()
- * runs. In this case, the running descriptor will be the next
- * descriptor, and will appear in the done list. So, if the argument
- * cookie matches the done list's cookie, we can assume the residue is
- * zero.
- */
- if (cookie != current_desc->vd.tx.cookie) {
- list_for_each_entry(desc, &channel->ld_free, node) {
- if (cookie == desc->vd.tx.cookie)
- return 0;
- }
-
- list_for_each_entry(desc, &channel->ld_queue, node) {
- if (cookie == desc->vd.tx.cookie)
- return desc->len;
- }
-
- list_for_each_entry(desc, &channel->ld_active, node) {
- if (cookie == desc->vd.tx.cookie)
- return desc->len;
- }
+ vd = vchan_find_desc(&channel->vc, cookie);
+ if (vd) {
+ /* Descriptor has been issued but not yet processed. */
+ desc = to_rz_dmac_desc(vd);
+ return desc->len;
+ } else if (channel->desc && channel->desc->vd.tx.cookie == cookie) {
+ /* Descriptor is currently processed. */
+ desc = channel->desc;
+ }
- /*
- * No descriptor found for the cookie, there's thus no residue.
- * This shouldn't happen if the calling driver passes a correct
- * cookie value.
- */
- WARN(1, "No descriptor for cookie!");
+ if (!desc) {
+ /* Descriptor was not found. May be already completed by now. */
return 0;
}
* Calculate number of bytes transferred in processing virtual descriptor.
* One virtual descriptor can have many lmdesc.
*/
- return crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, current_desc, crla);
+ return crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, desc, crla);
}
static enum dma_status rz_dmac_tx_status(struct dma_chan *chan,
enum dma_status status;
u32 residue;
- status = dma_cookie_status(chan, cookie, txstate);
- if (status == DMA_COMPLETE || !txstate)
- return status;
-
scoped_guard(spinlock_irqsave, &channel->vc.lock) {
+ status = dma_cookie_status(chan, cookie, txstate);
+ if (status == DMA_COMPLETE || !txstate)
+ return status;
+
residue = rz_dmac_chan_get_residue(channel, cookie);
- if (rz_dmac_chan_is_paused(channel))
+ if (status == DMA_IN_PROGRESS && rz_dmac_chan_is_paused(channel))
status = DMA_PAUSED;
}
- /* if there's no residue and no paused, the cookie is complete */
- if (!residue && status != DMA_PAUSED)
- return DMA_COMPLETE;
-
dma_set_residue(txstate, residue);
return status;
static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
{
struct rz_dmac_chan *channel = dev_id;
- struct rz_dmac_desc *desc = NULL;
- unsigned long flags;
+ struct rz_dmac_desc *desc;
- spin_lock_irqsave(&channel->vc.lock, flags);
+ guard(spinlock_irqsave)(&channel->vc.lock);
- if (list_empty(&channel->ld_active)) {
- /* Someone might have called terminate all */
- goto out;
- }
+ desc = channel->desc;
+ if (!desc)
+ return IRQ_HANDLED;
- desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
vchan_cookie_complete(&desc->vd);
- list_move_tail(channel->ld_active.next, &channel->ld_free);
- if (!list_empty(&channel->ld_queue)) {
- desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
- node);
- channel->desc = desc;
- if (rz_dmac_xfer_desc(channel) == 0)
- list_move_tail(channel->ld_queue.next, &channel->ld_active);
- }
-out:
- spin_unlock_irqrestore(&channel->vc.lock, flags);
+ channel->desc = NULL;
+
+ rz_dmac_xfer_desc(channel);
return IRQ_HANDLED;
}
channel->vc.desc_free = rz_dmac_virt_desc_free;
vchan_init(&channel->vc, &dmac->engine);
- INIT_LIST_HEAD(&channel->ld_queue);
INIT_LIST_HEAD(&channel->ld_free);
- INIT_LIST_HEAD(&channel->ld_active);
/* Initialize register for each channel */
rz_dmac_disable_hw(channel);
projects / thirdparty / kernel / linux.git / commitdiff
