[thirdparty/u-boot.git] / drivers / dma / apbh_dma.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * Freescale i.MX28 APBH DMA driver
 *
 * Copyright (C) 2011 Marek Vasut <marek.vasut@gmail.com>
 * on behalf of DENX Software Engineering GmbH
 *
 * Based on code from LTIB:
 * Copyright (C) 2010 Freescale Semiconductor, Inc. All Rights Reserved.
 * Copyright 2017 NXP
 *
 */

#include <cpu_func.h>
#include <asm/cache.h>
#include <linux/list.h>

#include <malloc.h>
#include <linux/errno.h>
#include <asm/io.h>
#include <asm/arch/clock.h>
#include <asm/arch/imx-regs.h>
#include <asm/arch/sys_proto.h>
#include <asm/mach-imx/dma.h>
#include <asm/mach-imx/regs-apbh.h>

static struct mxs_dma_chan mxs_dma_channels[MXS_MAX_DMA_CHANNELS];

/*
 * Test is the DMA channel is valid channel
 */
int mxs_dma_validate_chan(int channel)
{
	struct mxs_dma_chan *pchan;

	if ((channel < 0) || (channel >= MXS_MAX_DMA_CHANNELS))
		return -EINVAL;

	pchan = mxs_dma_channels + channel;
	if (!(pchan->flags & MXS_DMA_FLAGS_ALLOCATED))
		return -EINVAL;

	return 0;
}

/*
 * Return the address of the command within a descriptor.
 */
static unsigned int mxs_dma_cmd_address(struct mxs_dma_desc *desc)
{
	return desc->address + offsetof(struct mxs_dma_desc, cmd);
}

/*
 * Read a DMA channel's hardware semaphore.
 *
 * As used by the MXS platform's DMA software, the DMA channel's hardware
 * semaphore reflects the number of DMA commands the hardware will process, but
 * has not yet finished. This is a volatile value read directly from hardware,
 * so it must be be viewed as immediately stale.
 *
 * If the channel is not marked busy, or has finished processing all its
 * commands, this value should be zero.
 *
 * See mxs_dma_append() for details on how DMA command blocks must be configured
 * to maintain the expected behavior of the semaphore's value.
 */
static int mxs_dma_read_semaphore(int channel)
{
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;
	uint32_t tmp;
	int ret;

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	tmp = readl(&apbh_regs->ch[channel].hw_apbh_ch_sema);

	tmp &= APBH_CHn_SEMA_PHORE_MASK;
	tmp >>= APBH_CHn_SEMA_PHORE_OFFSET;

	return tmp;
}

#if !CONFIG_IS_ENABLED(SYS_DCACHE_OFF)
void mxs_dma_flush_desc(struct mxs_dma_desc *desc)
{
	uint32_t addr;
	uint32_t size;

	addr = (uintptr_t)desc;
	size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);

	flush_dcache_range(addr, addr + size);
}
#else
inline void mxs_dma_flush_desc(struct mxs_dma_desc *desc) {}
#endif

/*
 * Enable a DMA channel.
 *
 * If the given channel has any DMA descriptors on its active list, this
 * function causes the DMA hardware to begin processing them.
 *
 * This function marks the DMA channel as "busy," whether or not there are any
 * descriptors to process.
 */
static int mxs_dma_enable(int channel)
{
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;
	unsigned int sem;
	struct mxs_dma_chan *pchan;
	struct mxs_dma_desc *pdesc;
	int ret;

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	pchan = mxs_dma_channels + channel;

	if (pchan->pending_num == 0) {
		pchan->flags |= MXS_DMA_FLAGS_BUSY;
		return 0;
	}

	pdesc = list_first_entry(&pchan->active, struct mxs_dma_desc, node);
	if (pdesc == NULL)
		return -EFAULT;

	if (pchan->flags & MXS_DMA_FLAGS_BUSY) {
		if (!(pdesc->cmd.data & MXS_DMA_DESC_CHAIN))
			return 0;

		sem = mxs_dma_read_semaphore(channel);
		if (sem == 0)
			return 0;

		if (sem == 1) {
			pdesc = list_entry(pdesc->node.next,
					   struct mxs_dma_desc, node);
			writel(mxs_dma_cmd_address(pdesc),
				&apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar);
		}
		writel(pchan->pending_num,
			&apbh_regs->ch[channel].hw_apbh_ch_sema);
		pchan->active_num += pchan->pending_num;
		pchan->pending_num = 0;
	} else {
		pchan->active_num += pchan->pending_num;
		pchan->pending_num = 0;
		writel(mxs_dma_cmd_address(pdesc),
			&apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar);
		writel(pchan->active_num,
			&apbh_regs->ch[channel].hw_apbh_ch_sema);
		writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
			&apbh_regs->hw_apbh_ctrl0_clr);
	}

	pchan->flags |= MXS_DMA_FLAGS_BUSY;
	return 0;
}

/*
 * Disable a DMA channel.
 *
 * This function shuts down a DMA channel and marks it as "not busy." Any
 * descriptors on the active list are immediately moved to the head of the
 * "done" list, whether or not they have actually been processed by the
 * hardware. The "ready" flags of these descriptors are NOT cleared, so they
 * still appear to be active.
 *
 * This function immediately shuts down a DMA channel's hardware, aborting any
 * I/O that may be in progress, potentially leaving I/O hardware in an undefined
 * state. It is unwise to call this function if there is ANY chance the hardware
 * is still processing a command.
 */
static int mxs_dma_disable(int channel)
{
	struct mxs_dma_chan *pchan;
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;
	int ret;

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	pchan = mxs_dma_channels + channel;

	if (!(pchan->flags & MXS_DMA_FLAGS_BUSY))
		return -EINVAL;

	writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
		&apbh_regs->hw_apbh_ctrl0_set);

	pchan->flags &= ~MXS_DMA_FLAGS_BUSY;
	pchan->active_num = 0;
	pchan->pending_num = 0;
	list_splice_init(&pchan->active, &pchan->done);

	return 0;
}

/*
 * Resets the DMA channel hardware.
 */
static int mxs_dma_reset(int channel)
{
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;
	int ret;
#if defined(CONFIG_MX23)
	uint32_t setreg = (uint32_t)(&apbh_regs->hw_apbh_ctrl0_set);
	uint32_t offset = APBH_CTRL0_RESET_CHANNEL_OFFSET;
#elif defined(CONFIG_MX28) || defined(CONFIG_MX6) || defined(CONFIG_MX7) || \
	defined(CONFIG_IMX8) || defined(CONFIG_IMX8M)
	u32 setreg = (uintptr_t)(&apbh_regs->hw_apbh_channel_ctrl_set);
	u32 offset = APBH_CHANNEL_CTRL_RESET_CHANNEL_OFFSET;
#endif

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	writel(1 << (channel + offset), (uintptr_t)setreg);

	return 0;
}

/*
 * Enable or disable DMA interrupt.
 *
 * This function enables the given DMA channel to interrupt the CPU.
 */
static int mxs_dma_enable_irq(int channel, int enable)
{
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;
	int ret;

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	if (enable)
		writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET),
			&apbh_regs->hw_apbh_ctrl1_set);
	else
		writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET),
			&apbh_regs->hw_apbh_ctrl1_clr);

	return 0;
}

/*
 * Clear DMA interrupt.
 *
 * The software that is using the DMA channel must register to receive its
 * interrupts and, when they arrive, must call this function to clear them.
 */
static int mxs_dma_ack_irq(int channel)
{
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;
	int ret;

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	writel(1 << channel, &apbh_regs->hw_apbh_ctrl1_clr);
	writel(1 << channel, &apbh_regs->hw_apbh_ctrl2_clr);

	return 0;
}

/*
 * Request to reserve a DMA channel
 */
static int mxs_dma_request(int channel)
{
	struct mxs_dma_chan *pchan;

	if ((channel < 0) || (channel >= MXS_MAX_DMA_CHANNELS))
		return -EINVAL;

	pchan = mxs_dma_channels + channel;
	if ((pchan->flags & MXS_DMA_FLAGS_VALID) != MXS_DMA_FLAGS_VALID)
		return -ENODEV;

	if (pchan->flags & MXS_DMA_FLAGS_ALLOCATED)
		return -EBUSY;

	pchan->flags |= MXS_DMA_FLAGS_ALLOCATED;
	pchan->active_num = 0;
	pchan->pending_num = 0;

	INIT_LIST_HEAD(&pchan->active);
	INIT_LIST_HEAD(&pchan->done);

	return 0;
}

/*
 * Release a DMA channel.
 *
 * This function releases a DMA channel from its current owner.
 *
 * The channel will NOT be released if it's marked "busy" (see
 * mxs_dma_enable()).
 */
int mxs_dma_release(int channel)
{
	struct mxs_dma_chan *pchan;
	int ret;

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	pchan = mxs_dma_channels + channel;

	if (pchan->flags & MXS_DMA_FLAGS_BUSY)
		return -EBUSY;

	pchan->dev = 0;
	pchan->active_num = 0;
	pchan->pending_num = 0;
	pchan->flags &= ~MXS_DMA_FLAGS_ALLOCATED;

	return 0;
}

/*
 * Allocate DMA descriptor
 */
struct mxs_dma_desc *mxs_dma_desc_alloc(void)
{
	struct mxs_dma_desc *pdesc;
	uint32_t size;

	size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
	pdesc = memalign(MXS_DMA_ALIGNMENT, size);

	if (pdesc == NULL)
		return NULL;

	memset(pdesc, 0, sizeof(*pdesc));
	pdesc->address = (dma_addr_t)pdesc;

	return pdesc;
};

/*
 * Free DMA descriptor
 */
void mxs_dma_desc_free(struct mxs_dma_desc *pdesc)
{
	if (pdesc == NULL)
		return;

	free(pdesc);
}

/*
 * Add a DMA descriptor to a channel.
 *
 * If the descriptor list for this channel is not empty, this function sets the
 * CHAIN bit and the NEXTCMD_ADDR fields in the last descriptor's DMA command so
 * it will chain to the new descriptor's command.
 *
 * Then, this function marks the new descriptor as "ready," adds it to the end
 * of the active descriptor list, and increments the count of pending
 * descriptors.
 *
 * The MXS platform DMA software imposes some rules on DMA commands to maintain
 * important invariants. These rules are NOT checked, but they must be carefully
 * applied by software that uses MXS DMA channels.
 *
 * Invariant:
 *     The DMA channel's hardware semaphore must reflect the number of DMA
 *     commands the hardware will process, but has not yet finished.
 *
 * Explanation:
 *     A DMA channel begins processing commands when its hardware semaphore is
 *     written with a value greater than zero, and it stops processing commands
 *     when the semaphore returns to zero.
 *
 *     When a channel finishes a DMA command, it will decrement its semaphore if
 *     the DECREMENT_SEMAPHORE bit is set in that command's flags bits.
 *
 *     In principle, it's not necessary for the DECREMENT_SEMAPHORE to be set,
 *     unless it suits the purposes of the software. For example, one could
 *     construct a series of five DMA commands, with the DECREMENT_SEMAPHORE
 *     bit set only in the last one. Then, setting the DMA channel's hardware
 *     semaphore to one would cause the entire series of five commands to be
 *     processed. However, this example would violate the invariant given above.
 *
 * Rule:
 *    ALL DMA commands MUST have the DECREMENT_SEMAPHORE bit set so that the DMA
 *    channel's hardware semaphore will be decremented EVERY time a command is
 *    processed.
 */
int mxs_dma_desc_append(int channel, struct mxs_dma_desc *pdesc)
{
	struct mxs_dma_chan *pchan;
	struct mxs_dma_desc *last;
	int ret;

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	pchan = mxs_dma_channels + channel;

	pdesc->cmd.next = mxs_dma_cmd_address(pdesc);
	pdesc->flags |= MXS_DMA_DESC_FIRST | MXS_DMA_DESC_LAST;

	if (!list_empty(&pchan->active)) {
		last = list_entry(pchan->active.prev, struct mxs_dma_desc,
					node);

		pdesc->flags &= ~MXS_DMA_DESC_FIRST;
		last->flags &= ~MXS_DMA_DESC_LAST;

		last->cmd.next = mxs_dma_cmd_address(pdesc);
		last->cmd.data |= MXS_DMA_DESC_CHAIN;

		mxs_dma_flush_desc(last);
	}
	pdesc->flags |= MXS_DMA_DESC_READY;
	if (pdesc->flags & MXS_DMA_DESC_FIRST)
		pchan->pending_num++;
	list_add_tail(&pdesc->node, &pchan->active);

	mxs_dma_flush_desc(pdesc);

	return ret;
}

/*
 * Clean up processed DMA descriptors.
 *
 * This function removes processed DMA descriptors from the "active" list. Pass
 * in a non-NULL list head to get the descriptors moved to your list. Pass NULL
 * to get the descriptors moved to the channel's "done" list. Descriptors on
 * the "done" list can be retrieved with mxs_dma_get_finished().
 *
 * This function marks the DMA channel as "not busy" if no unprocessed
 * descriptors remain on the "active" list.
 */
static int mxs_dma_finish(int channel, struct list_head *head)
{
	int sem;
	struct mxs_dma_chan *pchan;
	struct list_head *p, *q;
	struct mxs_dma_desc *pdesc;
	int ret;

	ret = mxs_dma_validate_chan(channel);
	if (ret)
		return ret;

	pchan = mxs_dma_channels + channel;

	sem = mxs_dma_read_semaphore(channel);
	if (sem < 0)
		return sem;

	if (sem == pchan->active_num)
		return 0;

	list_for_each_safe(p, q, &pchan->active) {
		if ((pchan->active_num) <= sem)
			break;

		pdesc = list_entry(p, struct mxs_dma_desc, node);
		pdesc->flags &= ~MXS_DMA_DESC_READY;

		if (head)
			list_move_tail(p, head);
		else
			list_move_tail(p, &pchan->done);

		if (pdesc->flags & MXS_DMA_DESC_LAST)
			pchan->active_num--;
	}

	if (sem == 0)
		pchan->flags &= ~MXS_DMA_FLAGS_BUSY;

	return 0;
}

/*
 * Wait for DMA channel to complete
 */
static int mxs_dma_wait_complete(uint32_t timeout, unsigned int chan)
{
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;
	int ret;

	ret = mxs_dma_validate_chan(chan);
	if (ret)
		return ret;

	if (mxs_wait_mask_set(&apbh_regs->hw_apbh_ctrl1_reg,
				1 << chan, timeout)) {
		ret = -ETIMEDOUT;
		mxs_dma_reset(chan);
	}

	return ret;
}

/*
 * Execute the DMA channel
 */
int mxs_dma_go(int chan)
{
	uint32_t timeout = 10000000;
	int ret;

	LIST_HEAD(tmp_desc_list);

	mxs_dma_enable_irq(chan, 1);
	mxs_dma_enable(chan);

	/* Wait for DMA to finish. */
	ret = mxs_dma_wait_complete(timeout, chan);

	/* Clear out the descriptors we just ran. */
	mxs_dma_finish(chan, &tmp_desc_list);

	/* Shut the DMA channel down. */
	mxs_dma_ack_irq(chan);
	mxs_dma_reset(chan);
	mxs_dma_enable_irq(chan, 0);
	mxs_dma_disable(chan);

	return ret;
}

/*
 * Execute a continuously running circular DMA descriptor.
 * NOTE: This is not intended for general use, but rather
 *	 for the LCD driver in Smart-LCD mode. It allows
 *	 continuous triggering of the RUN bit there.
 */
void mxs_dma_circ_start(int chan, struct mxs_dma_desc *pdesc)
{
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;

	mxs_dma_flush_desc(pdesc);

	mxs_dma_enable_irq(chan, 1);

	writel(mxs_dma_cmd_address(pdesc),
		&apbh_regs->ch[chan].hw_apbh_ch_nxtcmdar);
	writel(1, &apbh_regs->ch[chan].hw_apbh_ch_sema);
	writel(1 << (chan + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
		&apbh_regs->hw_apbh_ctrl0_clr);
}

/*
 * Initialize the DMA hardware
 */
void mxs_dma_init(void)
{
	struct mxs_apbh_regs *apbh_regs =
		(struct mxs_apbh_regs *)MXS_APBH_BASE;

	mxs_reset_block(&apbh_regs->hw_apbh_ctrl0_reg);

#ifdef CONFIG_APBH_DMA_BURST8
	writel(APBH_CTRL0_AHB_BURST8_EN,
		&apbh_regs->hw_apbh_ctrl0_set);
#else
	writel(APBH_CTRL0_AHB_BURST8_EN,
		&apbh_regs->hw_apbh_ctrl0_clr);
#endif

#ifdef CONFIG_APBH_DMA_BURST
	writel(APBH_CTRL0_APB_BURST_EN,
		&apbh_regs->hw_apbh_ctrl0_set);
#else
	writel(APBH_CTRL0_APB_BURST_EN,
		&apbh_regs->hw_apbh_ctrl0_clr);
#endif
}

int mxs_dma_init_channel(int channel)
{
	struct mxs_dma_chan *pchan;
	int ret;

	pchan = mxs_dma_channels + channel;
	pchan->flags = MXS_DMA_FLAGS_VALID;

	ret = mxs_dma_request(channel);

	if (ret) {
		printf("MXS DMA: Can't acquire DMA channel %i\n",
			channel);
		return ret;
	}

	mxs_dma_reset(channel);
	mxs_dma_ack_irq(channel);

	return 0;
}
Commit	Line	Data
83d290c5	1	// SPDX-License-Identifier: GPL-2.0+
31650d64 MV	2	/*
	3	* Freescale i.MX28 APBH DMA driver
	4	*
	5	* Copyright (C) 2011 Marek Vasut <marek.vasut@gmail.com>
	6	* on behalf of DENX Software Engineering GmbH
	7	*
	8	* Based on code from LTIB:
	9	* Copyright (C) 2010 Freescale Semiconductor, Inc. All Rights Reserved.
39320e72 PF	10	* Copyright 2017 NXP
39320e72 PF	11	*
31650d64 MV	12	*/
31650d64 MV	13
1eb69ae4	14	#include <cpu_func.h>
90526e9f	15	#include <asm/cache.h>
31650d64 MV	16	#include <linux/list.h>
31650d64 MV	17
31650d64	18	#include <malloc.h>
1221ce45	19	#include <linux/errno.h>
31650d64 MV	20	#include <asm/io.h>
	21	#include <asm/arch/clock.h>
	22	#include <asm/arch/imx-regs.h>
	23	#include <asm/arch/sys_proto.h>
552a848e SB	24	#include <asm/mach-imx/dma.h>
552a848e SB	25	#include <asm/mach-imx/regs-apbh.h>
31650d64 MV	26
	27	static struct mxs_dma_chan mxs_dma_channels[MXS_MAX_DMA_CHANNELS];
	28
	29	/*
	30	* Test is the DMA channel is valid channel
	31	*/
	32	int mxs_dma_validate_chan(int channel)
	33	{
	34	struct mxs_dma_chan *pchan;
	35
	36	if ((channel < 0) \|\| (channel >= MXS_MAX_DMA_CHANNELS))
	37	return -EINVAL;
	38
	39	pchan = mxs_dma_channels + channel;
	40	if (!(pchan->flags & MXS_DMA_FLAGS_ALLOCATED))
	41	return -EINVAL;
	42
	43	return 0;
	44	}
	45
aa72e43b MV	46	/*
	47	* Return the address of the command within a descriptor.
	48	*/
	49	static unsigned int mxs_dma_cmd_address(struct mxs_dma_desc *desc)
	50	{
	51	return desc->address + offsetof(struct mxs_dma_desc, cmd);
	52	}
	53
	54	/*
	55	* Read a DMA channel's hardware semaphore.
	56	*
	57	* As used by the MXS platform's DMA software, the DMA channel's hardware
	58	* semaphore reflects the number of DMA commands the hardware will process, but
	59	* has not yet finished. This is a volatile value read directly from hardware,
	60	* so it must be be viewed as immediately stale.
	61	*
	62	* If the channel is not marked busy, or has finished processing all its
	63	* commands, this value should be zero.
	64	*
	65	* See mxs_dma_append() for details on how DMA command blocks must be configured
	66	* to maintain the expected behavior of the semaphore's value.
	67	*/
	68	static int mxs_dma_read_semaphore(int channel)
	69	{
9c471142 OS	70	struct mxs_apbh_regs *apbh_regs =
9c471142 OS	71	(struct mxs_apbh_regs *)MXS_APBH_BASE;
aa72e43b MV	72	uint32_t tmp;
	73	int ret;
	74
	75	ret = mxs_dma_validate_chan(channel);
	76	if (ret)
	77	return ret;
	78
	79	tmp = readl(&apbh_regs->ch[channel].hw_apbh_ch_sema);
	80
	81	tmp &= APBH_CHn_SEMA_PHORE_MASK;
	82	tmp >>= APBH_CHn_SEMA_PHORE_OFFSET;
	83
	84	return tmp;
	85	}
	86
10015025	87	#if !CONFIG_IS_ENABLED(SYS_DCACHE_OFF)
c3dfe707 MV	88	void mxs_dma_flush_desc(struct mxs_dma_desc *desc)
	89	{
	90	uint32_t addr;
	91	uint32_t size;
	92
39320e72	93	addr = (uintptr_t)desc;
c3dfe707 MV	94	size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
	95
	96	flush_dcache_range(addr, addr + size);
	97	}
	98	#else
	99	inline void mxs_dma_flush_desc(struct mxs_dma_desc *desc) {}
	100	#endif
	101
31650d64 MV	102	/*
	103	* Enable a DMA channel.
	104	*
	105	* If the given channel has any DMA descriptors on its active list, this
	106	* function causes the DMA hardware to begin processing them.
	107	*
	108	* This function marks the DMA channel as "busy," whether or not there are any
	109	* descriptors to process.
	110	*/
aa72e43b	111	static int mxs_dma_enable(int channel)
31650d64	112	{
9c471142 OS	113	struct mxs_apbh_regs *apbh_regs =
9c471142 OS	114	(struct mxs_apbh_regs *)MXS_APBH_BASE;
31650d64 MV	115	unsigned int sem;
	116	struct mxs_dma_chan *pchan;
	117	struct mxs_dma_desc *pdesc;
	118	int ret;
	119
	120	ret = mxs_dma_validate_chan(channel);
	121	if (ret)
	122	return ret;
	123
	124	pchan = mxs_dma_channels + channel;
	125
	126	if (pchan->pending_num == 0) {
	127	pchan->flags \|= MXS_DMA_FLAGS_BUSY;
	128	return 0;
	129	}
	130
	131	pdesc = list_first_entry(&pchan->active, struct mxs_dma_desc, node);
	132	if (pdesc == NULL)
	133	return -EFAULT;
	134
	135	if (pchan->flags & MXS_DMA_FLAGS_BUSY) {
	136	if (!(pdesc->cmd.data & MXS_DMA_DESC_CHAIN))
	137	return 0;
	138
	139	sem = mxs_dma_read_semaphore(channel);
	140	if (sem == 0)
	141	return 0;
	142
	143	if (sem == 1) {
	144	pdesc = list_entry(pdesc->node.next,
	145	struct mxs_dma_desc, node);
	146	writel(mxs_dma_cmd_address(pdesc),
	147	&apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar);
	148	}
	149	writel(pchan->pending_num,
	150	&apbh_regs->ch[channel].hw_apbh_ch_sema);
	151	pchan->active_num += pchan->pending_num;
	152	pchan->pending_num = 0;
	153	} else {
	154	pchan->active_num += pchan->pending_num;
	155	pchan->pending_num = 0;
	156	writel(mxs_dma_cmd_address(pdesc),
	157	&apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar);
	158	writel(pchan->active_num,
	159	&apbh_regs->ch[channel].hw_apbh_ch_sema);
	160	writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
	161	&apbh_regs->hw_apbh_ctrl0_clr);
	162	}
	163
	164	pchan->flags \|= MXS_DMA_FLAGS_BUSY;
	165	return 0;
	166	}
	167
	168	/*
	169	* Disable a DMA channel.
	170	*
	171	* This function shuts down a DMA channel and marks it as "not busy." Any
	172	* descriptors on the active list are immediately moved to the head of the
	173	* "done" list, whether or not they have actually been processed by the
	174	* hardware. The "ready" flags of these descriptors are NOT cleared, so they
	175	* still appear to be active.
	176	*
	177	* This function immediately shuts down a DMA channel's hardware, aborting any
	178	* I/O that may be in progress, potentially leaving I/O hardware in an undefined
179	* state. It is unwise to call this function if there is ANY chance the hardware
180	* is still processing a command.
181	*/
aa72e43b	182	static int mxs_dma_disable(int channel)
31650d64 MV	183	{
31650d64 MV	184	struct mxs_dma_chan *pchan;
9c471142 OS	185	struct mxs_apbh_regs *apbh_regs =
9c471142 OS	186	(struct mxs_apbh_regs *)MXS_APBH_BASE;
31650d64 MV	187	int ret;
	188
	189	ret = mxs_dma_validate_chan(channel);
	190	if (ret)
	191	return ret;
	192
	193	pchan = mxs_dma_channels + channel;
	194
	195	if (!(pchan->flags & MXS_DMA_FLAGS_BUSY))
	196	return -EINVAL;
	197
	198	writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
	199	&apbh_regs->hw_apbh_ctrl0_set);
	200
	201	pchan->flags &= ~MXS_DMA_FLAGS_BUSY;
	202	pchan->active_num = 0;
	203	pchan->pending_num = 0;
	204	list_splice_init(&pchan->active, &pchan->done);
	205
	206	return 0;
	207	}
	208
	209	/*
	210	* Resets the DMA channel hardware.
	211	*/
aa72e43b	212	static int mxs_dma_reset(int channel)
31650d64	213	{
9c471142 OS	214	struct mxs_apbh_regs *apbh_regs =
9c471142 OS	215	(struct mxs_apbh_regs *)MXS_APBH_BASE;
31650d64	216	int ret;
0e5c05ef MV	217	#if defined(CONFIG_MX23)
	218	uint32_t setreg = (uint32_t)(&apbh_regs->hw_apbh_ctrl0_set);
	219	uint32_t offset = APBH_CTRL0_RESET_CHANNEL_OFFSET;
39320e72 PF	220	#elif defined(CONFIG_MX28) \|\| defined(CONFIG_MX6) \|\| defined(CONFIG_MX7) \|\| \
	221	defined(CONFIG_IMX8) \|\| defined(CONFIG_IMX8M)
	222	u32 setreg = (uintptr_t)(&apbh_regs->hw_apbh_channel_ctrl_set);
	223	u32 offset = APBH_CHANNEL_CTRL_RESET_CHANNEL_OFFSET;
0e5c05ef	224	#endif
31650d64 MV	225
	226	ret = mxs_dma_validate_chan(channel);
	227	if (ret)
	228	return ret;
	229
39320e72	230	writel(1 << (channel + offset), (uintptr_t)setreg);
31650d64 MV	231
	232	return 0;
	233	}
	234
31650d64 MV	235	/*
	236	* Enable or disable DMA interrupt.
	237	*
	238	* This function enables the given DMA channel to interrupt the CPU.
	239	*/
aa72e43b	240	static int mxs_dma_enable_irq(int channel, int enable)
31650d64	241	{
9c471142 OS	242	struct mxs_apbh_regs *apbh_regs =
9c471142 OS	243	(struct mxs_apbh_regs *)MXS_APBH_BASE;
31650d64 MV	244	int ret;
	245
	246	ret = mxs_dma_validate_chan(channel);
	247	if (ret)
	248	return ret;
	249
	250	if (enable)
	251	writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET),
	252	&apbh_regs->hw_apbh_ctrl1_set);
	253	else
	254	writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET),
	255	&apbh_regs->hw_apbh_ctrl1_clr);
	256
	257	return 0;
	258	}
	259
31650d64 MV	260	/*
	261	* Clear DMA interrupt.
	262	*
	263	* The software that is using the DMA channel must register to receive its
	264	* interrupts and, when they arrive, must call this function to clear them.
	265	*/
aa72e43b	266	static int mxs_dma_ack_irq(int channel)
31650d64	267	{
9c471142 OS	268	struct mxs_apbh_regs *apbh_regs =
9c471142 OS	269	(struct mxs_apbh_regs *)MXS_APBH_BASE;
31650d64 MV	270	int ret;
	271
	272	ret = mxs_dma_validate_chan(channel);
	273	if (ret)
	274	return ret;
	275
	276	writel(1 << channel, &apbh_regs->hw_apbh_ctrl1_clr);
	277	writel(1 << channel, &apbh_regs->hw_apbh_ctrl2_clr);
	278
	279	return 0;
	280	}
	281
	282	/*
	283	* Request to reserve a DMA channel
	284	*/
aa72e43b	285	static int mxs_dma_request(int channel)
31650d64 MV	286	{
	287	struct mxs_dma_chan *pchan;
	288
	289	if ((channel < 0) \|\| (channel >= MXS_MAX_DMA_CHANNELS))
	290	return -EINVAL;
	291
	292	pchan = mxs_dma_channels + channel;
	293	if ((pchan->flags & MXS_DMA_FLAGS_VALID) != MXS_DMA_FLAGS_VALID)
	294	return -ENODEV;
	295
	296	if (pchan->flags & MXS_DMA_FLAGS_ALLOCATED)
	297	return -EBUSY;
	298
	299	pchan->flags \|= MXS_DMA_FLAGS_ALLOCATED;
	300	pchan->active_num = 0;
	301	pchan->pending_num = 0;
	302
	303	INIT_LIST_HEAD(&pchan->active);
	304	INIT_LIST_HEAD(&pchan->done);
	305
	306	return 0;
	307	}
	308
	309	/*
	310	* Release a DMA channel.
	311	*
	312	* This function releases a DMA channel from its current owner.
	313	*
	314	* The channel will NOT be released if it's marked "busy" (see
	315	* mxs_dma_enable()).
	316	*/
96666a39	317	int mxs_dma_release(int channel)
31650d64 MV	318	{
	319	struct mxs_dma_chan *pchan;
	320	int ret;
	321
	322	ret = mxs_dma_validate_chan(channel);
	323	if (ret)
	324	return ret;
	325
	326	pchan = mxs_dma_channels + channel;
	327
	328	if (pchan->flags & MXS_DMA_FLAGS_BUSY)
	329	return -EBUSY;
	330
	331	pchan->dev = 0;
	332	pchan->active_num = 0;
	333	pchan->pending_num = 0;
	334	pchan->flags &= ~MXS_DMA_FLAGS_ALLOCATED;
	335
	336	return 0;
	337	}
	338
	339	/*
	340	* Allocate DMA descriptor
	341	*/
	342	struct mxs_dma_desc *mxs_dma_desc_alloc(void)
	343	{
	344	struct mxs_dma_desc *pdesc;
c3dfe707	345	uint32_t size;
31650d64	346
c3dfe707 MV	347	size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
c3dfe707 MV	348	pdesc = memalign(MXS_DMA_ALIGNMENT, size);
31650d64 MV	349
	350	if (pdesc == NULL)
	351	return NULL;
	352
	353	memset(pdesc, 0, sizeof(*pdesc));
	354	pdesc->address = (dma_addr_t)pdesc;
	355
	356	return pdesc;
	357	};
	358
	359	/*
	360	* Free DMA descriptor
	361	*/
	362	void mxs_dma_desc_free(struct mxs_dma_desc *pdesc)
	363	{
	364	if (pdesc == NULL)
	365	return;
	366
	367	free(pdesc);
	368	}
	369
31650d64 MV	370	/*
	371	* Add a DMA descriptor to a channel.
	372	*
	373	* If the descriptor list for this channel is not empty, this function sets the
	374	* CHAIN bit and the NEXTCMD_ADDR fields in the last descriptor's DMA command so
	375	* it will chain to the new descriptor's command.
	376	*
	377	* Then, this function marks the new descriptor as "ready," adds it to the end
	378	* of the active descriptor list, and increments the count of pending
	379	* descriptors.
	380	*
	381	* The MXS platform DMA software imposes some rules on DMA commands to maintain
	382	* important invariants. These rules are NOT checked, but they must be carefully
	383	* applied by software that uses MXS DMA channels.
	384	*
	385	* Invariant:
	386	* The DMA channel's hardware semaphore must reflect the number of DMA
	387	* commands the hardware will process, but has not yet finished.
	388	*
	389	* Explanation:
	390	* A DMA channel begins processing commands when its hardware semaphore is
	391	* written with a value greater than zero, and it stops processing commands
	392	* when the semaphore returns to zero.
	393	*
	394	* When a channel finishes a DMA command, it will decrement its semaphore if
	395	* the DECREMENT_SEMAPHORE bit is set in that command's flags bits.
	396	*
	397	* In principle, it's not necessary for the DECREMENT_SEMAPHORE to be set,
	398	* unless it suits the purposes of the software. For example, one could
	399	* construct a series of five DMA commands, with the DECREMENT_SEMAPHORE
	400	* bit set only in the last one. Then, setting the DMA channel's hardware
	401	* semaphore to one would cause the entire series of five commands to be
	402	* processed. However, this example would violate the invariant given above.
	403	*
	404	* Rule:
	405	* ALL DMA commands MUST have the DECREMENT_SEMAPHORE bit set so that the DMA
	406	* channel's hardware semaphore will be decremented EVERY time a command is
	407	* processed.
	408	*/
	409	int mxs_dma_desc_append(int channel, struct mxs_dma_desc *pdesc)
	410	{
	411	struct mxs_dma_chan *pchan;
	412	struct mxs_dma_desc *last;
	413	int ret;
	414
	415	ret = mxs_dma_validate_chan(channel);
	416	if (ret)
	417	return ret;
	418
	419	pchan = mxs_dma_channels + channel;
	420
	421	pdesc->cmd.next = mxs_dma_cmd_address(pdesc);
	422	pdesc->flags \|= MXS_DMA_DESC_FIRST \| MXS_DMA_DESC_LAST;
	423
	424	if (!list_empty(&pchan->active)) {
	425	last = list_entry(pchan->active.prev, struct mxs_dma_desc,
	426	node);
	427
	428	pdesc->flags &= ~MXS_DMA_DESC_FIRST;
	429	last->flags &= ~MXS_DMA_DESC_LAST;
	430
	431	last->cmd.next = mxs_dma_cmd_address(pdesc);
	432	last->cmd.data \|= MXS_DMA_DESC_CHAIN;
c3dfe707 MV	433
c3dfe707 MV	434	mxs_dma_flush_desc(last);
31650d64 MV	435	}
	436	pdesc->flags \|= MXS_DMA_DESC_READY;
	437	if (pdesc->flags & MXS_DMA_DESC_FIRST)
	438	pchan->pending_num++;
	439	list_add_tail(&pdesc->node, &pchan->active);
	440
c3dfe707 MV	441	mxs_dma_flush_desc(pdesc);
c3dfe707 MV	442
31650d64 MV	443	return ret;
	444	}
	445
31650d64 MV	446	/*
	447	* Clean up processed DMA descriptors.
	448	*
	449	* This function removes processed DMA descriptors from the "active" list. Pass
	450	* in a non-NULL list head to get the descriptors moved to your list. Pass NULL
	451	* to get the descriptors moved to the channel's "done" list. Descriptors on
	452	* the "done" list can be retrieved with mxs_dma_get_finished().
	453	*
	454	* This function marks the DMA channel as "not busy" if no unprocessed
	455	* descriptors remain on the "active" list.
	456	*/
aa72e43b	457	static int mxs_dma_finish(int channel, struct list_head *head)
31650d64 MV	458	{
	459	int sem;
	460	struct mxs_dma_chan *pchan;
	461	struct list_head p, q;
	462	struct mxs_dma_desc *pdesc;
	463	int ret;
	464
	465	ret = mxs_dma_validate_chan(channel);
	466	if (ret)
	467	return ret;
	468
	469	pchan = mxs_dma_channels + channel;
	470
	471	sem = mxs_dma_read_semaphore(channel);
	472	if (sem < 0)
	473	return sem;
	474
	475	if (sem == pchan->active_num)
	476	return 0;
	477
	478	list_for_each_safe(p, q, &pchan->active) {
	479	if ((pchan->active_num) <= sem)
	480	break;
	481
	482	pdesc = list_entry(p, struct mxs_dma_desc, node);
	483	pdesc->flags &= ~MXS_DMA_DESC_READY;
	484
	485	if (head)
	486	list_move_tail(p, head);
	487	else
	488	list_move_tail(p, &pchan->done);
	489
	490	if (pdesc->flags & MXS_DMA_DESC_LAST)
	491	pchan->active_num--;
	492	}
	493
	494	if (sem == 0)
	495	pchan->flags &= ~MXS_DMA_FLAGS_BUSY;
	496
	497	return 0;
	498	}
	499
	500	/*
	501	* Wait for DMA channel to complete
	502	*/
aa72e43b	503	static int mxs_dma_wait_complete(uint32_t timeout, unsigned int chan)
31650d64	504	{
9c471142 OS	505	struct mxs_apbh_regs *apbh_regs =
9c471142 OS	506	(struct mxs_apbh_regs *)MXS_APBH_BASE;
31650d64 MV	507	int ret;
	508
	509	ret = mxs_dma_validate_chan(chan);
	510	if (ret)
	511	return ret;
	512
fa7a51cb	513	if (mxs_wait_mask_set(&apbh_regs->hw_apbh_ctrl1_reg,
31650d64 MV	514	1 << chan, timeout)) {
	515	ret = -ETIMEDOUT;
	516	mxs_dma_reset(chan);
	517	}
	518
12dab4ce	519	return ret;
31650d64 MV	520	}
	521
	522	/*
	523	* Execute the DMA channel
	524	*/
	525	int mxs_dma_go(int chan)
	526	{
1375f044	527	uint32_t timeout = 10000000;
31650d64 MV	528	int ret;
	529
	530	LIST_HEAD(tmp_desc_list);
	531
	532	mxs_dma_enable_irq(chan, 1);
	533	mxs_dma_enable(chan);
	534
	535	/* Wait for DMA to finish. */
	536	ret = mxs_dma_wait_complete(timeout, chan);
	537
	538	/* Clear out the descriptors we just ran. */
	539	mxs_dma_finish(chan, &tmp_desc_list);
	540
	541	/* Shut the DMA channel down. */
	542	mxs_dma_ack_irq(chan);
	543	mxs_dma_reset(chan);
	544	mxs_dma_enable_irq(chan, 0);
	545	mxs_dma_disable(chan);
	546
	547	return ret;
	548	}
	549
69f7345c MV	550	/*
	551	* Execute a continuously running circular DMA descriptor.
	552	* NOTE: This is not intended for general use, but rather
	553	* for the LCD driver in Smart-LCD mode. It allows
	554	* continuous triggering of the RUN bit there.
	555	*/
	556	void mxs_dma_circ_start(int chan, struct mxs_dma_desc *pdesc)
	557	{
	558	struct mxs_apbh_regs *apbh_regs =
	559	(struct mxs_apbh_regs *)MXS_APBH_BASE;
	560
	561	mxs_dma_flush_desc(pdesc);
	562
	563	mxs_dma_enable_irq(chan, 1);
	564
	565	writel(mxs_dma_cmd_address(pdesc),
	566	&apbh_regs->ch[chan].hw_apbh_ch_nxtcmdar);
	567	writel(1, &apbh_regs->ch[chan].hw_apbh_ch_sema);
	568	writel(1 << (chan + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
	569	&apbh_regs->hw_apbh_ctrl0_clr);
	570	}
	571
31650d64 MV	572	/*
	573	* Initialize the DMA hardware
	574	*/
96666a39	575	void mxs_dma_init(void)
31650d64	576	{
9c471142 OS	577	struct mxs_apbh_regs *apbh_regs =
9c471142 OS	578	(struct mxs_apbh_regs *)MXS_APBH_BASE;
31650d64	579
fa7a51cb	580	mxs_reset_block(&apbh_regs->hw_apbh_ctrl0_reg);
31650d64 MV	581
	582	#ifdef CONFIG_APBH_DMA_BURST8
	583	writel(APBH_CTRL0_AHB_BURST8_EN,
	584	&apbh_regs->hw_apbh_ctrl0_set);
	585	#else
	586	writel(APBH_CTRL0_AHB_BURST8_EN,
	587	&apbh_regs->hw_apbh_ctrl0_clr);
	588	#endif
	589
	590	#ifdef CONFIG_APBH_DMA_BURST
	591	writel(APBH_CTRL0_APB_BURST_EN,
	592	&apbh_regs->hw_apbh_ctrl0_set);
	593	#else
	594	writel(APBH_CTRL0_APB_BURST_EN,
	595	&apbh_regs->hw_apbh_ctrl0_clr);
	596	#endif
96666a39	597	}
31650d64	598
96666a39 MV	599	int mxs_dma_init_channel(int channel)
	600	{
	601	struct mxs_dma_chan *pchan;
	602	int ret;
31650d64	603
96666a39 MV	604	pchan = mxs_dma_channels + channel;
96666a39 MV	605	pchan->flags = MXS_DMA_FLAGS_VALID;
31650d64	606
96666a39	607	ret = mxs_dma_request(channel);
31650d64	608
96666a39 MV	609	if (ret) {
	610	printf("MXS DMA: Can't acquire DMA channel %i\n",
	611	channel);
	612	return ret;
31650d64 MV	613	}
31650d64 MV	614
96666a39 MV	615	mxs_dma_reset(channel);
96666a39 MV	616	mxs_dma_ack_irq(channel);
31650d64	617
96666a39	618	return 0;
31650d64	619	}