[thirdparty/kernel/stable.git] / drivers / net / wireless / mediatek / mt76 / dma.c

/*
 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <linux/dma-mapping.h>
#include "mt76.h"
#include "dma.h"

#define DMA_DUMMY_TXWI	((void *) ~0)

static int
mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q)
{
	int size;
	int i;

	spin_lock_init(&q->lock);
	INIT_LIST_HEAD(&q->swq);

	size = q->ndesc * sizeof(struct mt76_desc);
	q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
	if (!q->desc)
		return -ENOMEM;

	size = q->ndesc * sizeof(*q->entry);
	q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
	if (!q->entry)
		return -ENOMEM;

	/* clear descriptors */
	for (i = 0; i < q->ndesc; i++)
		q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);

	iowrite32(q->desc_dma, &q->regs->desc_base);
	iowrite32(0, &q->regs->cpu_idx);
	iowrite32(0, &q->regs->dma_idx);
	iowrite32(q->ndesc, &q->regs->ring_size);

	return 0;
}

static int
mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
		 struct mt76_queue_buf *buf, int nbufs, u32 info,
		 struct sk_buff *skb, void *txwi)
{
	struct mt76_desc *desc;
	u32 ctrl;
	int i, idx = -1;

	if (txwi)
		q->entry[q->head].txwi = DMA_DUMMY_TXWI;

	for (i = 0; i < nbufs; i += 2, buf += 2) {
		u32 buf0 = buf[0].addr, buf1 = 0;

		ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
		if (i < nbufs - 1) {
			buf1 = buf[1].addr;
			ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
		}

		if (i == nbufs - 1)
			ctrl |= MT_DMA_CTL_LAST_SEC0;
		else if (i == nbufs - 2)
			ctrl |= MT_DMA_CTL_LAST_SEC1;

		idx = q->head;
		q->head = (q->head + 1) % q->ndesc;

		desc = &q->desc[idx];

		WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
		WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
		WRITE_ONCE(desc->info, cpu_to_le32(info));
		WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));

		q->queued++;
	}

	q->entry[idx].txwi = txwi;
	q->entry[idx].skb = skb;

	return idx;
}

static void
mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
			struct mt76_queue_entry *prev_e)
{
	struct mt76_queue_entry *e = &q->entry[idx];
	__le32 __ctrl = READ_ONCE(q->desc[idx].ctrl);
	u32 ctrl = le32_to_cpu(__ctrl);

	if (!e->txwi || !e->skb) {
		__le32 addr = READ_ONCE(q->desc[idx].buf0);
		u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);

		dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
				 DMA_TO_DEVICE);
	}

	if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) {
		__le32 addr = READ_ONCE(q->desc[idx].buf1);
		u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl);

		dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
				 DMA_TO_DEVICE);
	}

	if (e->txwi == DMA_DUMMY_TXWI)
		e->txwi = NULL;

	*prev_e = *e;
	memset(e, 0, sizeof(*e));
}

static void
mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
{
	q->head = ioread32(&q->regs->dma_idx);
	q->tail = q->head;
	iowrite32(q->head, &q->regs->cpu_idx);
}

static void
mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
{
	struct mt76_queue *q = &dev->q_tx[qid];
	struct mt76_queue_entry entry;
	bool wake = false;
	int last;

	if (!q->ndesc)
		return;

	spin_lock_bh(&q->lock);
	if (flush)
		last = -1;
	else
		last = ioread32(&q->regs->dma_idx);

	while (q->queued && q->tail != last) {
		mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
		if (entry.schedule)
			q->swq_queued--;

		if (entry.skb)
			dev->drv->tx_complete_skb(dev, q, &entry, flush);

		if (entry.txwi) {
			mt76_put_txwi(dev, entry.txwi);
			wake = true;
		}

		q->tail = (q->tail + 1) % q->ndesc;
		q->queued--;

		if (!flush && q->tail == last)
			last = ioread32(&q->regs->dma_idx);
	}

	if (!flush)
		mt76_txq_schedule(dev, q);
	else
		mt76_dma_sync_idx(dev, q);

	wake = wake && qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;

	if (!q->queued)
		wake_up(&dev->tx_wait);

	spin_unlock_bh(&q->lock);

	if (wake)
		ieee80211_wake_queue(dev->hw, qid);
}

static void *
mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
		 int *len, u32 *info, bool *more)
{
	struct mt76_queue_entry *e = &q->entry[idx];
	struct mt76_desc *desc = &q->desc[idx];
	dma_addr_t buf_addr;
	void *buf = e->buf;
	int buf_len = SKB_WITH_OVERHEAD(q->buf_size);

	buf_addr = le32_to_cpu(READ_ONCE(desc->buf0));
	if (len) {
		u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
		*len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
		*more = !(ctl & MT_DMA_CTL_LAST_SEC0);
	}

	if (info)
		*info = le32_to_cpu(desc->info);

	dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
	e->buf = NULL;

	return buf;
}

static void *
mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
		 int *len, u32 *info, bool *more)
{
	int idx = q->tail;

	*more = false;
	if (!q->queued)
		return NULL;

	if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
		return NULL;

	q->tail = (q->tail + 1) % q->ndesc;
	q->queued--;

	return mt76_dma_get_buf(dev, q, idx, len, info, more);
}

static void
mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
{
	iowrite32(q->head, &q->regs->cpu_idx);
}

static int
mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, bool napi)
{
	dma_addr_t addr;
	void *buf;
	int frames = 0;
	int len = SKB_WITH_OVERHEAD(q->buf_size);
	int offset = q->buf_offset;
	int idx;
	void *(*alloc)(unsigned int fragsz);

	if (napi)
		alloc = napi_alloc_frag;
	else
		alloc = netdev_alloc_frag;

	spin_lock_bh(&q->lock);

	while (q->queued < q->ndesc - 1) {
		struct mt76_queue_buf qbuf;

		buf = alloc(q->buf_size);
		if (!buf)
			break;

		addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
		if (dma_mapping_error(dev->dev, addr)) {
			skb_free_frag(buf);
			break;
		}

		qbuf.addr = addr + offset;
		qbuf.len = len - offset;
		idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
		frames++;
	}

	if (frames)
		mt76_dma_kick_queue(dev, q);

	spin_unlock_bh(&q->lock);

	return frames;
}

static void
mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
{
	void *buf;
	bool more;

	spin_lock_bh(&q->lock);
	do {
		buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
		if (!buf)
			break;

		skb_free_frag(buf);
	} while (1);
	spin_unlock_bh(&q->lock);
}

static void
mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
{
	struct mt76_queue *q = &dev->q_rx[qid];
	int i;

	for (i = 0; i < q->ndesc; i++)
		q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE);

	mt76_dma_rx_cleanup(dev, q);
	mt76_dma_sync_idx(dev, q);
	mt76_dma_rx_fill(dev, q, false);
}

static void
mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
		  int len, bool more)
{
	struct page *page = virt_to_head_page(data);
	int offset = data - page_address(page);
	struct sk_buff *skb = q->rx_head;

	offset += q->buf_offset;
	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, len,
			q->buf_size);

	if (more)
		return;

	q->rx_head = NULL;
	dev->drv->rx_skb(dev, q - dev->q_rx, skb);
}

static int
mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
{
	struct sk_buff *skb;
	unsigned char *data;
	int len;
	int done = 0;
	bool more;

	while (done < budget) {
		u32 info;

		data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
		if (!data)
			break;

		if (q->rx_head) {
			mt76_add_fragment(dev, q, data, len, more);
			continue;
		}

		skb = build_skb(data, q->buf_size);
		if (!skb) {
			skb_free_frag(data);
			continue;
		}

		skb_reserve(skb, q->buf_offset);
		if (skb->tail + len > skb->end) {
			dev_kfree_skb(skb);
			continue;
		}

		if (q == &dev->q_rx[MT_RXQ_MCU]) {
			u32 *rxfce = (u32 *) skb->cb;
			*rxfce = info;
		}

		__skb_put(skb, len);
		done++;

		if (more) {
			q->rx_head = skb;
			continue;
		}

		dev->drv->rx_skb(dev, q - dev->q_rx, skb);
	}

	mt76_dma_rx_fill(dev, q, true);
	return done;
}

static int
mt76_dma_rx_poll(struct napi_struct *napi, int budget)
{
	struct mt76_dev *dev;
	int qid, done = 0, cur;

	dev = container_of(napi->dev, struct mt76_dev, napi_dev);
	qid = napi - dev->napi;

	rcu_read_lock();

	do {
		cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
		mt76_rx_poll_complete(dev, qid);
		done += cur;
	} while (cur && done < budget);

	rcu_read_unlock();

	if (done < budget) {
		napi_complete(napi);
		dev->drv->rx_poll_complete(dev, qid);
	}

	return done;
}

static int
mt76_dma_init(struct mt76_dev *dev)
{
	int i;

	init_dummy_netdev(&dev->napi_dev);

	for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
		netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
			       64);
		mt76_dma_rx_fill(dev, &dev->q_rx[i], false);
		skb_queue_head_init(&dev->rx_skb[i]);
		napi_enable(&dev->napi[i]);
	}

	return 0;
}

static const struct mt76_queue_ops mt76_dma_ops = {
	.init = mt76_dma_init,
	.alloc = mt76_dma_alloc_queue,
	.add_buf = mt76_dma_add_buf,
	.tx_cleanup = mt76_dma_tx_cleanup,
	.rx_reset = mt76_dma_rx_reset,
	.kick = mt76_dma_kick_queue,
};

int mt76_dma_attach(struct mt76_dev *dev)
{
	dev->queue_ops = &mt76_dma_ops;
	return 0;
}
EXPORT_SYMBOL_GPL(mt76_dma_attach);

void mt76_dma_cleanup(struct mt76_dev *dev)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
		mt76_dma_tx_cleanup(dev, i, true);

	for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
		netif_napi_del(&dev->napi[i]);
		mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
	}
}
EXPORT_SYMBOL_GPL(mt76_dma_cleanup);
Commit	Line	Data
17f1de56 FF	1	/*
	2	* Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
	3	*
	4	* Permission to use, copy, modify, and/or distribute this software for any
	5	* purpose with or without fee is hereby granted, provided that the above
	6	* copyright notice and this permission notice appear in all copies.
	7	*
	8	* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
	9	* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
	10	* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
	11	* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	12	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
	13	* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
	14	* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	15	*/
	16
	17	#include <linux/dma-mapping.h>
	18	#include "mt76.h"
	19	#include "dma.h"
	20
	21	#define DMA_DUMMY_TXWI ((void *) ~0)
	22
	23	static int
	24	mt76_dma_alloc_queue(struct mt76_dev dev, struct mt76_queue q)
	25	{
	26	int size;
	27	int i;
	28
	29	spin_lock_init(&q->lock);
	30	INIT_LIST_HEAD(&q->swq);
	31
	32	size = q->ndesc * sizeof(struct mt76_desc);
	33	q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
	34	if (!q->desc)
	35	return -ENOMEM;
	36
	37	size = q->ndesc * sizeof(*q->entry);
	38	q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
	39	if (!q->entry)
	40	return -ENOMEM;
	41
	42	/* clear descriptors */
	43	for (i = 0; i < q->ndesc; i++)
	44	q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
	45
	46	iowrite32(q->desc_dma, &q->regs->desc_base);
	47	iowrite32(0, &q->regs->cpu_idx);
	48	iowrite32(0, &q->regs->dma_idx);
	49	iowrite32(q->ndesc, &q->regs->ring_size);
	50
	51	return 0;
	52	}
	53
	54	static int
	55	mt76_dma_add_buf(struct mt76_dev dev, struct mt76_queue q,
	56	struct mt76_queue_buf *buf, int nbufs, u32 info,
	57	struct sk_buff skb, void txwi)
	58	{
	59	struct mt76_desc *desc;
	60	u32 ctrl;
	61	int i, idx = -1;
	62
	63	if (txwi)
	64	q->entry[q->head].txwi = DMA_DUMMY_TXWI;
65
66	for (i = 0; i < nbufs; i += 2, buf += 2) {
67	u32 buf0 = buf[0].addr, buf1 = 0;
68
69	ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
70	if (i < nbufs - 1) {
71	buf1 = buf[1].addr;
72	ctrl \|= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
73	}
74
75	if (i == nbufs - 1)
76	ctrl \|= MT_DMA_CTL_LAST_SEC0;
77	else if (i == nbufs - 2)
78	ctrl \|= MT_DMA_CTL_LAST_SEC1;
79
80	idx = q->head;
81	q->head = (q->head + 1) % q->ndesc;
82
83	desc = &q->desc[idx];
84
85	WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
86	WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
87	WRITE_ONCE(desc->info, cpu_to_le32(info));
88	WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
89
90	q->queued++;
91	}
92
93	q->entry[idx].txwi = txwi;
94	q->entry[idx].skb = skb;
95
96	return idx;
97	}
98
99	static void
100	mt76_dma_tx_cleanup_idx(struct mt76_dev dev, struct mt76_queue q, int idx,
101	struct mt76_queue_entry *prev_e)
102	{
103	struct mt76_queue_entry *e = &q->entry[idx];
104	__le32 __ctrl = READ_ONCE(q->desc[idx].ctrl);
105	u32 ctrl = le32_to_cpu(__ctrl);
106
107	if (!e->txwi \|\| !e->skb) {
108	__le32 addr = READ_ONCE(q->desc[idx].buf0);
109	u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
110
111	dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
112	DMA_TO_DEVICE);
113	}
114
115	if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) {
116	__le32 addr = READ_ONCE(q->desc[idx].buf1);
117	u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl);
118
119	dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
120	DMA_TO_DEVICE);
121	}
122
123	if (e->txwi == DMA_DUMMY_TXWI)
124	e->txwi = NULL;
125
126	prev_e = e;
127	memset(e, 0, sizeof(*e));
128	}
129
130	static void
131	mt76_dma_sync_idx(struct mt76_dev dev, struct mt76_queue q)
132	{
133	q->head = ioread32(&q->regs->dma_idx);
134	q->tail = q->head;
135	iowrite32(q->head, &q->regs->cpu_idx);
136	}
137
138	static void
139	mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
140	{
141	struct mt76_queue *q = &dev->q_tx[qid];
142	struct mt76_queue_entry entry;
143	bool wake = false;
144	int last;
145
146	if (!q->ndesc)
147	return;
148
149	spin_lock_bh(&q->lock);
150	if (flush)
151	last = -1;
152	else
153	last = ioread32(&q->regs->dma_idx);
154
155	while (q->queued && q->tail != last) {
156	mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
157	if (entry.schedule)
158	q->swq_queued--;
159
160	if (entry.skb)
161	dev->drv->tx_complete_skb(dev, q, &entry, flush);
162
163	if (entry.txwi) {
164	mt76_put_txwi(dev, entry.txwi);
165	wake = true;
166	}
167
168	q->tail = (q->tail + 1) % q->ndesc;
169	q->queued--;
170
171	if (!flush && q->tail == last)
172	last = ioread32(&q->regs->dma_idx);
173	}
174
175	if (!flush)
176	mt76_txq_schedule(dev, q);
177	else
178	mt76_dma_sync_idx(dev, q);
179
180	wake = wake && qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
26e40d4c FF	181
	182	if (!q->queued)
	183	wake_up(&dev->tx_wait);
	184
17f1de56 FF	185	spin_unlock_bh(&q->lock);
	186
	187	if (wake)
	188	ieee80211_wake_queue(dev->hw, qid);
	189	}
	190
	191	static void *
	192	mt76_dma_get_buf(struct mt76_dev dev, struct mt76_queue q, int idx,
	193	int len, u32 info, bool *more)
	194	{
	195	struct mt76_queue_entry *e = &q->entry[idx];
	196	struct mt76_desc *desc = &q->desc[idx];
	197	dma_addr_t buf_addr;
	198	void *buf = e->buf;
	199	int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
	200
	201	buf_addr = le32_to_cpu(READ_ONCE(desc->buf0));
	202	if (len) {
	203	u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
	204	*len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
	205	*more = !(ctl & MT_DMA_CTL_LAST_SEC0);
	206	}
	207
	208	if (info)
	209	*info = le32_to_cpu(desc->info);
	210
	211	dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
	212	e->buf = NULL;
	213
	214	return buf;
	215	}
	216
	217	static void *
	218	mt76_dma_dequeue(struct mt76_dev dev, struct mt76_queue q, bool flush,
	219	int len, u32 info, bool *more)
	220	{
	221	int idx = q->tail;
	222
	223	*more = false;
	224	if (!q->queued)
	225	return NULL;
	226
	227	if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
	228	return NULL;
	229
	230	q->tail = (q->tail + 1) % q->ndesc;
	231	q->queued--;
	232
	233	return mt76_dma_get_buf(dev, q, idx, len, info, more);
	234	}
	235
	236	static void
	237	mt76_dma_kick_queue(struct mt76_dev dev, struct mt76_queue q)
	238	{
	239	iowrite32(q->head, &q->regs->cpu_idx);
	240	}
	241
	242	static int
	243	mt76_dma_rx_fill(struct mt76_dev dev, struct mt76_queue q, bool napi)
	244	{
	245	dma_addr_t addr;
	246	void *buf;
	247	int frames = 0;
	248	int len = SKB_WITH_OVERHEAD(q->buf_size);
249	int offset = q->buf_offset;
250	int idx;
251	void (alloc)(unsigned int fragsz);
252
253	if (napi)
254	alloc = napi_alloc_frag;
255	else
256	alloc = netdev_alloc_frag;
257
258	spin_lock_bh(&q->lock);
259
260	while (q->queued < q->ndesc - 1) {
261	struct mt76_queue_buf qbuf;
262
263	buf = alloc(q->buf_size);
264	if (!buf)
265	break;
266
267	addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
268	if (dma_mapping_error(dev->dev, addr)) {
269	skb_free_frag(buf);
270	break;
271	}
272
273	qbuf.addr = addr + offset;
274	qbuf.len = len - offset;
275	idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
276	frames++;
277	}
278
279	if (frames)
280	mt76_dma_kick_queue(dev, q);
281
282	spin_unlock_bh(&q->lock);
283
284	return frames;
285	}
286
287	static void
288	mt76_dma_rx_cleanup(struct mt76_dev dev, struct mt76_queue q)
289	{
290	void *buf;
291	bool more;
292
293	spin_lock_bh(&q->lock);
294	do {
295	buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
296	if (!buf)
297	break;
298
299	skb_free_frag(buf);
300	} while (1);
301	spin_unlock_bh(&q->lock);
302	}
303
304	static void
305	mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
306	{
307	struct mt76_queue *q = &dev->q_rx[qid];
308	int i;
309
310	for (i = 0; i < q->ndesc; i++)
311	q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE);
312
313	mt76_dma_rx_cleanup(dev, q);
314	mt76_dma_sync_idx(dev, q);
315	mt76_dma_rx_fill(dev, q, false);
316	}
317
318	static void
319	mt76_add_fragment(struct mt76_dev dev, struct mt76_queue q, void *data,
320	int len, bool more)
321	{
322	struct page *page = virt_to_head_page(data);
323	int offset = data - page_address(page);
324	struct sk_buff *skb = q->rx_head;
325
326	offset += q->buf_offset;
327	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, len,
328	q->buf_size);
329
330	if (more)
331	return;
332
333	q->rx_head = NULL;
334	dev->drv->rx_skb(dev, q - dev->q_rx, skb);
335	}
336
337	static int
338	mt76_dma_rx_process(struct mt76_dev dev, struct mt76_queue q, int budget)
339	{
340	struct sk_buff *skb;
341	unsigned char *data;
342	int len;
343	int done = 0;
344	bool more;
345
346	while (done < budget) {
347	u32 info;
348
349	data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
350	if (!data)
351	break;
352
353	if (q->rx_head) {
354	mt76_add_fragment(dev, q, data, len, more);
355	continue;
356	}
357
358	skb = build_skb(data, q->buf_size);
359	if (!skb) {
360	skb_free_frag(data);
361	continue;
362	}
363
364	skb_reserve(skb, q->buf_offset);
365	if (skb->tail + len > skb->end) {
366	dev_kfree_skb(skb);
367	continue;
368	}
369
370	if (q == &dev->q_rx[MT_RXQ_MCU]) {
371	u32 rxfce = (u32 ) skb->cb;
372	*rxfce = info;
373	}
374
375	__skb_put(skb, len);
376	done++;
377
378	if (more) {
379	q->rx_head = skb;
380	continue;
381	}
382
383	dev->drv->rx_skb(dev, q - dev->q_rx, skb);
384	}
385
386	mt76_dma_rx_fill(dev, q, true);
387	return done;
388	}
389
390	static int
391	mt76_dma_rx_poll(struct napi_struct *napi, int budget)
392	{
393	struct mt76_dev *dev;
2b4307f5	394	int qid, done = 0, cur;
17f1de56 FF	395
	396	dev = container_of(napi->dev, struct mt76_dev, napi_dev);
	397	qid = napi - dev->napi;
	398
9c68a57b FF	399	rcu_read_lock();
9c68a57b FF	400
2b4307f5 FF	401	do {
2b4307f5 FF	402	cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
9d9d738b	403	mt76_rx_poll_complete(dev, qid);
2b4307f5 FF	404	done += cur;
	405	} while (cur && done < budget);
	406
9c68a57b FF	407	rcu_read_unlock();
9c68a57b FF	408
17f1de56 FF	409	if (done < budget) {
	410	napi_complete(napi);
	411	dev->drv->rx_poll_complete(dev, qid);
	412	}
17f1de56 FF	413
	414	return done;
	415	}
	416
	417	static int
	418	mt76_dma_init(struct mt76_dev *dev)
	419	{
	420	int i;
	421
	422	init_dummy_netdev(&dev->napi_dev);
	423
	424	for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
	425	netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
	426	64);
	427	mt76_dma_rx_fill(dev, &dev->q_rx[i], false);
	428	skb_queue_head_init(&dev->rx_skb[i]);
	429	napi_enable(&dev->napi[i]);
	430	}
	431
	432	return 0;
	433	}
	434
	435	static const struct mt76_queue_ops mt76_dma_ops = {
	436	.init = mt76_dma_init,
	437	.alloc = mt76_dma_alloc_queue,
	438	.add_buf = mt76_dma_add_buf,
	439	.tx_cleanup = mt76_dma_tx_cleanup,
	440	.rx_reset = mt76_dma_rx_reset,
	441	.kick = mt76_dma_kick_queue,
	442	};
	443
	444	int mt76_dma_attach(struct mt76_dev *dev)
	445	{
	446	dev->queue_ops = &mt76_dma_ops;
	447	return 0;
	448	}
	449	EXPORT_SYMBOL_GPL(mt76_dma_attach);
	450
	451	void mt76_dma_cleanup(struct mt76_dev *dev)
	452	{
	453	int i;
	454
	455	for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
	456	mt76_dma_tx_cleanup(dev, i, true);
	457
	458	for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
	459	netif_napi_del(&dev->napi[i]);
	460	mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
	461	}
	462	}
	463	EXPORT_SYMBOL_GPL(mt76_dma_cleanup);