#include "idpf_ptp.h"
#include "idpf_virtchnl.h"
#include "xdp.h"
+#include "xsk.h"
#define idpf_tx_buf_next(buf) (*(u32 *)&(buf)->priv)
LIBETH_SQE_CHECK_PRIV(u32);
}
}
-/**
- * idpf_tx_buf_rel_all - Free any empty Tx buffers
- * @txq: queue to be cleaned
- */
-static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq)
+static void idpf_tx_buf_clean(struct idpf_tx_queue *txq)
{
struct libeth_sq_napi_stats ss = { };
struct xdp_frame_bulk bq;
.bq = &bq,
.ss = &ss,
};
- u32 i;
-
- /* Buffers already cleared, nothing to do */
- if (!txq->tx_buf)
- return;
xdp_frame_bulk_init(&bq);
/* Free all the Tx buffer sk_buffs */
- for (i = 0; i < txq->buf_pool_size; i++)
+ for (u32 i = 0; i < txq->buf_pool_size; i++)
libeth_tx_complete_any(&txq->tx_buf[i], &cp);
xdp_flush_frame_bulk(&bq);
+}
+
+/**
+ * idpf_tx_buf_rel_all - Free any empty Tx buffers
+ * @txq: queue to be cleaned
+ */
+static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq)
+{
+ /* Buffers already cleared, nothing to do */
+ if (!txq->tx_buf)
+ return;
+
+ if (idpf_queue_has(XSK, txq))
+ idpf_xsksq_clean(txq);
+ else
+ idpf_tx_buf_clean(txq);
kfree(txq->tx_buf);
txq->tx_buf = NULL;
if (!xdp)
netdev_tx_reset_subqueue(txq->netdev, txq->idx);
+ idpf_xsk_clear_queue(txq, VIRTCHNL2_QUEUE_TYPE_TX);
+
if (!txq->desc_ring)
return;
*/
static void idpf_compl_desc_rel(struct idpf_compl_queue *complq)
{
+ idpf_xsk_clear_queue(complq, VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION);
+
if (!complq->comp)
return;
tx_q->next_to_clean = 0;
idpf_queue_set(GEN_CHK, tx_q);
+ idpf_xsk_setup_queue(vport, tx_q, VIRTCHNL2_QUEUE_TYPE_TX);
+
if (!idpf_queue_has(FLOW_SCH_EN, tx_q))
return 0;
complq->next_to_clean = 0;
idpf_queue_set(GEN_CHK, complq);
+ idpf_xsk_setup_queue(vport, complq,
+ VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION);
+
return 0;
}
static struct idpf_queue_set *
idpf_vector_to_queue_set(struct idpf_q_vector *qv)
{
- bool xdp = qv->vport->xdp_txq_offset;
+ bool xdp = qv->vport->xdp_txq_offset && !qv->num_xsksq;
struct idpf_vport *vport = qv->vport;
struct idpf_queue_set *qs;
u32 num;
num = qv->num_rxq + qv->num_bufq + qv->num_txq + qv->num_complq;
- num += xdp ? qv->num_rxq * 2 : 0;
+ num += xdp ? qv->num_rxq * 2 : qv->num_xsksq * 2;
if (!num)
return NULL;
qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION;
qs->qs[num++].complq = vport->txqs[idx]->complq;
}
+ } else {
+ for (u32 i = 0; i < qv->num_xsksq; i++) {
+ qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX;
+ qs->qs[num++].txq = qv->xsksq[i];
+
+ qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION;
+ qs->qs[num++].complq = qv->xsksq[i]->complq;
+ }
}
finalize:
return err;
}
+ if (!vport->xdp_txq_offset)
+ goto config;
+
+ q_vector->xsksq = kcalloc(DIV_ROUND_UP(vport->num_rxq_grp,
+ vport->num_q_vectors),
+ sizeof(*q_vector->xsksq), GFP_KERNEL);
+ if (!q_vector->xsksq)
+ return -ENOMEM;
+
+ for (u32 i = 0; i < qs->num; i++) {
+ const struct idpf_queue_ptr *q = &qs->qs[i];
+
+ if (q->type != VIRTCHNL2_QUEUE_TYPE_TX)
+ continue;
+
+ if (!idpf_queue_has(XSK, q->txq))
+ continue;
+
+ q->txq->q_vector = q_vector;
+ q_vector->xsksq[q_vector->num_xsksq++] = q->txq;
+ }
+
+config:
err = idpf_send_config_queue_set_msg(qs);
if (err) {
netdev_err(vport->netdev, "Could not configure queues in pair %u: %pe\n",
idpf_clean_queue_set(qs);
+ kfree(q_vector->xsksq);
+ q_vector->num_xsksq = 0;
+
return 0;
}
struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data;
q_vector->total_events++;
- napi_schedule(&q_vector->napi);
+ napi_schedule_irqoff(&q_vector->napi);
return IRQ_HANDLED;
}
for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx];
+ kfree(q_vector->xsksq);
+ q_vector->xsksq = NULL;
kfree(q_vector->complq);
q_vector->complq = NULL;
kfree(q_vector->bufq);
{
struct idpf_q_vector *q_vector =
container_of(napi, struct idpf_q_vector, napi);
- bool clean_complete;
+ bool clean_complete = true;
int work_done = 0;
/* Handle case where we are called by netpoll with a budget of 0 */
return 0;
}
- clean_complete = idpf_rx_splitq_clean_all(q_vector, budget, &work_done);
- clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done);
+ for (u32 i = 0; i < q_vector->num_xsksq; i++)
+ clean_complete &= idpf_xsk_xmit(q_vector->xsksq[i]);
+
+ clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget,
+ &work_done);
+ clean_complete &= idpf_rx_splitq_clean_all(q_vector, budget,
+ &work_done);
/* If work not completed, return budget and polling will return */
if (!clean_complete) {
/* Exit the polling mode, but don't re-enable interrupts if stack might
* poll us due to busy-polling
*/
- if (likely(napi_complete_done(napi, work_done)))
+ if (napi_complete_done(napi, work_done))
idpf_vport_intr_update_itr_ena_irq(q_vector);
else
idpf_vport_intr_set_wb_on_itr(q_vector);
qv_idx++;
}
+
+ for (i = 0; i < vport->num_xdp_txq; i++) {
+ struct idpf_tx_queue *xdpsq;
+ struct idpf_q_vector *qv;
+
+ xdpsq = vport->txqs[vport->xdp_txq_offset + i];
+ if (!idpf_queue_has(XSK, xdpsq))
+ continue;
+
+ qv = idpf_find_rxq_vec(vport, i);
+
+ xdpsq->q_vector = qv;
+ qv->xsksq[qv->num_xsksq++] = xdpsq;
+ }
}
/**
GFP_KERNEL);
if (!q_vector->complq)
goto error;
+
+ if (!vport->xdp_txq_offset)
+ continue;
+
+ q_vector->xsksq = kcalloc(rxqs_per_vector,
+ sizeof(*q_vector->xsksq),
+ GFP_KERNEL);
+ if (!q_vector->xsksq)
+ goto error;
}
return 0;
* queue
* @__IDPF_Q_NOIRQ: queue is polling-driven and has no interrupt
* @__IDPF_Q_XDP: this is an XDP queue
+ * @__IDPF_Q_XSK: the queue has an XSk pool installed
* @__IDPF_Q_FLAGS_NBITS: Must be last
*/
enum idpf_queue_flags_t {
__IDPF_Q_PTP,
__IDPF_Q_NOIRQ,
__IDPF_Q_XDP,
+ __IDPF_Q_XSK,
__IDPF_Q_FLAGS_NBITS,
};
* @num_txq: Number of TX queues
* @num_bufq: Number of buffer queues
* @num_complq: number of completion queues
+ * @num_xsksq: number of XSk send queues
* @rx: Array of RX queues to service
* @tx: Array of TX queues to service
* @bufq: Array of buffer queues to service
* @complq: array of completion queues
+ * @xsksq: array of XSk send queues
* @intr_reg: See struct idpf_intr_reg
* @napi: napi handler
* @total_events: Number of interrupts processed
u16 num_txq;
u16 num_bufq;
u16 num_complq;
+ u16 num_xsksq;
struct idpf_rx_queue **rx;
struct idpf_tx_queue **tx;
struct idpf_buf_queue **bufq;
struct idpf_compl_queue **complq;
+ struct idpf_tx_queue **xsksq;
struct idpf_intr_reg intr_reg;
__cacheline_group_end_aligned(read_mostly);
__cacheline_group_end_aligned(cold);
};
-libeth_cacheline_set_assert(struct idpf_q_vector, 120,
+libeth_cacheline_set_assert(struct idpf_q_vector, 136,
24 + sizeof(struct napi_struct) +
2 * sizeof(struct dim),
8);
* @txq_grp: See struct idpf_txq_group
* @complq: corresponding completion queue in XDP mode
* @dev: Device back pointer for DMA mapping
+ * @pool: corresponding XSk pool if installed
* @tail: Tail offset. Used for both queue models single and split
* @flags: See enum idpf_queue_flags_t
* @idx: For TX queue, it is used as index to map between TX queue group and
struct idpf_txq_group *txq_grp;
struct idpf_compl_queue *complq;
};
- struct device *dev;
+ union {
+ struct device *dev;
+ struct xsk_buff_pool *pool;
+ };
void __iomem *tail;
DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS);
return upper_16_bits(val);
}
-static u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget)
+u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget)
{
struct idpf_compl_queue *cq = xdpsq->complq;
u32 tx_ntc = xdpsq->next_to_clean;
int idpf_xdpsqs_get(const struct idpf_vport *vport);
void idpf_xdpsqs_put(const struct idpf_vport *vport);
+u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget);
bool idpf_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags);
/**
#include <net/libeth/xsk.h>
#include "idpf.h"
+#include "xdp.h"
#include "xsk.h"
+static void idpf_xsk_tx_timer(struct work_struct *work);
+
+static void idpf_xsk_setup_txq(const struct idpf_vport *vport,
+ struct idpf_tx_queue *txq)
+{
+ struct xsk_buff_pool *pool;
+ u32 qid;
+
+ idpf_queue_clear(XSK, txq);
+
+ if (!idpf_queue_has(XDP, txq))
+ return;
+
+ qid = txq->idx - vport->xdp_txq_offset;
+
+ pool = xsk_get_pool_from_qid(vport->netdev, qid);
+ if (!pool || !pool->dev)
+ return;
+
+ txq->pool = pool;
+ libeth_xdpsq_init_timer(txq->timer, txq, &txq->xdp_lock,
+ idpf_xsk_tx_timer);
+
+ idpf_queue_assign(NOIRQ, txq, xsk_uses_need_wakeup(pool));
+ idpf_queue_set(XSK, txq);
+}
+
+static void idpf_xsk_setup_complq(const struct idpf_vport *vport,
+ struct idpf_compl_queue *complq)
+{
+ const struct xsk_buff_pool *pool;
+ u32 qid;
+
+ idpf_queue_clear(XSK, complq);
+
+ if (!idpf_queue_has(XDP, complq))
+ return;
+
+ qid = complq->txq_grp->txqs[0]->idx - vport->xdp_txq_offset;
+
+ pool = xsk_get_pool_from_qid(vport->netdev, qid);
+ if (!pool || !pool->dev)
+ return;
+
+ idpf_queue_set(XSK, complq);
+}
+
+void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q,
+ enum virtchnl2_queue_type type)
+{
+ if (!idpf_xdp_enabled(vport))
+ return;
+
+ switch (type) {
+ case VIRTCHNL2_QUEUE_TYPE_TX:
+ idpf_xsk_setup_txq(vport, q);
+ break;
+ case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION:
+ idpf_xsk_setup_complq(vport, q);
+ break;
+ default:
+ break;
+ }
+}
+
+void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type)
+{
+ struct idpf_compl_queue *complq;
+ struct idpf_tx_queue *txq;
+
+ switch (type) {
+ case VIRTCHNL2_QUEUE_TYPE_TX:
+ txq = q;
+ if (!idpf_queue_has_clear(XSK, txq))
+ return;
+
+ idpf_queue_set(NOIRQ, txq);
+ txq->dev = txq->netdev->dev.parent;
+ break;
+ case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION:
+ complq = q;
+ idpf_queue_clear(XSK, complq);
+ break;
+ default:
+ break;
+ }
+}
+
+void idpf_xsksq_clean(struct idpf_tx_queue *xdpsq)
+{
+ struct libeth_xdpsq_napi_stats ss = { };
+ u32 ntc = xdpsq->next_to_clean;
+ struct xdp_frame_bulk bq;
+ struct libeth_cq_pp cp = {
+ .dev = xdpsq->pool->dev,
+ .bq = &bq,
+ .xss = &ss,
+ };
+ u32 xsk_frames = 0;
+
+ xdp_frame_bulk_init(&bq);
+
+ while (ntc != xdpsq->next_to_use) {
+ struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc];
+
+ if (sqe->type)
+ libeth_xdp_complete_tx(sqe, &cp);
+ else
+ xsk_frames++;
+
+ if (unlikely(++ntc == xdpsq->desc_count))
+ ntc = 0;
+ }
+
+ xdp_flush_frame_bulk(&bq);
+
+ if (xsk_frames)
+ xsk_tx_completed(xdpsq->pool, xsk_frames);
+}
+
+static noinline u32 idpf_xsksq_complete_slow(struct idpf_tx_queue *xdpsq,
+ u32 done)
+{
+ struct libeth_xdpsq_napi_stats ss = { };
+ u32 ntc = xdpsq->next_to_clean;
+ u32 cnt = xdpsq->desc_count;
+ struct xdp_frame_bulk bq;
+ struct libeth_cq_pp cp = {
+ .dev = xdpsq->pool->dev,
+ .bq = &bq,
+ .xss = &ss,
+ .napi = true,
+ };
+ u32 xsk_frames = 0;
+
+ xdp_frame_bulk_init(&bq);
+
+ for (u32 i = 0; likely(i < done); i++) {
+ struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc];
+
+ if (sqe->type)
+ libeth_xdp_complete_tx(sqe, &cp);
+ else
+ xsk_frames++;
+
+ if (unlikely(++ntc == cnt))
+ ntc = 0;
+ }
+
+ xdp_flush_frame_bulk(&bq);
+
+ xdpsq->next_to_clean = ntc;
+ xdpsq->xdp_tx -= cp.xdp_tx;
+
+ return xsk_frames;
+}
+
+static __always_inline u32 idpf_xsksq_complete(void *_xdpsq, u32 budget)
+{
+ struct idpf_tx_queue *xdpsq = _xdpsq;
+ u32 tx_ntc = xdpsq->next_to_clean;
+ u32 tx_cnt = xdpsq->desc_count;
+ u32 done_frames;
+ u32 xsk_frames;
+
+ done_frames = idpf_xdpsq_poll(xdpsq, budget);
+ if (unlikely(!done_frames))
+ return 0;
+
+ if (likely(!xdpsq->xdp_tx)) {
+ tx_ntc += done_frames;
+ if (tx_ntc >= tx_cnt)
+ tx_ntc -= tx_cnt;
+
+ xdpsq->next_to_clean = tx_ntc;
+ xsk_frames = done_frames;
+
+ goto finalize;
+ }
+
+ xsk_frames = idpf_xsksq_complete_slow(xdpsq, done_frames);
+ if (xsk_frames)
+finalize:
+ xsk_tx_completed(xdpsq->pool, xsk_frames);
+
+ xdpsq->pending -= done_frames;
+
+ return done_frames;
+}
+
+static u32 idpf_xsk_xmit_prep(void *_xdpsq, struct libeth_xdpsq *sq)
+{
+ struct idpf_tx_queue *xdpsq = _xdpsq;
+
+ *sq = (struct libeth_xdpsq){
+ .pool = xdpsq->pool,
+ .sqes = xdpsq->tx_buf,
+ .descs = xdpsq->desc_ring,
+ .count = xdpsq->desc_count,
+ .lock = &xdpsq->xdp_lock,
+ .ntu = &xdpsq->next_to_use,
+ .pending = &xdpsq->pending,
+ };
+
+ /*
+ * The queue is cleaned, the budget is already known, optimize out
+ * the second min() by passing the type limit.
+ */
+ return U32_MAX;
+}
+
+bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq)
+{
+ u32 free;
+
+ libeth_xdpsq_lock(&xsksq->xdp_lock);
+
+ free = xsksq->desc_count - xsksq->pending;
+ if (free < xsksq->thresh)
+ free += idpf_xsksq_complete(xsksq, xsksq->thresh);
+
+ return libeth_xsk_xmit_do_bulk(xsksq->pool, xsksq,
+ min(free - 1, xsksq->thresh),
+ libeth_xsktmo, idpf_xsk_xmit_prep,
+ idpf_xdp_tx_xmit, idpf_xdp_tx_finalize);
+}
+
+LIBETH_XDP_DEFINE_START();
+LIBETH_XDP_DEFINE_TIMER(static idpf_xsk_tx_timer, idpf_xsksq_complete);
+LIBETH_XDP_DEFINE_END();
+
int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *bpf)
{
struct xsk_buff_pool *pool = bpf->xsk.pool;
#include <linux/types.h>
+enum virtchnl2_queue_type;
+struct idpf_tx_queue;
struct idpf_vport;
struct netdev_bpf;
+void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q,
+ enum virtchnl2_queue_type type);
+void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type);
+
+void idpf_xsksq_clean(struct idpf_tx_queue *xdpq);
+bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq);
+
int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *xdp);
#endif /* !_IDPF_XSK_H_ */
projects / thirdparty / linux.git / commitdiff
