[people/ms/linux.git] / net / core / xdp.c

// SPDX-License-Identifier: GPL-2.0-only
/* net/core/xdp.c
 *
 * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
 */
#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/netdevice.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/rhashtable.h>
#include <net/page_pool.h>

#include <net/xdp.h>
#include <net/xdp_priv.h> /* struct xdp_mem_allocator */
#include <trace/events/xdp.h>

#define REG_STATE_NEW		0x0
#define REG_STATE_REGISTERED	0x1
#define REG_STATE_UNREGISTERED	0x2
#define REG_STATE_UNUSED	0x3

static DEFINE_IDA(mem_id_pool);
static DEFINE_MUTEX(mem_id_lock);
#define MEM_ID_MAX 0xFFFE
#define MEM_ID_MIN 1
static int mem_id_next = MEM_ID_MIN;

static bool mem_id_init; /* false */
static struct rhashtable *mem_id_ht;

static u32 xdp_mem_id_hashfn(const void *data, u32 len, u32 seed)
{
	const u32 *k = data;
	const u32 key = *k;

	BUILD_BUG_ON(FIELD_SIZEOF(struct xdp_mem_allocator, mem.id)
		     != sizeof(u32));

	/* Use cyclic increasing ID as direct hash key */
	return key;
}

static int xdp_mem_id_cmp(struct rhashtable_compare_arg *arg,
			  const void *ptr)
{
	const struct xdp_mem_allocator *xa = ptr;
	u32 mem_id = *(u32 *)arg->key;

	return xa->mem.id != mem_id;
}

static const struct rhashtable_params mem_id_rht_params = {
	.nelem_hint = 64,
	.head_offset = offsetof(struct xdp_mem_allocator, node),
	.key_offset  = offsetof(struct xdp_mem_allocator, mem.id),
	.key_len = FIELD_SIZEOF(struct xdp_mem_allocator, mem.id),
	.max_size = MEM_ID_MAX,
	.min_size = 8,
	.automatic_shrinking = true,
	.hashfn    = xdp_mem_id_hashfn,
	.obj_cmpfn = xdp_mem_id_cmp,
};

static void __xdp_mem_allocator_rcu_free(struct rcu_head *rcu)
{
	struct xdp_mem_allocator *xa;

	xa = container_of(rcu, struct xdp_mem_allocator, rcu);

	/* Allocator have indicated safe to remove before this is called */
	if (xa->mem.type == MEM_TYPE_PAGE_POOL)
		page_pool_free(xa->page_pool);

	/* Allow this ID to be reused */
	ida_simple_remove(&mem_id_pool, xa->mem.id);

	/* Poison memory */
	xa->mem.id = 0xFFFF;
	xa->mem.type = 0xF0F0;
	xa->allocator = (void *)0xDEAD9001;

	kfree(xa);
}

bool __mem_id_disconnect(int id, bool force)
{
	struct xdp_mem_allocator *xa;
	bool safe_to_remove = true;

	mutex_lock(&mem_id_lock);

	xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params);
	if (!xa) {
		mutex_unlock(&mem_id_lock);
		WARN(1, "Request remove non-existing id(%d), driver bug?", id);
		return true;
	}
	xa->disconnect_cnt++;

	/* Detects in-flight packet-pages for page_pool */
	if (xa->mem.type == MEM_TYPE_PAGE_POOL)
		safe_to_remove = page_pool_request_shutdown(xa->page_pool);

	trace_mem_disconnect(xa, safe_to_remove, force);

	if ((safe_to_remove || force) &&
	    !rhashtable_remove_fast(mem_id_ht, &xa->node, mem_id_rht_params))
		call_rcu(&xa->rcu, __xdp_mem_allocator_rcu_free);

	mutex_unlock(&mem_id_lock);
	return (safe_to_remove|force);
}

#define DEFER_TIME (msecs_to_jiffies(1000))
#define DEFER_WARN_INTERVAL (30 * HZ)
#define DEFER_MAX_RETRIES 120

static void mem_id_disconnect_defer_retry(struct work_struct *wq)
{
	struct delayed_work *dwq = to_delayed_work(wq);
	struct xdp_mem_allocator *xa = container_of(dwq, typeof(*xa), defer_wq);
	bool force = false;

	if (xa->disconnect_cnt > DEFER_MAX_RETRIES)
		force = true;

	if (__mem_id_disconnect(xa->mem.id, force))
		return;

	/* Periodic warning */
	if (time_after_eq(jiffies, xa->defer_warn)) {
		int sec = (s32)((u32)jiffies - (u32)xa->defer_start) / HZ;

		pr_warn("%s() stalled mem.id=%u shutdown %d attempts %d sec\n",
			__func__, xa->mem.id, xa->disconnect_cnt, sec);
		xa->defer_warn = jiffies + DEFER_WARN_INTERVAL;
	}

	/* Still not ready to be disconnected, retry later */
	schedule_delayed_work(&xa->defer_wq, DEFER_TIME);
}

void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq)
{
	struct xdp_mem_allocator *xa;
	int id = xdp_rxq->mem.id;

	if (xdp_rxq->reg_state != REG_STATE_REGISTERED) {
		WARN(1, "Missing register, driver bug");
		return;
	}

	if (xdp_rxq->mem.type != MEM_TYPE_PAGE_POOL &&
	    xdp_rxq->mem.type != MEM_TYPE_ZERO_COPY) {
		return;
	}

	if (id == 0)
		return;

	if (__mem_id_disconnect(id, false))
		return;

	/* Could not disconnect, defer new disconnect attempt to later */
	mutex_lock(&mem_id_lock);

	xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params);
	if (!xa) {
		mutex_unlock(&mem_id_lock);
		return;
	}
	xa->defer_start = jiffies;
	xa->defer_warn  = jiffies + DEFER_WARN_INTERVAL;

	INIT_DELAYED_WORK(&xa->defer_wq, mem_id_disconnect_defer_retry);
	mutex_unlock(&mem_id_lock);
	schedule_delayed_work(&xa->defer_wq, DEFER_TIME);
}
EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg_mem_model);

/* This unregister operation will also cleanup and destroy the
 * allocator. The page_pool_free() operation is first called when it's
 * safe to remove, possibly deferred to a workqueue.
 */
void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq)
{
	/* Simplify driver cleanup code paths, allow unreg "unused" */
	if (xdp_rxq->reg_state == REG_STATE_UNUSED)
		return;

	WARN(!(xdp_rxq->reg_state == REG_STATE_REGISTERED), "Driver BUG");

	xdp_rxq_info_unreg_mem_model(xdp_rxq);

	xdp_rxq->reg_state = REG_STATE_UNREGISTERED;
	xdp_rxq->dev = NULL;

	/* Reset mem info to defaults */
	xdp_rxq->mem.id = 0;
	xdp_rxq->mem.type = 0;
}
EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg);

static void xdp_rxq_info_init(struct xdp_rxq_info *xdp_rxq)
{
	memset(xdp_rxq, 0, sizeof(*xdp_rxq));
}

/* Returns 0 on success, negative on failure */
int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
		     struct net_device *dev, u32 queue_index)
{
	if (xdp_rxq->reg_state == REG_STATE_UNUSED) {
		WARN(1, "Driver promised not to register this");
		return -EINVAL;
	}

	if (xdp_rxq->reg_state == REG_STATE_REGISTERED) {
		WARN(1, "Missing unregister, handled but fix driver");
		xdp_rxq_info_unreg(xdp_rxq);
	}

	if (!dev) {
		WARN(1, "Missing net_device from driver");
		return -ENODEV;
	}

	/* State either UNREGISTERED or NEW */
	xdp_rxq_info_init(xdp_rxq);
	xdp_rxq->dev = dev;
	xdp_rxq->queue_index = queue_index;

	xdp_rxq->reg_state = REG_STATE_REGISTERED;
	return 0;
}
EXPORT_SYMBOL_GPL(xdp_rxq_info_reg);

void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq)
{
	xdp_rxq->reg_state = REG_STATE_UNUSED;
}
EXPORT_SYMBOL_GPL(xdp_rxq_info_unused);

bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq)
{
	return (xdp_rxq->reg_state == REG_STATE_REGISTERED);
}
EXPORT_SYMBOL_GPL(xdp_rxq_info_is_reg);

static int __mem_id_init_hash_table(void)
{
	struct rhashtable *rht;
	int ret;

	if (unlikely(mem_id_init))
		return 0;

	rht = kzalloc(sizeof(*rht), GFP_KERNEL);
	if (!rht)
		return -ENOMEM;

	ret = rhashtable_init(rht, &mem_id_rht_params);
	if (ret < 0) {
		kfree(rht);
		return ret;
	}
	mem_id_ht = rht;
	smp_mb(); /* mutex lock should provide enough pairing */
	mem_id_init = true;

	return 0;
}

/* Allocate a cyclic ID that maps to allocator pointer.
 * See: https://www.kernel.org/doc/html/latest/core-api/idr.html
 *
 * Caller must lock mem_id_lock.
 */
static int __mem_id_cyclic_get(gfp_t gfp)
{
	int retries = 1;
	int id;

again:
	id = ida_simple_get(&mem_id_pool, mem_id_next, MEM_ID_MAX, gfp);
	if (id < 0) {
		if (id == -ENOSPC) {
			/* Cyclic allocator, reset next id */
			if (retries--) {
				mem_id_next = MEM_ID_MIN;
				goto again;
			}
		}
		return id; /* errno */
	}
	mem_id_next = id + 1;

	return id;
}

static bool __is_supported_mem_type(enum xdp_mem_type type)
{
	if (type == MEM_TYPE_PAGE_POOL)
		return is_page_pool_compiled_in();

	if (type >= MEM_TYPE_MAX)
		return false;

	return true;
}

int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
			       enum xdp_mem_type type, void *allocator)
{
	struct xdp_mem_allocator *xdp_alloc;
	gfp_t gfp = GFP_KERNEL;
	int id, errno, ret;
	void *ptr;

	if (xdp_rxq->reg_state != REG_STATE_REGISTERED) {
		WARN(1, "Missing register, driver bug");
		return -EFAULT;
	}

	if (!__is_supported_mem_type(type))
		return -EOPNOTSUPP;

	xdp_rxq->mem.type = type;

	if (!allocator) {
		if (type == MEM_TYPE_PAGE_POOL || type == MEM_TYPE_ZERO_COPY)
			return -EINVAL; /* Setup time check page_pool req */
		return 0;
	}

	/* Delay init of rhashtable to save memory if feature isn't used */
	if (!mem_id_init) {
		mutex_lock(&mem_id_lock);
		ret = __mem_id_init_hash_table();
		mutex_unlock(&mem_id_lock);
		if (ret < 0) {
			WARN_ON(1);
			return ret;
		}
	}

	xdp_alloc = kzalloc(sizeof(*xdp_alloc), gfp);
	if (!xdp_alloc)
		return -ENOMEM;

	mutex_lock(&mem_id_lock);
	id = __mem_id_cyclic_get(gfp);
	if (id < 0) {
		errno = id;
		goto err;
	}
	xdp_rxq->mem.id = id;
	xdp_alloc->mem  = xdp_rxq->mem;
	xdp_alloc->allocator = allocator;

	/* Insert allocator into ID lookup table */
	ptr = rhashtable_insert_slow(mem_id_ht, &id, &xdp_alloc->node);
	if (IS_ERR(ptr)) {
		ida_simple_remove(&mem_id_pool, xdp_rxq->mem.id);
		xdp_rxq->mem.id = 0;
		errno = PTR_ERR(ptr);
		goto err;
	}

	mutex_unlock(&mem_id_lock);

	trace_mem_connect(xdp_alloc, xdp_rxq);
	return 0;
err:
	mutex_unlock(&mem_id_lock);
	kfree(xdp_alloc);
	return errno;
}
EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);

/* XDP RX runs under NAPI protection, and in different delivery error
 * scenarios (e.g. queue full), it is possible to return the xdp_frame
 * while still leveraging this protection.  The @napi_direct boolian
 * is used for those calls sites.  Thus, allowing for faster recycling
 * of xdp_frames/pages in those cases.
 */
static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
			 unsigned long handle)
{
	struct xdp_mem_allocator *xa;
	struct page *page;

	switch (mem->type) {
	case MEM_TYPE_PAGE_POOL:
		rcu_read_lock();
		/* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */
		xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
		page = virt_to_head_page(data);
		if (likely(xa)) {
			napi_direct &= !xdp_return_frame_no_direct();
			page_pool_put_page(xa->page_pool, page, napi_direct);
		} else {
			/* Hopefully stack show who to blame for late return */
			WARN_ONCE(1, "page_pool gone mem.id=%d", mem->id);
			trace_mem_return_failed(mem, page);
			put_page(page);
		}
		rcu_read_unlock();
		break;
	case MEM_TYPE_PAGE_SHARED:
		page_frag_free(data);
		break;
	case MEM_TYPE_PAGE_ORDER0:
		page = virt_to_page(data); /* Assumes order0 page*/
		put_page(page);
		break;
	case MEM_TYPE_ZERO_COPY:
		/* NB! Only valid from an xdp_buff! */
		rcu_read_lock();
		/* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */
		xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
		xa->zc_alloc->free(xa->zc_alloc, handle);
		rcu_read_unlock();
	default:
		/* Not possible, checked in xdp_rxq_info_reg_mem_model() */
		break;
	}
}

void xdp_return_frame(struct xdp_frame *xdpf)
{
	__xdp_return(xdpf->data, &xdpf->mem, false, 0);
}
EXPORT_SYMBOL_GPL(xdp_return_frame);

void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
{
	__xdp_return(xdpf->data, &xdpf->mem, true, 0);
}
EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);

void xdp_return_buff(struct xdp_buff *xdp)
{
	__xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle);
}
EXPORT_SYMBOL_GPL(xdp_return_buff);

/* Only called for MEM_TYPE_PAGE_POOL see xdp.h */
void __xdp_release_frame(void *data, struct xdp_mem_info *mem)
{
	struct xdp_mem_allocator *xa;
	struct page *page;

	rcu_read_lock();
	xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
	page = virt_to_head_page(data);
	if (xa)
		page_pool_release_page(xa->page_pool, page);
	rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(__xdp_release_frame);

int xdp_attachment_query(struct xdp_attachment_info *info,
			 struct netdev_bpf *bpf)
{
	bpf->prog_id = info->prog ? info->prog->aux->id : 0;
	bpf->prog_flags = info->prog ? info->flags : 0;
	return 0;
}
EXPORT_SYMBOL_GPL(xdp_attachment_query);

bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
			     struct netdev_bpf *bpf)
{
	if (info->prog && (bpf->flags ^ info->flags) & XDP_FLAGS_MODES) {
		NL_SET_ERR_MSG(bpf->extack,
			       "program loaded with different flags");
		return false;
	}
	return true;
}
EXPORT_SYMBOL_GPL(xdp_attachment_flags_ok);

void xdp_attachment_setup(struct xdp_attachment_info *info,
			  struct netdev_bpf *bpf)
{
	if (info->prog)
		bpf_prog_put(info->prog);
	info->prog = bpf->prog;
	info->flags = bpf->flags;
}
EXPORT_SYMBOL_GPL(xdp_attachment_setup);

struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)
{
	unsigned int metasize, totsize;
	void *addr, *data_to_copy;
	struct xdp_frame *xdpf;
	struct page *page;

	/* Clone into a MEM_TYPE_PAGE_ORDER0 xdp_frame. */
	metasize = xdp_data_meta_unsupported(xdp) ? 0 :
		   xdp->data - xdp->data_meta;
	totsize = xdp->data_end - xdp->data + metasize;

	if (sizeof(*xdpf) + totsize > PAGE_SIZE)
		return NULL;

	page = dev_alloc_page();
	if (!page)
		return NULL;

	addr = page_to_virt(page);
	xdpf = addr;
	memset(xdpf, 0, sizeof(*xdpf));

	addr += sizeof(*xdpf);
	data_to_copy = metasize ? xdp->data_meta : xdp->data;
	memcpy(addr, data_to_copy, totsize);

	xdpf->data = addr + metasize;
	xdpf->len = totsize - metasize;
	xdpf->headroom = 0;
	xdpf->metasize = metasize;
	xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;

	xdp_return_buff(xdp);
	return xdpf;
}
EXPORT_SYMBOL_GPL(xdp_convert_zc_to_xdp_frame);
Commit	Line	Data
ddc64d0a	1	// SPDX-License-Identifier: GPL-2.0-only
aecd67b6 JDB	2	/* net/core/xdp.c
	3	*
	4	* Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
aecd67b6	5	*/
05296620 JK	6	#include <linux/bpf.h>
05296620 JK	7	#include <linux/filter.h>
aecd67b6 JDB	8	#include <linux/types.h>
aecd67b6 JDB	9	#include <linux/mm.h>
05296620	10	#include <linux/netdevice.h>
8d5d8852 JDB	11	#include <linux/slab.h>
	12	#include <linux/idr.h>
	13	#include <linux/rhashtable.h>
57d0a1c1	14	#include <net/page_pool.h>
aecd67b6 JDB	15
aecd67b6 JDB	16	#include <net/xdp.h>
f033b688 JDB	17	#include <net/xdp_priv.h> /* struct xdp_mem_allocator */
f033b688 JDB	18	#include <trace/events/xdp.h>
aecd67b6 JDB	19
	20	#define REG_STATE_NEW 0x0
	21	#define REG_STATE_REGISTERED 0x1
	22	#define REG_STATE_UNREGISTERED 0x2
	23	#define REG_STATE_UNUSED 0x3
	24
8d5d8852 JDB	25	static DEFINE_IDA(mem_id_pool);
	26	static DEFINE_MUTEX(mem_id_lock);
	27	#define MEM_ID_MAX 0xFFFE
	28	#define MEM_ID_MIN 1
	29	static int mem_id_next = MEM_ID_MIN;
	30
	31	static bool mem_id_init; /* false */
	32	static struct rhashtable *mem_id_ht;
	33
8d5d8852 JDB	34	static u32 xdp_mem_id_hashfn(const void *data, u32 len, u32 seed)
	35	{
	36	const u32 *k = data;
	37	const u32 key = *k;
	38
	39	BUILD_BUG_ON(FIELD_SIZEOF(struct xdp_mem_allocator, mem.id)
	40	!= sizeof(u32));
	41
9f9a7077 N	42	/* Use cyclic increasing ID as direct hash key */
9f9a7077 N	43	return key;
8d5d8852 JDB	44	}
	45
	46	static int xdp_mem_id_cmp(struct rhashtable_compare_arg *arg,
	47	const void *ptr)
	48	{
	49	const struct xdp_mem_allocator *xa = ptr;
	50	u32 mem_id = (u32 )arg->key;
	51
	52	return xa->mem.id != mem_id;
	53	}
	54
	55	static const struct rhashtable_params mem_id_rht_params = {
	56	.nelem_hint = 64,
	57	.head_offset = offsetof(struct xdp_mem_allocator, node),
	58	.key_offset = offsetof(struct xdp_mem_allocator, mem.id),
	59	.key_len = FIELD_SIZEOF(struct xdp_mem_allocator, mem.id),
	60	.max_size = MEM_ID_MAX,
	61	.min_size = 8,
	62	.automatic_shrinking = true,
	63	.hashfn = xdp_mem_id_hashfn,
	64	.obj_cmpfn = xdp_mem_id_cmp,
	65	};
	66
	67	static void __xdp_mem_allocator_rcu_free(struct rcu_head *rcu)
	68	{
	69	struct xdp_mem_allocator *xa;
	70
	71	xa = container_of(rcu, struct xdp_mem_allocator, rcu);
	72
99c07c43 JDB	73	/* Allocator have indicated safe to remove before this is called */
	74	if (xa->mem.type == MEM_TYPE_PAGE_POOL)
	75	page_pool_free(xa->page_pool);
	76
8d5d8852 JDB	77	/* Allow this ID to be reused */
	78	ida_simple_remove(&mem_id_pool, xa->mem.id);
	79
8d5d8852 JDB	80	/* Poison memory */
	81	xa->mem.id = 0xFFFF;
	82	xa->mem.type = 0xF0F0;
	83	xa->allocator = (void *)0xDEAD9001;
	84
	85	kfree(xa);
	86	}
	87
d956a048	88	bool __mem_id_disconnect(int id, bool force)
99c07c43 JDB	89	{
	90	struct xdp_mem_allocator *xa;
	91	bool safe_to_remove = true;
	92
	93	mutex_lock(&mem_id_lock);
	94
	95	xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params);
	96	if (!xa) {
	97	mutex_unlock(&mem_id_lock);
	98	WARN(1, "Request remove non-existing id(%d), driver bug?", id);
	99	return true;
	100	}
d956a048	101	xa->disconnect_cnt++;
99c07c43 JDB	102
	103	/* Detects in-flight packet-pages for page_pool */
	104	if (xa->mem.type == MEM_TYPE_PAGE_POOL)
	105	safe_to_remove = page_pool_request_shutdown(xa->page_pool);
	106
f033b688	107	trace_mem_disconnect(xa, safe_to_remove, force);
d956a048 JDB	108
d956a048 JDB	109	if ((safe_to_remove \|\| force) &&
99c07c43 JDB	110	!rhashtable_remove_fast(mem_id_ht, &xa->node, mem_id_rht_params))
	111	call_rcu(&xa->rcu, __xdp_mem_allocator_rcu_free);
	112
	113	mutex_unlock(&mem_id_lock);
d956a048	114	return (safe_to_remove\|force);
99c07c43 JDB	115	}
	116
	117	#define DEFER_TIME (msecs_to_jiffies(1000))
d956a048 JDB	118	#define DEFER_WARN_INTERVAL (30 * HZ)
d956a048 JDB	119	#define DEFER_MAX_RETRIES 120
99c07c43 JDB	120
	121	static void mem_id_disconnect_defer_retry(struct work_struct *wq)
	122	{
	123	struct delayed_work *dwq = to_delayed_work(wq);
	124	struct xdp_mem_allocator xa = container_of(dwq, typeof(xa), defer_wq);
d956a048 JDB	125	bool force = false;
	126
	127	if (xa->disconnect_cnt > DEFER_MAX_RETRIES)
	128	force = true;
99c07c43	129
d956a048	130	if (__mem_id_disconnect(xa->mem.id, force))
99c07c43 JDB	131	return;
99c07c43 JDB	132
d956a048 JDB	133	/* Periodic warning */
	134	if (time_after_eq(jiffies, xa->defer_warn)) {
	135	int sec = (s32)((u32)jiffies - (u32)xa->defer_start) / HZ;
	136
	137	pr_warn("%s() stalled mem.id=%u shutdown %d attempts %d sec\n",
	138	__func__, xa->mem.id, xa->disconnect_cnt, sec);
	139	xa->defer_warn = jiffies + DEFER_WARN_INTERVAL;
	140	}
	141
99c07c43 JDB	142	/* Still not ready to be disconnected, retry later */
	143	schedule_delayed_work(&xa->defer_wq, DEFER_TIME);
	144	}
	145
dce5bd61	146	void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq)
8d5d8852 JDB	147	{
	148	struct xdp_mem_allocator *xa;
	149	int id = xdp_rxq->mem.id;
8d5d8852	150
dce5bd61 BT	151	if (xdp_rxq->reg_state != REG_STATE_REGISTERED) {
	152	WARN(1, "Missing register, driver bug");
	153	return;
	154	}
	155
	156	if (xdp_rxq->mem.type != MEM_TYPE_PAGE_POOL &&
	157	xdp_rxq->mem.type != MEM_TYPE_ZERO_COPY) {
	158	return;
	159	}
	160
8d5d8852 JDB	161	if (id == 0)
	162	return;
	163
d956a048	164	if (__mem_id_disconnect(id, false))
99c07c43 JDB	165	return;
	166
	167	/* Could not disconnect, defer new disconnect attempt to later */
8d5d8852 JDB	168	mutex_lock(&mem_id_lock);
8d5d8852 JDB	169
21b172ee	170	xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params);
99c07c43 JDB	171	if (!xa) {
	172	mutex_unlock(&mem_id_lock);
	173	return;
	174	}
d956a048 JDB	175	xa->defer_start = jiffies;
d956a048 JDB	176	xa->defer_warn = jiffies + DEFER_WARN_INTERVAL;
8d5d8852	177
99c07c43	178	INIT_DELAYED_WORK(&xa->defer_wq, mem_id_disconnect_defer_retry);
8d5d8852	179	mutex_unlock(&mem_id_lock);
99c07c43	180	schedule_delayed_work(&xa->defer_wq, DEFER_TIME);
8d5d8852	181	}
dce5bd61	182	EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg_mem_model);
8d5d8852	183
99c07c43 JDB	184	/* This unregister operation will also cleanup and destroy the
	185	* allocator. The page_pool_free() operation is first called when it's
	186	* safe to remove, possibly deferred to a workqueue.
	187	*/
aecd67b6 JDB	188	void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq)
	189	{
	190	/* Simplify driver cleanup code paths, allow unreg "unused" */
	191	if (xdp_rxq->reg_state == REG_STATE_UNUSED)
	192	return;
	193
	194	WARN(!(xdp_rxq->reg_state == REG_STATE_REGISTERED), "Driver BUG");
	195
dce5bd61	196	xdp_rxq_info_unreg_mem_model(xdp_rxq);
8d5d8852	197
aecd67b6 JDB	198	xdp_rxq->reg_state = REG_STATE_UNREGISTERED;
aecd67b6 JDB	199	xdp_rxq->dev = NULL;
8d5d8852 JDB	200
	201	/* Reset mem info to defaults */
	202	xdp_rxq->mem.id = 0;
	203	xdp_rxq->mem.type = 0;
aecd67b6 JDB	204	}
	205	EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg);
	206
	207	static void xdp_rxq_info_init(struct xdp_rxq_info *xdp_rxq)
	208	{
	209	memset(xdp_rxq, 0, sizeof(*xdp_rxq));
	210	}
	211
	212	/* Returns 0 on success, negative on failure */
	213	int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
	214	struct net_device *dev, u32 queue_index)
	215	{
	216	if (xdp_rxq->reg_state == REG_STATE_UNUSED) {
	217	WARN(1, "Driver promised not to register this");
	218	return -EINVAL;
	219	}
	220
	221	if (xdp_rxq->reg_state == REG_STATE_REGISTERED) {
	222	WARN(1, "Missing unregister, handled but fix driver");
	223	xdp_rxq_info_unreg(xdp_rxq);
	224	}
	225
	226	if (!dev) {
	227	WARN(1, "Missing net_device from driver");
	228	return -ENODEV;
	229	}
	230
	231	/* State either UNREGISTERED or NEW */
	232	xdp_rxq_info_init(xdp_rxq);
	233	xdp_rxq->dev = dev;
	234	xdp_rxq->queue_index = queue_index;
	235
	236	xdp_rxq->reg_state = REG_STATE_REGISTERED;
	237	return 0;
	238	}
	239	EXPORT_SYMBOL_GPL(xdp_rxq_info_reg);
	240
	241	void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq)
	242	{
	243	xdp_rxq->reg_state = REG_STATE_UNUSED;
	244	}
	245	EXPORT_SYMBOL_GPL(xdp_rxq_info_unused);
c0124f32 JDB	246
	247	bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq)
	248	{
	249	return (xdp_rxq->reg_state == REG_STATE_REGISTERED);
	250	}
	251	EXPORT_SYMBOL_GPL(xdp_rxq_info_is_reg);
5ab073ff	252
8d5d8852 JDB	253	static int __mem_id_init_hash_table(void)
	254	{
	255	struct rhashtable *rht;
	256	int ret;
	257
	258	if (unlikely(mem_id_init))
	259	return 0;
	260
	261	rht = kzalloc(sizeof(*rht), GFP_KERNEL);
	262	if (!rht)
	263	return -ENOMEM;
	264
	265	ret = rhashtable_init(rht, &mem_id_rht_params);
	266	if (ret < 0) {
	267	kfree(rht);
	268	return ret;
	269	}
	270	mem_id_ht = rht;
	271	smp_mb(); /* mutex lock should provide enough pairing */
	272	mem_id_init = true;
	273
	274	return 0;
	275	}
	276
	277	/* Allocate a cyclic ID that maps to allocator pointer.
	278	* See: https://www.kernel.org/doc/html/latest/core-api/idr.html
	279	*
	280	* Caller must lock mem_id_lock.
	281	*/
	282	static int __mem_id_cyclic_get(gfp_t gfp)
	283	{
	284	int retries = 1;
	285	int id;
	286
	287	again:
	288	id = ida_simple_get(&mem_id_pool, mem_id_next, MEM_ID_MAX, gfp);
	289	if (id < 0) {
	290	if (id == -ENOSPC) {
	291	/* Cyclic allocator, reset next id */
	292	if (retries--) {
	293	mem_id_next = MEM_ID_MIN;
	294	goto again;
	295	}
	296	}
	297	return id; /* errno */
	298	}
	299	mem_id_next = id + 1;
	300
	301	return id;
	302	}
	303
57d0a1c1 JDB	304	static bool __is_supported_mem_type(enum xdp_mem_type type)
	305	{
	306	if (type == MEM_TYPE_PAGE_POOL)
	307	return is_page_pool_compiled_in();
	308
	309	if (type >= MEM_TYPE_MAX)
	310	return false;
	311
	312	return true;
	313	}
	314
5ab073ff JDB	315	int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
	316	enum xdp_mem_type type, void *allocator)
	317	{
8d5d8852 JDB	318	struct xdp_mem_allocator *xdp_alloc;
	319	gfp_t gfp = GFP_KERNEL;
	320	int id, errno, ret;
	321	void *ptr;
	322
	323	if (xdp_rxq->reg_state != REG_STATE_REGISTERED) {
	324	WARN(1, "Missing register, driver bug");
	325	return -EFAULT;
	326	}
	327
57d0a1c1 JDB	328	if (!__is_supported_mem_type(type))
57d0a1c1 JDB	329	return -EOPNOTSUPP;
5ab073ff JDB	330
	331	xdp_rxq->mem.type = type;
	332
57d0a1c1	333	if (!allocator) {
02b55e56	334	if (type == MEM_TYPE_PAGE_POOL \|\| type == MEM_TYPE_ZERO_COPY)
57d0a1c1	335	return -EINVAL; /* Setup time check page_pool req */
8d5d8852	336	return 0;
57d0a1c1	337	}
8d5d8852 JDB	338
	339	/* Delay init of rhashtable to save memory if feature isn't used */
	340	if (!mem_id_init) {
	341	mutex_lock(&mem_id_lock);
	342	ret = __mem_id_init_hash_table();
	343	mutex_unlock(&mem_id_lock);
	344	if (ret < 0) {
	345	WARN_ON(1);
	346	return ret;
	347	}
	348	}
	349
	350	xdp_alloc = kzalloc(sizeof(*xdp_alloc), gfp);
	351	if (!xdp_alloc)
	352	return -ENOMEM;
	353
	354	mutex_lock(&mem_id_lock);
	355	id = __mem_id_cyclic_get(gfp);
	356	if (id < 0) {
	357	errno = id;
	358	goto err;
	359	}
	360	xdp_rxq->mem.id = id;
	361	xdp_alloc->mem = xdp_rxq->mem;
	362	xdp_alloc->allocator = allocator;
	363
	364	/* Insert allocator into ID lookup table */
	365	ptr = rhashtable_insert_slow(mem_id_ht, &id, &xdp_alloc->node);
	366	if (IS_ERR(ptr)) {
516a7593 JDB	367	ida_simple_remove(&mem_id_pool, xdp_rxq->mem.id);
516a7593 JDB	368	xdp_rxq->mem.id = 0;
8d5d8852 JDB	369	errno = PTR_ERR(ptr);
	370	goto err;
	371	}
	372
	373	mutex_unlock(&mem_id_lock);
5ab073ff	374
f033b688	375	trace_mem_connect(xdp_alloc, xdp_rxq);
5ab073ff	376	return 0;
8d5d8852 JDB	377	err:
	378	mutex_unlock(&mem_id_lock);
	379	kfree(xdp_alloc);
	380	return errno;
5ab073ff JDB	381	}
5ab073ff JDB	382	EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);
8d5d8852	383
389ab7f0 JDB	384	/* XDP RX runs under NAPI protection, and in different delivery error
	385	* scenarios (e.g. queue full), it is possible to return the xdp_frame
	386	* while still leveraging this protection. The @napi_direct boolian
	387	* is used for those calls sites. Thus, allowing for faster recycling
	388	* of xdp_frames/pages in those cases.
	389	*/
02b55e56 BT	390	static void __xdp_return(void data, struct xdp_mem_info mem, bool napi_direct,
02b55e56 BT	391	unsigned long handle)
8d5d8852	392	{
57d0a1c1 JDB	393	struct xdp_mem_allocator *xa;
	394	struct page *page;
	395
	396	switch (mem->type) {
	397	case MEM_TYPE_PAGE_POOL:
	398	rcu_read_lock();
	399	/* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */
	400	xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
	401	page = virt_to_head_page(data);
d956a048	402	if (likely(xa)) {
2539650f	403	napi_direct &= !xdp_return_frame_no_direct();
389ab7f0	404	page_pool_put_page(xa->page_pool, page, napi_direct);
2539650f	405	} else {
d956a048 JDB	406	/* Hopefully stack show who to blame for late return */
d956a048 JDB	407	WARN_ONCE(1, "page_pool gone mem.id=%d", mem->id);
f033b688	408	trace_mem_return_failed(mem, page);
57d0a1c1	409	put_page(page);
2539650f	410	}
57d0a1c1 JDB	411	rcu_read_unlock();
	412	break;
	413	case MEM_TYPE_PAGE_SHARED:
8d5d8852	414	page_frag_free(data);
57d0a1c1 JDB	415	break;
	416	case MEM_TYPE_PAGE_ORDER0:
	417	page = virt_to_page(data); /* Assumes order0 page*/
8d5d8852	418	put_page(page);
57d0a1c1	419	break;
02b55e56 BT	420	case MEM_TYPE_ZERO_COPY:
	421	/* NB! Only valid from an xdp_buff! */
	422	rcu_read_lock();
	423	/* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */
	424	xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
eb91e4d4	425	xa->zc_alloc->free(xa->zc_alloc, handle);
02b55e56	426	rcu_read_unlock();
57d0a1c1 JDB	427	default:
	428	/* Not possible, checked in xdp_rxq_info_reg_mem_model() */
	429	break;
8d5d8852 JDB	430	}
8d5d8852 JDB	431	}
c497176c BT	432
	433	void xdp_return_frame(struct xdp_frame *xdpf)
	434	{
02b55e56	435	__xdp_return(xdpf->data, &xdpf->mem, false, 0);
c497176c	436	}
8d5d8852	437	EXPORT_SYMBOL_GPL(xdp_return_frame);
c497176c	438
389ab7f0 JDB	439	void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
389ab7f0 JDB	440	{
02b55e56	441	__xdp_return(xdpf->data, &xdpf->mem, true, 0);
389ab7f0 JDB	442	}
	443	EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
	444
c497176c BT	445	void xdp_return_buff(struct xdp_buff *xdp)
c497176c BT	446	{
02b55e56	447	__xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle);
c497176c BT	448	}
c497176c BT	449	EXPORT_SYMBOL_GPL(xdp_return_buff);
05296620	450
6bf071bf JDB	451	/* Only called for MEM_TYPE_PAGE_POOL see xdp.h */
	452	void __xdp_release_frame(void data, struct xdp_mem_info mem)
	453	{
	454	struct xdp_mem_allocator *xa;
	455	struct page *page;
	456
	457	rcu_read_lock();
	458	xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
	459	page = virt_to_head_page(data);
	460	if (xa)
	461	page_pool_release_page(xa->page_pool, page);
	462	rcu_read_unlock();
	463	}
	464	EXPORT_SYMBOL_GPL(__xdp_release_frame);
	465
05296620 JK	466	int xdp_attachment_query(struct xdp_attachment_info *info,
	467	struct netdev_bpf *bpf)
	468	{
	469	bpf->prog_id = info->prog ? info->prog->aux->id : 0;
	470	bpf->prog_flags = info->prog ? info->flags : 0;
	471	return 0;
	472	}
	473	EXPORT_SYMBOL_GPL(xdp_attachment_query);
	474
	475	bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
	476	struct netdev_bpf *bpf)
	477	{
	478	if (info->prog && (bpf->flags ^ info->flags) & XDP_FLAGS_MODES) {
	479	NL_SET_ERR_MSG(bpf->extack,
	480	"program loaded with different flags");
	481	return false;
	482	}
	483	return true;
	484	}
	485	EXPORT_SYMBOL_GPL(xdp_attachment_flags_ok);
	486
	487	void xdp_attachment_setup(struct xdp_attachment_info *info,
	488	struct netdev_bpf *bpf)
	489	{
	490	if (info->prog)
	491	bpf_prog_put(info->prog);
	492	info->prog = bpf->prog;
	493	info->flags = bpf->flags;
	494	}
	495	EXPORT_SYMBOL_GPL(xdp_attachment_setup);
b0d1beef BT	496
	497	struct xdp_frame xdp_convert_zc_to_xdp_frame(struct xdp_buff xdp)
	498	{
72962167	499	unsigned int metasize, totsize;
b0d1beef BT	500	void addr, data_to_copy;
	501	struct xdp_frame *xdpf;
	502	struct page *page;
	503
	504	/* Clone into a MEM_TYPE_PAGE_ORDER0 xdp_frame. */
	505	metasize = xdp_data_meta_unsupported(xdp) ? 0 :
	506	xdp->data - xdp->data_meta;
b0d1beef BT	507	totsize = xdp->data_end - xdp->data + metasize;
	508
	509	if (sizeof(*xdpf) + totsize > PAGE_SIZE)
	510	return NULL;
	511
	512	page = dev_alloc_page();
	513	if (!page)
	514	return NULL;
	515
	516	addr = page_to_virt(page);
	517	xdpf = addr;
	518	memset(xdpf, 0, sizeof(*xdpf));
	519
	520	addr += sizeof(*xdpf);
	521	data_to_copy = metasize ? xdp->data_meta : xdp->data;
	522	memcpy(addr, data_to_copy, totsize);
	523
	524	xdpf->data = addr + metasize;
	525	xdpf->len = totsize - metasize;
	526	xdpf->headroom = 0;
	527	xdpf->metasize = metasize;
	528	xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;
	529
	530	xdp_return_buff(xdp);
	531	return xdpf;
	532	}
	533	EXPORT_SYMBOL_GPL(xdp_convert_zc_to_xdp_frame);