1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/mutex.h>
4 #include <linux/netdevice.h>
5 #include <linux/xarray.h>
6 #include <net/net_debug.h>
7 #include <net/page_pool/types.h>
8 #include <net/page_pool/helpers.h>
11 #include "page_pool_priv.h"
12 #include "netdev-genl-gen.h"
14 static DEFINE_XARRAY_FLAGS(page_pools
, XA_FLAGS_ALLOC1
);
15 /* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user.
16 * Ordering: inside rtnl_lock
18 static DEFINE_MUTEX(page_pools_lock
);
20 /* Page pools are only reachable from user space (via netlink) if they are
21 * linked to a netdev at creation time. Following page pool "visibility"
22 * states are possible:
24 * - user.list: linked to real netdev, netdev: real netdev
25 * - orphaned - real netdev has disappeared
26 * - user.list: linked to lo, netdev: lo
27 * - invisible - either (a) created without netdev linking, (b) unlisted due
28 * to error, or (c) the entire namespace which owned this pool disappeared
29 * - user.list: unhashed, netdev: unknown
32 typedef int (*pp_nl_fill_cb
)(struct sk_buff
*rsp
, const struct page_pool
*pool
,
33 const struct genl_info
*info
);
36 netdev_nl_page_pool_get_do(struct genl_info
*info
, u32 id
, pp_nl_fill_cb fill
)
38 struct page_pool
*pool
;
42 mutex_lock(&page_pools_lock
);
43 pool
= xa_load(&page_pools
, id
);
44 if (!pool
|| hlist_unhashed(&pool
->user
.list
) ||
45 !net_eq(dev_net(pool
->slow
.netdev
), genl_info_net(info
))) {
50 rsp
= genlmsg_new(GENLMSG_DEFAULT_SIZE
, GFP_KERNEL
);
56 err
= fill(rsp
, pool
, info
);
60 mutex_unlock(&page_pools_lock
);
62 return genlmsg_reply(rsp
, info
);
67 mutex_unlock(&page_pools_lock
);
71 struct page_pool_dump_cb
{
72 unsigned long ifindex
;
77 netdev_nl_page_pool_get_dump(struct sk_buff
*skb
, struct netlink_callback
*cb
,
80 struct page_pool_dump_cb
*state
= (void *)cb
->ctx
;
81 const struct genl_info
*info
= genl_info_dump(cb
);
82 struct net
*net
= sock_net(skb
->sk
);
83 struct net_device
*netdev
;
84 struct page_pool
*pool
;
88 mutex_lock(&page_pools_lock
);
89 for_each_netdev_dump(net
, netdev
, state
->ifindex
) {
90 hlist_for_each_entry(pool
, &netdev
->page_pools
, user
.list
) {
91 if (state
->pp_id
&& state
->pp_id
< pool
->user
.id
)
94 state
->pp_id
= pool
->user
.id
;
95 err
= fill(skb
, pool
, info
);
103 mutex_unlock(&page_pools_lock
);
110 page_pool_nl_stats_fill(struct sk_buff
*rsp
, const struct page_pool
*pool
,
111 const struct genl_info
*info
)
113 #ifdef CONFIG_PAGE_POOL_STATS
114 struct page_pool_stats stats
= {};
118 if (!page_pool_get_stats(pool
, &stats
))
121 hdr
= genlmsg_iput(rsp
, info
);
125 nest
= nla_nest_start(rsp
, NETDEV_A_PAGE_POOL_STATS_INFO
);
127 if (nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_ID
, pool
->user
.id
) ||
128 (pool
->slow
.netdev
->ifindex
!= LOOPBACK_IFINDEX
&&
129 nla_put_u32(rsp
, NETDEV_A_PAGE_POOL_IFINDEX
,
130 pool
->slow
.netdev
->ifindex
)))
131 goto err_cancel_nest
;
133 nla_nest_end(rsp
, nest
);
135 if (nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST
,
136 stats
.alloc_stats
.fast
) ||
137 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW
,
138 stats
.alloc_stats
.slow
) ||
139 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER
,
140 stats
.alloc_stats
.slow_high_order
) ||
141 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY
,
142 stats
.alloc_stats
.empty
) ||
143 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL
,
144 stats
.alloc_stats
.refill
) ||
145 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE
,
146 stats
.alloc_stats
.waive
) ||
147 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED
,
148 stats
.recycle_stats
.cached
) ||
149 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL
,
150 stats
.recycle_stats
.cache_full
) ||
151 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING
,
152 stats
.recycle_stats
.ring
) ||
153 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL
,
154 stats
.recycle_stats
.ring_full
) ||
155 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT
,
156 stats
.recycle_stats
.released_refcnt
))
159 genlmsg_end(rsp
, hdr
);
163 nla_nest_cancel(rsp
, nest
);
165 genlmsg_cancel(rsp
, hdr
);
168 GENL_SET_ERR_MSG(info
, "kernel built without CONFIG_PAGE_POOL_STATS");
173 int netdev_nl_page_pool_stats_get_doit(struct sk_buff
*skb
,
174 struct genl_info
*info
)
176 struct nlattr
*tb
[ARRAY_SIZE(netdev_page_pool_info_nl_policy
)];
181 if (GENL_REQ_ATTR_CHECK(info
, NETDEV_A_PAGE_POOL_STATS_INFO
))
184 nest
= info
->attrs
[NETDEV_A_PAGE_POOL_STATS_INFO
];
185 err
= nla_parse_nested(tb
, ARRAY_SIZE(tb
) - 1, nest
,
186 netdev_page_pool_info_nl_policy
,
191 if (NL_REQ_ATTR_CHECK(info
->extack
, nest
, tb
, NETDEV_A_PAGE_POOL_ID
))
193 if (tb
[NETDEV_A_PAGE_POOL_IFINDEX
]) {
194 NL_SET_ERR_MSG_ATTR(info
->extack
,
195 tb
[NETDEV_A_PAGE_POOL_IFINDEX
],
196 "selecting by ifindex not supported");
200 id
= nla_get_uint(tb
[NETDEV_A_PAGE_POOL_ID
]);
202 return netdev_nl_page_pool_get_do(info
, id
, page_pool_nl_stats_fill
);
205 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff
*skb
,
206 struct netlink_callback
*cb
)
208 return netdev_nl_page_pool_get_dump(skb
, cb
, page_pool_nl_stats_fill
);
212 page_pool_nl_fill(struct sk_buff
*rsp
, const struct page_pool
*pool
,
213 const struct genl_info
*info
)
215 size_t inflight
, refsz
;
218 hdr
= genlmsg_iput(rsp
, info
);
222 if (nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_ID
, pool
->user
.id
))
225 if (pool
->slow
.netdev
->ifindex
!= LOOPBACK_IFINDEX
&&
226 nla_put_u32(rsp
, NETDEV_A_PAGE_POOL_IFINDEX
,
227 pool
->slow
.netdev
->ifindex
))
229 if (pool
->user
.napi_id
&&
230 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_NAPI_ID
, pool
->user
.napi_id
))
233 inflight
= page_pool_inflight(pool
, false);
234 refsz
= PAGE_SIZE
<< pool
->p
.order
;
235 if (nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_INFLIGHT
, inflight
) ||
236 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_INFLIGHT_MEM
,
239 if (pool
->user
.detach_time
&&
240 nla_put_uint(rsp
, NETDEV_A_PAGE_POOL_DETACH_TIME
,
241 pool
->user
.detach_time
))
244 genlmsg_end(rsp
, hdr
);
248 genlmsg_cancel(rsp
, hdr
);
252 static void netdev_nl_page_pool_event(const struct page_pool
*pool
, u32 cmd
)
254 struct genl_info info
;
258 lockdep_assert_held(&page_pools_lock
);
260 /* 'invisible' page pools don't matter */
261 if (hlist_unhashed(&pool
->user
.list
))
263 net
= dev_net(pool
->slow
.netdev
);
265 if (!genl_has_listeners(&netdev_nl_family
, net
, NETDEV_NLGRP_PAGE_POOL
))
268 genl_info_init_ntf(&info
, &netdev_nl_family
, cmd
);
270 ntf
= genlmsg_new(GENLMSG_DEFAULT_SIZE
, GFP_KERNEL
);
274 if (page_pool_nl_fill(ntf
, pool
, &info
)) {
279 genlmsg_multicast_netns(&netdev_nl_family
, net
, ntf
,
280 0, NETDEV_NLGRP_PAGE_POOL
, GFP_KERNEL
);
283 int netdev_nl_page_pool_get_doit(struct sk_buff
*skb
, struct genl_info
*info
)
287 if (GENL_REQ_ATTR_CHECK(info
, NETDEV_A_PAGE_POOL_ID
))
290 id
= nla_get_uint(info
->attrs
[NETDEV_A_PAGE_POOL_ID
]);
292 return netdev_nl_page_pool_get_do(info
, id
, page_pool_nl_fill
);
295 int netdev_nl_page_pool_get_dumpit(struct sk_buff
*skb
,
296 struct netlink_callback
*cb
)
298 return netdev_nl_page_pool_get_dump(skb
, cb
, page_pool_nl_fill
);
301 int page_pool_list(struct page_pool
*pool
)
303 static u32 id_alloc_next
;
306 mutex_lock(&page_pools_lock
);
307 err
= xa_alloc_cyclic(&page_pools
, &pool
->user
.id
, pool
, xa_limit_32b
,
308 &id_alloc_next
, GFP_KERNEL
);
312 INIT_HLIST_NODE(&pool
->user
.list
);
313 if (pool
->slow
.netdev
) {
314 hlist_add_head(&pool
->user
.list
,
315 &pool
->slow
.netdev
->page_pools
);
316 pool
->user
.napi_id
= pool
->p
.napi
? pool
->p
.napi
->napi_id
: 0;
318 netdev_nl_page_pool_event(pool
, NETDEV_CMD_PAGE_POOL_ADD_NTF
);
321 mutex_unlock(&page_pools_lock
);
325 mutex_unlock(&page_pools_lock
);
329 void page_pool_detached(struct page_pool
*pool
)
331 mutex_lock(&page_pools_lock
);
332 pool
->user
.detach_time
= ktime_get_boottime_seconds();
333 netdev_nl_page_pool_event(pool
, NETDEV_CMD_PAGE_POOL_CHANGE_NTF
);
334 mutex_unlock(&page_pools_lock
);
337 void page_pool_unlist(struct page_pool
*pool
)
339 mutex_lock(&page_pools_lock
);
340 netdev_nl_page_pool_event(pool
, NETDEV_CMD_PAGE_POOL_DEL_NTF
);
341 xa_erase(&page_pools
, pool
->user
.id
);
342 if (!hlist_unhashed(&pool
->user
.list
))
343 hlist_del(&pool
->user
.list
);
344 mutex_unlock(&page_pools_lock
);
347 static void page_pool_unreg_netdev_wipe(struct net_device
*netdev
)
349 struct page_pool
*pool
;
350 struct hlist_node
*n
;
352 mutex_lock(&page_pools_lock
);
353 hlist_for_each_entry_safe(pool
, n
, &netdev
->page_pools
, user
.list
) {
354 hlist_del_init(&pool
->user
.list
);
355 pool
->slow
.netdev
= NET_PTR_POISON
;
357 mutex_unlock(&page_pools_lock
);
360 static void page_pool_unreg_netdev(struct net_device
*netdev
)
362 struct page_pool
*pool
, *last
;
363 struct net_device
*lo
;
365 lo
= dev_net(netdev
)->loopback_dev
;
367 mutex_lock(&page_pools_lock
);
369 hlist_for_each_entry(pool
, &netdev
->page_pools
, user
.list
) {
370 pool
->slow
.netdev
= lo
;
371 netdev_nl_page_pool_event(pool
,
372 NETDEV_CMD_PAGE_POOL_CHANGE_NTF
);
376 hlist_splice_init(&netdev
->page_pools
, &last
->user
.list
,
378 mutex_unlock(&page_pools_lock
);
382 page_pool_netdevice_event(struct notifier_block
*nb
,
383 unsigned long event
, void *ptr
)
385 struct net_device
*netdev
= netdev_notifier_info_to_dev(ptr
);
387 if (event
!= NETDEV_UNREGISTER
)
390 if (hlist_empty(&netdev
->page_pools
))
393 if (netdev
->ifindex
!= LOOPBACK_IFINDEX
)
394 page_pool_unreg_netdev(netdev
);
396 page_pool_unreg_netdev_wipe(netdev
);
400 static struct notifier_block page_pool_netdevice_nb
= {
401 .notifier_call
= page_pool_netdevice_event
,
404 static int __init
page_pool_user_init(void)
406 return register_netdevice_notifier(&page_pool_netdevice_nb
);
409 subsys_initcall(page_pool_user_init
);