netkit: Add netkit notifier to check for unregistering devices

Add a netdevice notifier in netkit to watch for NETDEV_UNREGISTER events.
If the target device is indeed NETREG_UNREGISTERING and previously leased
a queue to a netkit device, then collect the related netkit devices and
batch-unregister_netdevice_many() them.

If this were not done, then the netkit device would hold a reference on
the physical device preventing it from going away. However, in case of
both io_uring zero-copy as well as AF_XDP this situation is handled
gracefully and the allocated resources are torn down.

In the case where mentioned infra is used through netkit, the applications
have a reference on netkit, and netkit in turn holds a reference on the
physical device. In order to have netkit release the reference on the
physical device, we need such watcher to then unregister the netkit ones.

This is generally quite similar to the dependency handling in case of
tunnels (e.g. vxlan bound to a underlying netdev) where the tunnel device
gets removed along with the physical device.

  # ip a
  [...]
  4: enp10s0f0np0: <BROADCAST,MULTICAST> mtu 1500 qdisc mq state DOWN group default qlen 1000
      link/ether e8:eb:d3:a3:43:f6 brd ff:ff:ff:ff:ff:ff
      inet 10.0.0.2/24 scope global enp10s0f0np0
         valid_lft forever preferred_lft forever
  [...]
  8: nk@NONE: <BROADCAST,MULTICAST,NOARP> mtu 1500 qdisc noop state DOWN group default qlen 1000
      link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff
  [...]

  # rmmod mlx5_ib
  # rmmod mlx5_core
  [...]
  [  309.261822] mlx5_core 0000:0a:00.0 mlx5_0: Port: 1 Link DOWN
  [  344.235236] mlx5_core 0000:0a:00.1: E-Switch: Unload vfs: mode(LEGACY), nvfs(0), necvfs(0), active vports(0)
  [  344.246948] mlx5_core 0000:0a:00.1: E-Switch: Disable: mode(LEGACY), nvfs(0), necvfs(0), active vports(0)
  [  344.463754] mlx5_core 0000:0a:00.1: E-Switch: Disable: mode(LEGACY), nvfs(0), necvfs(0), active vports(0)
  [  344.770155] mlx5_core 0000:0a:00.1: E-Switch: cleanup
  [...]

  # ip a
  [...]
  [ both enp10s0f0np0 and nk gone ]
  [...]

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Co-developed-by: David Wei <dw@davidwei.uk>
Signed-off-by: David Wei <dw@davidwei.uk>
Reviewed-by: Nikolay Aleksandrov <razor@blackwall.org>
Link: https://patch.msgid.link/20260402231031.447597-13-daniel@iogearbox.net
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/netkit.c b/drivers/net/netkit.c
index b22bd0b6508aa08aa161b4c11ce04cf956dc972d..1ec21aef348fffc78b6d661d92feb4859a3b4579 100644 (file)
--- a/drivers/net/netkit.c
+++ b/drivers/net/netkit.c
@@ -983,7 +983,15 @@ static void netkit_del_link(struct net_device *dev, struct list_head *head)
        if (peer) {
                nk = netkit_priv(peer);
                RCU_INIT_POINTER(nk->peer, NULL);
-               unregister_netdevice_queue(peer, head);
+               /* Guard against the peer already being in an unregister
+                * list (e.g. same-namespace teardown where the peer is
+                * in the caller's dev_kill_list). list_move_tail() on an
+                * already-queued device would otherwise corrupt that
+                * list's iteration. This situation can occur via netkit
+                * notifier, hence guard against this scenario.
+                */
+               if (!unregister_netdevice_queued(peer))
+                       unregister_netdevice_queue(peer, head);
        }
 }
 
@@ -1051,6 +1059,50 @@ static int netkit_change_link(struct net_device *dev, struct nlattr *tb[],
        return 0;
 }
 
+static void netkit_check_lease_unregister(struct net_device *dev)
+{
+       LIST_HEAD(list_kill);
+       u32 q_idx;
+
+       if (READ_ONCE(dev->reg_state) != NETREG_UNREGISTERING ||
+           !dev->dev.parent)
+               return;
+
+       netdev_lock_ops(dev);
+       for (q_idx = 0; q_idx < dev->real_num_rx_queues; q_idx++) {
+               struct net_device *tmp = dev;
+               struct netdev_rx_queue *rxq;
+               u32 tmp_q_idx = q_idx;
+
+               rxq = __netif_get_rx_queue_lease(&tmp, &tmp_q_idx,
+                                                NETIF_PHYS_TO_VIRT);
+               if (rxq && tmp != dev &&
+                   tmp->netdev_ops == &netkit_netdev_ops) {
+                       /* A single phys device can have multiple queues leased
+                        * to one netkit device. We can only queue that netkit
+                        * device once to the list_kill. Queues of that phys
+                        * device can be leased with different individual netkit
+                        * devices, hence we batch via list_kill.
+                        */
+                       if (unregister_netdevice_queued(tmp))
+                               continue;
+                       netkit_del_link(tmp, &list_kill);
+               }
+       }
+       netdev_unlock_ops(dev);
+       unregister_netdevice_many(&list_kill);
+}
+
+static int netkit_notifier(struct notifier_block *this,
+                          unsigned long event, void *ptr)
+{
+       struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+
+       if (event == NETDEV_UNREGISTER)
+               netkit_check_lease_unregister(dev);
+       return NOTIFY_DONE;
+}
+
 static size_t netkit_get_size(const struct net_device *dev)
 {
        return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */
@@ -1127,18 +1179,31 @@ static struct rtnl_link_ops netkit_link_ops = {
        .maxtype        = IFLA_NETKIT_MAX,
 };
 
+static struct notifier_block netkit_netdev_notifier = {
+       .notifier_call  = netkit_notifier,
+};
+
 static __init int netkit_mod_init(void)
 {
+       int ret;
+
        BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT ||
                     (int)NETKIT_PASS != (int)TCX_PASS ||
                     (int)NETKIT_DROP != (int)TCX_DROP ||
                     (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
 
-       return rtnl_link_register(&netkit_link_ops);
+       ret = rtnl_link_register(&netkit_link_ops);
+       if (ret)
+               return ret;
+       ret = register_netdevice_notifier(&netkit_netdev_notifier);
+       if (ret)
+               rtnl_link_unregister(&netkit_link_ops);
+       return ret;
 }
 
 static __exit void netkit_mod_exit(void)
 {
+       unregister_netdevice_notifier(&netkit_netdev_notifier);
        rtnl_link_unregister(&netkit_link_ops);
 }
 

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index e8aa9cc4075d4d6496771a9550c26483fd4efd9e..47417b2d48a4d1a1eef3bb4d4afde5aa10e58631 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -3420,6 +3420,8 @@ static inline int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
 int register_netdevice(struct net_device *dev);
 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
 void unregister_netdevice_many(struct list_head *head);
+bool unregister_netdevice_queued(const struct net_device *dev);
+
 static inline void unregister_netdevice(struct net_device *dev)
 {
        unregister_netdevice_queue(dev, NULL);

diff --git a/net/core/dev.c b/net/core/dev.c
index 2df8a2a5ecf5b9f56810b9206eed5a7d9b7d4730..e7bc95cbd1faa7f25009d51945ed4405d52fffad 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -12384,6 +12384,12 @@ static void netif_close_many_and_unlock_cond(struct list_head *close_head)
 #endif
 }
 
+bool unregister_netdevice_queued(const struct net_device *dev)
+{
+       ASSERT_RTNL();
+       return !list_empty(&dev->unreg_list);
+}
+
 void unregister_netdevice_many_notify(struct list_head *head,
                                      u32 portid, const struct nlmsghdr *nlh)
 {