net: move ptype_all into net_hotdata

ptype_all is used in rx/tx fast paths.

Move it to net_hotdata for better cache locality.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240306160031.874438-5-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 6643452af5436c2fc806d7cdb230179f2debbe84..b18ac8072f1875ce3eb4b77e6d18df1ca563a0d6 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -5306,7 +5306,6 @@ static inline const char *netdev_reg_state(const struct net_device *dev)
 #define PTYPE_HASH_SIZE        (16)
 #define PTYPE_HASH_MASK        (PTYPE_HASH_SIZE - 1)
 
-extern struct list_head ptype_all __read_mostly;
 extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 
 extern struct net_device *blackhole_netdev;

diff --git a/include/net/hotdata.h b/include/net/hotdata.h
index 149e56528537d8ed3365e46d6dc96e39c73a733a..d462cb8f16bad459b439c566274c01a0f18a95d0 100644 (file)
--- a/include/net/hotdata.h
+++ b/include/net/hotdata.h
@@ -7,6 +7,7 @@
 /* Read mostly data used in network fast paths. */
 struct net_hotdata {
        struct list_head        offload_base;
+       struct list_head        ptype_all;
        int                     gro_normal_batch;
        int                     netdev_budget;
        int                     netdev_budget_usecs;

diff --git a/net/core/dev.c b/net/core/dev.c
index f2f44303c03517dc8c364f18da8c33a17eb9d10b..e1493e5584070009104b15ee4f0c8e36bf61c60c 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -161,7 +161,6 @@
 
 static DEFINE_SPINLOCK(ptype_lock);
 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
-struct list_head ptype_all __read_mostly;      /* Taps */
 
 static int netif_rx_internal(struct sk_buff *skb);
 static int call_netdevice_notifiers_extack(unsigned long val,
@@ -540,7 +539,7 @@ static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 static inline struct list_head *ptype_head(const struct packet_type *pt)
 {
        if (pt->type == htons(ETH_P_ALL))
-               return pt->dev ? &pt->dev->ptype_all : &ptype_all;
+               return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all;
        else
                return pt->dev ? &pt->dev->ptype_specific :
                                 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
@@ -2226,7 +2225,8 @@ static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
  */
 bool dev_nit_active(struct net_device *dev)
 {
-       return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
+       return !list_empty(&net_hotdata.ptype_all) ||
+              !list_empty(&dev->ptype_all);
 }
 EXPORT_SYMBOL_GPL(dev_nit_active);
 
@@ -2237,10 +2237,9 @@ EXPORT_SYMBOL_GPL(dev_nit_active);
 
 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
 {
-       struct packet_type *ptype;
+       struct list_head *ptype_list = &net_hotdata.ptype_all;
+       struct packet_type *ptype, *pt_prev = NULL;
        struct sk_buff *skb2 = NULL;
-       struct packet_type *pt_prev = NULL;
-       struct list_head *ptype_list = &ptype_all;
 
        rcu_read_lock();
 again:
@@ -2286,7 +2285,7 @@ again:
                pt_prev = ptype;
        }
 
-       if (ptype_list == &ptype_all) {
+       if (ptype_list == &net_hotdata.ptype_all) {
                ptype_list = &dev->ptype_all;
                goto again;
        }
@@ -5387,7 +5386,7 @@ another_round:
        if (pfmemalloc)
                goto skip_taps;
 
-       list_for_each_entry_rcu(ptype, &ptype_all, list) {
+       list_for_each_entry_rcu(ptype, &net_hotdata.ptype_all, list) {
                if (pt_prev)
                        ret = deliver_skb(skb, pt_prev, orig_dev);
                pt_prev = ptype;
@@ -11749,7 +11748,6 @@ static int __init net_dev_init(void)
        if (netdev_kobject_init())
                goto out;
 
-       INIT_LIST_HEAD(&ptype_all);
        for (i = 0; i < PTYPE_HASH_SIZE; i++)
                INIT_LIST_HEAD(&ptype_base[i]);
 

diff --git a/net/core/hotdata.c b/net/core/hotdata.c
index 087c4c84987df07f11a87112a778a5cac608a654..29fcfe89fd9a697120f826dbe2fd36a1617581a1 100644 (file)
--- a/net/core/hotdata.c
+++ b/net/core/hotdata.c
@@ -7,6 +7,7 @@
 
 struct net_hotdata net_hotdata __cacheline_aligned = {
        .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
+       .ptype_all = LIST_HEAD_INIT(net_hotdata.ptype_all),
        .gro_normal_batch = 8,
 
        .netdev_budget = 300,

diff --git a/net/core/net-procfs.c b/net/core/net-procfs.c
index 2e4e96d30ee1a7a51e49587378aab47aed1290da..a97eceb84e61ec347ad132ff0f22c8ce82f12e90 100644 (file)
--- a/net/core/net-procfs.c
+++ b/net/core/net-procfs.c
@@ -3,6 +3,7 @@
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <net/wext.h>
+#include <net/hotdata.h>
 
 #include "dev.h"
 
@@ -183,7 +184,7 @@ static void *ptype_get_idx(struct seq_file *seq, loff_t pos)
                }
        }
 
-       list_for_each_entry_rcu(pt, &ptype_all, list) {
+       list_for_each_entry_rcu(pt, &net_hotdata.ptype_all, list) {
                if (i == pos)
                        return pt;
                ++i;
@@ -231,13 +232,13 @@ static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
                        }
                }
 
-               nxt = ptype_all.next;
+               nxt = net_hotdata.ptype_all.next;
                goto ptype_all;
        }
 
        if (pt->type == htons(ETH_P_ALL)) {
 ptype_all:
-               if (nxt != &ptype_all)
+               if (nxt != &net_hotdata.ptype_all)
                        goto found;
                hash = 0;
                nxt = ptype_base[0].next;