neighbour: Convert iteration to use hlist+macro

Remove all usage of the bare neighbour::next pointer,
replacing them with neighbour::hash and its for_each macro.

Signed-off-by: Gilad Naaman <gnaaman@drivenets.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20241107160444.2913124-5-gnaaman@drivenets.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/neighbour.h b/include/net/neighbour.h
index 4b9068c5e668174891792338c1e6bd8c88a24e12..94cf4f8c118fbd92252a54cd83ba5456e69af71c 100644 (file)
--- a/include/net/neighbour.h
+++ b/include/net/neighbour.h
@@ -311,12 +311,9 @@ static inline struct neighbour *___neigh_lookup_noref(
        u32 hash_val;
 
        hash_val = hash(pkey, dev, nht->hash_rnd) >> (32 - nht->hash_shift);
-       for (n = rcu_dereference(nht->hash_buckets[hash_val]);
-            n != NULL;
-            n = rcu_dereference(n->next)) {
+       neigh_for_each_in_bucket_rcu(n, &nht->hash_heads[hash_val])
                if (n->dev == dev && key_eq(n, pkey))
                        return n;
-       }
 
        return NULL;
 }

diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 3485d6b3ba99a71777f70b407626fa18079b7571..f99354d768c24e681ab7400a5d7d763a3227384d 100644 (file)
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -387,11 +387,11 @@ static void neigh_flush_dev(struct neigh_table *tbl, struct net_device *dev,
                                        lockdep_is_held(&tbl->lock));
 
        for (i = 0; i < (1 << nht->hash_shift); i++) {
-               struct neighbour *n;
                struct neighbour __rcu **np = &nht->hash_buckets[i];
+               struct hlist_node *tmp;
+               struct neighbour *n;
 
-               while ((n = rcu_dereference_protected(*np,
-                                       lockdep_is_held(&tbl->lock))) != NULL) {
+               neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[i]) {
                        if (dev && n->dev != dev) {
                                np = &n->next;
                                continue;
@@ -587,18 +587,14 @@ static struct neigh_hash_table *neigh_hash_grow(struct neigh_table *tbl,
                return old_nht;
 
        for (i = 0; i < (1 << old_nht->hash_shift); i++) {
-               struct neighbour *n, *next;
+               struct hlist_node *tmp;
+               struct neighbour *n;
 
-               for (n = rcu_dereference_protected(old_nht->hash_buckets[i],
-                                                  lockdep_is_held(&tbl->lock));
-                    n != NULL;
-                    n = next) {
+               neigh_for_each_in_bucket_safe(n, tmp, &old_nht->hash_heads[i]) {
                        hash = tbl->hash(n->primary_key, n->dev,
                                         new_nht->hash_rnd);
 
                        hash >>= (32 - new_nht->hash_shift);
-                       next = rcu_dereference_protected(n->next,
-                                               lockdep_is_held(&tbl->lock));
 
                        rcu_assign_pointer(n->next,
                                           rcu_dereference_protected(
@@ -693,11 +689,7 @@ ___neigh_create(struct neigh_table *tbl, const void *pkey,
                goto out_tbl_unlock;
        }
 
-       for (n1 = rcu_dereference_protected(nht->hash_buckets[hash_val],
-                                           lockdep_is_held(&tbl->lock));
-            n1 != NULL;
-            n1 = rcu_dereference_protected(n1->next,
-                       lockdep_is_held(&tbl->lock))) {
+       neigh_for_each_in_bucket(n1, &nht->hash_heads[hash_val]) {
                if (dev == n1->dev && !memcmp(n1->primary_key, n->primary_key, key_len)) {
                        if (want_ref)
                                neigh_hold(n1);
@@ -949,10 +941,11 @@ static void neigh_connect(struct neighbour *neigh)
 static void neigh_periodic_work(struct work_struct *work)
 {
        struct neigh_table *tbl = container_of(work, struct neigh_table, gc_work.work);
-       struct neighbour *n;
+       struct neigh_hash_table *nht;
        struct neighbour __rcu **np;
+       struct hlist_node *tmp;
+       struct neighbour *n;
        unsigned int i;
-       struct neigh_hash_table *nht;
 
        NEIGH_CACHE_STAT_INC(tbl, periodic_gc_runs);
 
@@ -979,8 +972,7 @@ static void neigh_periodic_work(struct work_struct *work)
        for (i = 0 ; i < (1 << nht->hash_shift); i++) {
                np = &nht->hash_buckets[i];
 
-               while ((n = rcu_dereference_protected(*np,
-                               lockdep_is_held(&tbl->lock))) != NULL) {
+               neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[i]) {
                        unsigned int state;
 
                        write_lock(&n->lock);
@@ -2730,9 +2722,8 @@ static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
        for (h = s_h; h < (1 << nht->hash_shift); h++) {
                if (h > s_h)
                        s_idx = 0;
-               for (n = rcu_dereference(nht->hash_buckets[h]), idx = 0;
-                    n != NULL;
-                    n = rcu_dereference(n->next)) {
+               idx = 0;
+               neigh_for_each_in_bucket_rcu(n, &nht->hash_heads[h]) {
                        if (idx < s_idx || !net_eq(dev_net(n->dev), net))
                                goto next;
                        if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
@@ -3099,9 +3090,7 @@ void neigh_for_each(struct neigh_table *tbl, void (*cb)(struct neighbour *, void
        for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
                struct neighbour *n;
 
-               for (n = rcu_dereference(nht->hash_buckets[chain]);
-                    n != NULL;
-                    n = rcu_dereference(n->next))
+               neigh_for_each_in_bucket(n, &nht->hash_heads[chain])
                        cb(n, cookie);
        }
        read_unlock_bh(&tbl->lock);
@@ -3113,18 +3102,18 @@ EXPORT_SYMBOL(neigh_for_each);
 void __neigh_for_each_release(struct neigh_table *tbl,
                              int (*cb)(struct neighbour *))
 {
-       int chain;
        struct neigh_hash_table *nht;
+       int chain;
 
        nht = rcu_dereference_protected(tbl->nht,
                                        lockdep_is_held(&tbl->lock));
        for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
-               struct neighbour *n;
                struct neighbour __rcu **np;
+               struct hlist_node *tmp;
+               struct neighbour *n;
 
                np = &nht->hash_buckets[chain];
-               while ((n = rcu_dereference_protected(*np,
-                                       lockdep_is_held(&tbl->lock))) != NULL) {
+               neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[chain]) {
                        int release;
 
                        write_lock(&n->lock);