bpf: Convert percpu_freelist.c to rqspinlock

Convert the percpu_freelist.c code to use rqspinlock, and remove the
extralist fallback and trylock-based acquisitions to avoid deadlocks.

Key thing to note is the retained while (true) loop to search through
other CPUs when failing to push a node due to locking errors. This
retains the behavior of the old code, where it would keep trying until
it would be able to successfully push the node back into the freelist of
a CPU.

Technically, we should start iteration for this loop from
raw_smp_processor_id() + 1, but to avoid hitting the edge of nr_cpus,
we skip execution in the loop body instead.

Closes: https://lore.kernel.org/bpf/CAPPBnEa1_pZ6W24+WwtcNFvTUHTHO7KUmzEbOcMqxp+m2o15qQ@mail.gmail.com
Closes: https://lore.kernel.org/bpf/CAPPBnEYm+9zduStsZaDnq93q1jPLqO-PiKX9jy0MuL8LCXmCrQ@mail.gmail.com
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20250316040541.108729-21-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c
index 034cf87b54e9f0777ed9584b4768d5bf9a803b59..632762b5729989633fc27f8abc2ad73b5de558a2 100644 (file)
--- a/kernel/bpf/percpu_freelist.c
+++ b/kernel/bpf/percpu_freelist.c
@@ -14,11 +14,9 @@ int pcpu_freelist_init(struct pcpu_freelist *s)
        for_each_possible_cpu(cpu) {
                struct pcpu_freelist_head *head = per_cpu_ptr(s->freelist, cpu);
 
-               raw_spin_lock_init(&head->lock);
+               raw_res_spin_lock_init(&head->lock);
                head->first = NULL;
        }
-       raw_spin_lock_init(&s->extralist.lock);
-       s->extralist.first = NULL;
        return 0;
 }
 
@@ -34,58 +32,39 @@ static inline void pcpu_freelist_push_node(struct pcpu_freelist_head *head,
        WRITE_ONCE(head->first, node);
 }
 
-static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head,
+static inline bool ___pcpu_freelist_push(struct pcpu_freelist_head *head,
                                         struct pcpu_freelist_node *node)
 {
-       raw_spin_lock(&head->lock);
-       pcpu_freelist_push_node(head, node);
-       raw_spin_unlock(&head->lock);
-}
-
-static inline bool pcpu_freelist_try_push_extra(struct pcpu_freelist *s,
-                                               struct pcpu_freelist_node *node)
-{
-       if (!raw_spin_trylock(&s->extralist.lock))
+       if (raw_res_spin_lock(&head->lock))
                return false;
-
-       pcpu_freelist_push_node(&s->extralist, node);
-       raw_spin_unlock(&s->extralist.lock);
+       pcpu_freelist_push_node(head, node);
+       raw_res_spin_unlock(&head->lock);
        return true;
 }
 
-static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s,
-                                            struct pcpu_freelist_node *node)
+void __pcpu_freelist_push(struct pcpu_freelist *s,
+                       struct pcpu_freelist_node *node)
 {
-       int cpu, orig_cpu;
+       struct pcpu_freelist_head *head;
+       int cpu;
 
-       orig_cpu = raw_smp_processor_id();
-       while (1) {
-               for_each_cpu_wrap(cpu, cpu_possible_mask, orig_cpu) {
-                       struct pcpu_freelist_head *head;
+       if (___pcpu_freelist_push(this_cpu_ptr(s->freelist), node))
+               return;
 
+       while (true) {
+               for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
+                       if (cpu == raw_smp_processor_id())
+                               continue;
                        head = per_cpu_ptr(s->freelist, cpu);
-                       if (raw_spin_trylock(&head->lock)) {
-                               pcpu_freelist_push_node(head, node);
-                               raw_spin_unlock(&head->lock);
-                               return;
-                       }
-               }
-
-               /* cannot lock any per cpu lock, try extralist */
-               if (pcpu_freelist_try_push_extra(s, node))
+                       if (raw_res_spin_lock(&head->lock))
+                               continue;
+                       pcpu_freelist_push_node(head, node);
+                       raw_res_spin_unlock(&head->lock);
                        return;
+               }
        }
 }
 
-void __pcpu_freelist_push(struct pcpu_freelist *s,
-                       struct pcpu_freelist_node *node)
-{
-       if (in_nmi())
-               ___pcpu_freelist_push_nmi(s, node);
-       else
-               ___pcpu_freelist_push(this_cpu_ptr(s->freelist), node);
-}
-
 void pcpu_freelist_push(struct pcpu_freelist *s,
                        struct pcpu_freelist_node *node)
 {
@@ -120,71 +99,29 @@ void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
 
 static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
 {
+       struct pcpu_freelist_node *node = NULL;
        struct pcpu_freelist_head *head;
-       struct pcpu_freelist_node *node;
        int cpu;
 
        for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
                head = per_cpu_ptr(s->freelist, cpu);
                if (!READ_ONCE(head->first))
                        continue;
-               raw_spin_lock(&head->lock);
+               if (raw_res_spin_lock(&head->lock))
+                       continue;
                node = head->first;
                if (node) {
                        WRITE_ONCE(head->first, node->next);
-                       raw_spin_unlock(&head->lock);
+                       raw_res_spin_unlock(&head->lock);
                        return node;
                }
-               raw_spin_unlock(&head->lock);
+               raw_res_spin_unlock(&head->lock);
        }
-
-       /* per cpu lists are all empty, try extralist */
-       if (!READ_ONCE(s->extralist.first))
-               return NULL;
-       raw_spin_lock(&s->extralist.lock);
-       node = s->extralist.first;
-       if (node)
-               WRITE_ONCE(s->extralist.first, node->next);
-       raw_spin_unlock(&s->extralist.lock);
-       return node;
-}
-
-static struct pcpu_freelist_node *
-___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
-{
-       struct pcpu_freelist_head *head;
-       struct pcpu_freelist_node *node;
-       int cpu;
-
-       for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
-               head = per_cpu_ptr(s->freelist, cpu);
-               if (!READ_ONCE(head->first))
-                       continue;
-               if (raw_spin_trylock(&head->lock)) {
-                       node = head->first;
-                       if (node) {
-                               WRITE_ONCE(head->first, node->next);
-                               raw_spin_unlock(&head->lock);
-                               return node;
-                       }
-                       raw_spin_unlock(&head->lock);
-               }
-       }
-
-       /* cannot pop from per cpu lists, try extralist */
-       if (!READ_ONCE(s->extralist.first) || !raw_spin_trylock(&s->extralist.lock))
-               return NULL;
-       node = s->extralist.first;
-       if (node)
-               WRITE_ONCE(s->extralist.first, node->next);
-       raw_spin_unlock(&s->extralist.lock);
        return node;
 }
 
 struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s)
 {
-       if (in_nmi())
-               return ___pcpu_freelist_pop_nmi(s);
        return ___pcpu_freelist_pop(s);
 }
 

diff --git a/kernel/bpf/percpu_freelist.h b/kernel/bpf/percpu_freelist.h
index 3c76553cfe571f8e1ed427bc56154f0c7e000f4e..914798b74967298a1dcdaf3b92a7a2856080b4a0 100644 (file)
--- a/kernel/bpf/percpu_freelist.h
+++ b/kernel/bpf/percpu_freelist.h
@@ -5,15 +5,15 @@
 #define __PERCPU_FREELIST_H__
 #include <linux/spinlock.h>
 #include <linux/percpu.h>
+#include <asm/rqspinlock.h>
 
 struct pcpu_freelist_head {
        struct pcpu_freelist_node *first;
-       raw_spinlock_t lock;
+       rqspinlock_t lock;
 };
 
 struct pcpu_freelist {
        struct pcpu_freelist_head __percpu *freelist;
-       struct pcpu_freelist_head extralist;
 };
 
 struct pcpu_freelist_node {