bpf: Add bpf_prog_run_array_sleepable()

Add bpf_prog_run_array_sleepable() for running BPF program arrays
on faultable tracepoints. Unlike bpf_prog_run_array_uprobe(), it
includes per-program recursion checking for private stack safety
and hardcodes is_uprobe to false.

Skip dummy_bpf_prog at the top of the loop. When
bpf_prog_array_delete_safe() replaces a detached program with
dummy_bpf_prog on allocation failure, the dummy is statically
allocated and has NULL active, stats, and aux fields. Identify
it by prog->len == 0, since every real program has at least one
instruction.

Keep bpf_prog_run_array_uprobe() unchanged for uprobe callers.

Signed-off-by: Mykyta Yatsenko <yatsenko@meta.com>
Acked-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/bpf/20260422-sleepable_tracepoints-v13-2-99005dff21ef@meta.com
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 3cb6b9e70080f8581815ce61313f3debca526667..d3aea3931b85d16f78de7836cace34c8fc055065 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -3079,6 +3079,56 @@ void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
 void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip);
 
+static __always_inline u32
+bpf_prog_run_array_sleepable(const struct bpf_prog_array *array,
+                            const void *ctx, bpf_prog_run_fn run_prog)
+{
+       const struct bpf_prog_array_item *item;
+       struct bpf_prog *prog;
+       struct bpf_run_ctx *old_run_ctx;
+       struct bpf_trace_run_ctx run_ctx;
+       u32 ret = 1;
+
+       if (unlikely(!array))
+               return ret;
+
+       migrate_disable();
+
+       run_ctx.is_uprobe = false;
+
+       old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
+       item = &array->items[0];
+       while ((prog = READ_ONCE(item->prog))) {
+               /* Skip dummy_bpf_prog placeholder (len == 0) */
+               if (unlikely(!prog->len)) {
+                       item++;
+                       continue;
+               }
+
+               if (unlikely(!bpf_prog_get_recursion_context(prog))) {
+                       bpf_prog_inc_misses_counter(prog);
+                       bpf_prog_put_recursion_context(prog);
+                       item++;
+                       continue;
+               }
+
+               run_ctx.bpf_cookie = item->bpf_cookie;
+
+               if (!prog->sleepable) {
+                       guard(rcu)();
+                       ret &= run_prog(prog, ctx);
+               } else {
+                       ret &= run_prog(prog, ctx);
+               }
+
+               bpf_prog_put_recursion_context(prog);
+               item++;
+       }
+       bpf_reset_run_ctx(old_run_ctx);
+       migrate_enable();
+       return ret;
+}
+
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {