bpf: Add sleepable support for classic tracepoint programs

Add trace_call_bpf_faultable(), a variant of trace_call_bpf() for
faultable tracepoints that supports sleepable BPF programs. It uses
rcu_tasks_trace for lifetime protection and
bpf_prog_run_array_sleepable() for per-program RCU flavor selection,
following the uprobe_prog_run() pattern.

Restructure perf_syscall_enter() and perf_syscall_exit() to run BPF
programs before perf event processing. Previously, BPF ran after the
per-cpu perf trace buffer was allocated under preempt_disable,
requiring cleanup via perf_swevent_put_recursion_context() on filter.
Now BPF runs in faultable context before preempt_disable, reading
syscall arguments from local variables instead of the per-cpu trace
record, removing the dependency on buffer allocation. This allows
sleepable BPF programs to execute and avoids unnecessary buffer
allocation when BPF filters the event. The perf event submission
path (buffer allocation, fill, submit) remains under preempt_disable
as before. Since BPF no longer runs within the buffer allocation
context, the fake_regs output parameter to perf_trace_buf_alloc()
is no longer needed and is replaced with NULL.

Add an attach-time check in __perf_event_set_bpf_prog() to reject
sleepable BPF_PROG_TYPE_TRACEPOINT programs on non-syscall
tracepoints, since only syscall tracepoints run in faultable context.

This prepares the classic tracepoint runtime and attach paths for
sleepable programs. The verifier changes to allow loading sleepable
BPF_PROG_TYPE_TRACEPOINT programs are in a subsequent patch.

To: Peter Zijlstra <peterz@infradead.org>
To: Steven Rostedt <rostedt@goodmis.org>

Signed-off-by: Mykyta Yatsenko <yatsenko@meta.com>
Acked-by: Kumar Kartikeya Dwivedi <memxor@gmail.com> # for BPF bits
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Link: https://lore.kernel.org/bpf/20260422-sleepable_tracepoints-v13-3-99005dff21ef@meta.com
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>

diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
index 40a43a4c7caf454a30829d2c8607d74306fa7999..d49338c44014e5a6521ad3d8be1433616d9d47ba 100644 (file)
--- a/include/linux/trace_events.h
+++ b/include/linux/trace_events.h
@@ -770,6 +770,7 @@ trace_trigger_soft_disabled(struct trace_event_file *file)
 
 #ifdef CONFIG_BPF_EVENTS
 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);
+unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx);
 int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
 void perf_event_detach_bpf_prog(struct perf_event *event);
 int perf_event_query_prog_array(struct perf_event *event, void __user *info);
@@ -792,6 +793,11 @@ static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *c
        return 1;
 }
 
+static inline unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
+{
+       return 1;
+}
+
 static inline int
 perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie)
 {

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 6d1f8bad7e1c521023bd151fdf941871bf10890a..0f9cacfa7cb891c1da1e6454287ec7f42b994912 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -11643,6 +11643,15 @@ static int __perf_event_set_bpf_prog(struct perf_event *event,
                /* only uprobe programs are allowed to be sleepable */
                return -EINVAL;
 
+       if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) {
+               /*
+                * Sleepable tracepoint programs can only attach to faultable
+                * tracepoints. Currently only syscall tracepoints are faultable.
+                */
+               if (!is_syscall_tp)
+                       return -EINVAL;
+       }
+
        /* Kprobe override only works for kprobes, not uprobes. */
        if (prog->kprobe_override && !is_kprobe)
                return -EINVAL;

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 7276c72c1d31e7ce7ef3397f680c7e1673e04b55..a822c589c9bdac5d248dbc6ab23bd7e6d180e3ab 100644 (file)
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -152,6 +152,34 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
        return ret;
 }
 
+/**
+ * trace_call_bpf_faultable - invoke BPF program in faultable context
+ * @call: tracepoint event
+ * @ctx: opaque context pointer
+ *
+ * Variant of trace_call_bpf() for faultable tracepoints (syscall
+ * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace
+ * for lifetime protection and bpf_prog_run_array_sleepable() for per-program
+ * RCU flavor selection, following the uprobe pattern.
+ *
+ * Per-program recursion protection is provided by
+ * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not
+ * needed because syscall tracepoints cannot self-recurse.
+ *
+ * Must be called from a faultable/preemptible context.
+ */
+unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
+{
+       struct bpf_prog_array *prog_array;
+
+       might_fault();
+       guard(rcu_tasks_trace)();
+
+       prog_array = rcu_dereference_check(call->prog_array,
+                                          rcu_read_lock_trace_held());
+       return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run);
+}
+
 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
 BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)
 {

diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 8ad72e17d8eb917805a334ffaa9189353ce2d7a4..e98ee7e1e66f45e5285cea8b7e268c1cfea482b0 100644 (file)
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -1371,33 +1371,33 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
 static int sys_perf_refcount_enter;
 static int sys_perf_refcount_exit;
 
-static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,
+static int perf_call_bpf_enter(struct trace_event_call *call,
                               struct syscall_metadata *sys_data,
-                              struct syscall_trace_enter *rec)
+                              int syscall_nr, unsigned long *args)
 {
        struct syscall_tp_t {
                struct trace_entry ent;
                int syscall_nr;
                unsigned long args[SYSCALL_DEFINE_MAXARGS];
        } __aligned(8) param;
+       struct pt_regs regs = {};
        int i;
 
        BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *));
 
-       /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */
-       perf_fetch_caller_regs(regs);
-       *(struct pt_regs **)&param = regs;
-       param.syscall_nr = rec->nr;
+       /* bpf prog requires 'regs' to be the first member in the ctx */
+       perf_fetch_caller_regs(&regs);
+       *(struct pt_regs **)&param = &regs;
+       param.syscall_nr = syscall_nr;
        for (i = 0; i < sys_data->nb_args; i++)
-               param.args[i] = rec->args[i];
-       return trace_call_bpf(call, &param);
+               param.args[i] = args[i];
+       return trace_call_bpf_faultable(call, &param);
 }
 
 static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
 {
        struct syscall_metadata *sys_data;
        struct syscall_trace_enter *rec;
-       struct pt_regs *fake_regs;
        struct hlist_head *head;
        unsigned long args[6];
        bool valid_prog_array;
@@ -1410,12 +1410,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
        int size = 0;
        int uargs = 0;
 
-       /*
-        * Syscall probe called with preemption enabled, but the ring
-        * buffer and per-cpu data require preemption to be disabled.
-        */
        might_fault();
-       guard(preempt_notrace)();
 
        syscall_nr = trace_get_syscall_nr(current, regs);
        if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
@@ -1429,6 +1424,26 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
 
        syscall_get_arguments(current, regs, args);
 
+       /*
+        * Run BPF program in faultable context before per-cpu buffer
+        * allocation, allowing sleepable BPF programs to execute.
+        */
+       valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
+       if (valid_prog_array &&
+           !perf_call_bpf_enter(sys_data->enter_event, sys_data,
+                                syscall_nr, args))
+               return;
+
+       /*
+        * Per-cpu ring buffer and perf event list operations require
+        * preemption to be disabled.
+        */
+       guard(preempt_notrace)();
+
+       head = this_cpu_ptr(sys_data->enter_event->perf_events);
+       if (hlist_empty(head))
+               return;
+
        /* Check if this syscall event faults in user space memory */
        mayfault = sys_data->user_mask != 0;
 
@@ -1438,17 +1453,12 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
                        return;
        }
 
-       head = this_cpu_ptr(sys_data->enter_event->perf_events);
-       valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
-       if (!valid_prog_array && hlist_empty(head))
-               return;
-
        /* get the size after alignment with the u32 buffer size field */
        size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
        size = ALIGN(size + sizeof(u32), sizeof(u64));
        size -= sizeof(u32);
 
-       rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);
+       rec = perf_trace_buf_alloc(size, NULL, &rctx);
        if (!rec)
                return;
 
@@ -1458,13 +1468,6 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
        if (mayfault)
                syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs);
 
-       if ((valid_prog_array &&
-            !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) ||
-           hlist_empty(head)) {
-               perf_swevent_put_recursion_context(rctx);
-               return;
-       }
-
        perf_trace_buf_submit(rec, size, rctx,
                              sys_data->enter_event->event.type, 1, regs,
                              head, NULL);
@@ -1514,40 +1517,35 @@ static void perf_sysenter_disable(struct trace_event_call *call)
                syscall_fault_buffer_disable();
 }
 
-static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,
-                             struct syscall_trace_exit *rec)
+static int perf_call_bpf_exit(struct trace_event_call *call,
+                             int syscall_nr, long ret_val)
 {
        struct syscall_tp_t {
                struct trace_entry ent;
                int syscall_nr;
                unsigned long ret;
        } __aligned(8) param;
-
-       /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */
-       perf_fetch_caller_regs(regs);
-       *(struct pt_regs **)&param = regs;
-       param.syscall_nr = rec->nr;
-       param.ret = rec->ret;
-       return trace_call_bpf(call, &param);
+       struct pt_regs regs = {};
+
+       /* bpf prog requires 'regs' to be the first member in the ctx */
+       perf_fetch_caller_regs(&regs);
+       *(struct pt_regs **)&param = &regs;
+       param.syscall_nr = syscall_nr;
+       param.ret = ret_val;
+       return trace_call_bpf_faultable(call, &param);
 }
 
 static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
 {
        struct syscall_metadata *sys_data;
        struct syscall_trace_exit *rec;
-       struct pt_regs *fake_regs;
        struct hlist_head *head;
        bool valid_prog_array;
        int syscall_nr;
        int rctx;
        int size;
 
-       /*
-        * Syscall probe called with preemption enabled, but the ring
-        * buffer and per-cpu data require preemption to be disabled.
-        */
        might_fault();
-       guard(preempt_notrace)();
 
        syscall_nr = trace_get_syscall_nr(current, regs);
        if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
@@ -1559,29 +1557,37 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
        if (!sys_data)
                return;
 
-       head = this_cpu_ptr(sys_data->exit_event->perf_events);
+       /*
+        * Run BPF program in faultable context before per-cpu buffer
+        * allocation, allowing sleepable BPF programs to execute.
+        */
        valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);
-       if (!valid_prog_array && hlist_empty(head))
+       if (valid_prog_array &&
+           !perf_call_bpf_exit(sys_data->exit_event, syscall_nr,
+                               syscall_get_return_value(current, regs)))
+               return;
+
+       /*
+        * Per-cpu ring buffer and perf event list operations require
+        * preemption to be disabled.
+        */
+       guard(preempt_notrace)();
+
+       head = this_cpu_ptr(sys_data->exit_event->perf_events);
+       if (hlist_empty(head))
                return;
 
        /* We can probably do that at build time */
        size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
        size -= sizeof(u32);
 
-       rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);
+       rec = perf_trace_buf_alloc(size, NULL, &rctx);
        if (!rec)
                return;
 
        rec->nr = syscall_nr;
        rec->ret = syscall_get_return_value(current, regs);
 
-       if ((valid_prog_array &&
-            !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) ||
-           hlist_empty(head)) {
-               perf_swevent_put_recursion_context(rctx);
-               return;
-       }
-
        perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
                              1, regs, head, NULL);
 }