bpf: Check global subprog exception paths

Global subprogs are verified independently and are not descended into
when their callers are symbolically executed. This means a caller can
hold references or locks across a global subprog call that may throw,
while the verifier only checks the non-exceptional return path at the
call site.

Record whether a subprog might throw in the CFG summary pass, alongside
the existing might_sleep and packet-data-changing summaries, and
propagate that effect through reachable callees.

When a global subprog is marked as possibly throwing, push the normal
continuation and validate the exceptional path immediately at the call
site, avoiding a synthetic exception state and associated special case
in the pruning checks.

Fixes: f18b03fabaa9 ("bpf: Implement BPF exceptions")
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20260517075530.3461166-2-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index b148f816f25b08a4cff5bd7a2512b5272f73a35d..185b2aa43a420d6d80928b6014520075b725b4c4 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -729,6 +729,7 @@ struct bpf_subprog_info {
         */
        s16 fastcall_stack_off;
        bool has_tail_call: 1;
+       bool might_throw: 1;
        bool tail_call_reachable: 1;
        bool has_ld_abs: 1;
        bool is_cb: 1;
@@ -1308,6 +1309,7 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
 bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog);
 
 int bpf_find_subprog(struct bpf_verifier_env *env, int off);
+bool bpf_is_throw_kfunc(struct bpf_insn *insn);
 int bpf_compute_const_regs(struct bpf_verifier_env *env);
 int bpf_prune_dead_branches(struct bpf_verifier_env *env);
 int bpf_check_cfg(struct bpf_verifier_env *env);

diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c
index 998f42a8189a403e3f89efe2d63a313368307991..26d37066465f384da136bd5bc314d4af2510442e 100644 (file)
--- a/kernel/bpf/cfg.c
+++ b/kernel/bpf/cfg.c
@@ -64,11 +64,19 @@ static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
        subprog->might_sleep = true;
 }
 
+static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off)
+{
+       struct bpf_subprog_info *subprog;
+
+       subprog = bpf_find_containing_subprog(env, off);
+       subprog->might_throw = true;
+}
+
 /* 't' is an index of a call-site.
  * 'w' is a callee entry point.
  * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
  * Rely on DFS traversal order and absence of recursive calls to guarantee that
- * callee's change_pkt_data marks would be correct at that moment.
+ * callee's effect marks would be correct at that moment.
  */
 static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
 {
@@ -78,6 +86,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
        callee = bpf_find_containing_subprog(env, w);
        caller->changes_pkt_data |= callee->changes_pkt_data;
        caller->might_sleep |= callee->might_sleep;
+       caller->might_throw |= callee->might_throw;
 }
 
 enum {
@@ -509,6 +518,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
                                mark_subprog_might_sleep(env, t);
                        if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta))
                                mark_subprog_changes_pkt_data(env, t);
+                       if (ret == 0 && bpf_is_throw_kfunc(insn))
+                               mark_subprog_might_throw(env, t);
                }
                return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
 

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 88b40c979b567a377cfe4e05faf176766a2592a2..7fb88e1cd7c4d0ecb393874a6e11bd0e543e8d8d 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -442,7 +442,6 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id)
 static bool is_sync_callback_calling_kfunc(u32 btf_id);
 static bool is_async_callback_calling_kfunc(u32 btf_id);
 static bool is_callback_calling_kfunc(u32 btf_id);
-static bool is_bpf_throw_kfunc(struct bpf_insn *insn);
 
 static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);
 static bool is_task_work_add_kfunc(u32 func_id);
@@ -5405,7 +5404,7 @@ continue_func:
                if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) {
                        bool err = false;
 
-                       if (!is_bpf_throw_kfunc(insn + i))
+                       if (!bpf_is_throw_kfunc(insn + i))
                                continue;
                        for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) {
                                if (subprog[tmp].is_cb) {
@@ -9499,6 +9498,9 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
        return 0;
 }
 
+static int process_bpf_exit_full(struct bpf_verifier_env *env,
+                                bool *do_print_state, bool exception_exit);
+
 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                           int *insn_idx)
 {
@@ -9552,6 +9554,17 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                        caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
                }
 
+               if (env->subprog_info[subprog].might_throw) {
+                       struct bpf_verifier_state *branch;
+
+                       branch = push_stack(env, *insn_idx + 1, *insn_idx, false);
+                       if (IS_ERR(branch)) {
+                               verbose(env, "failed to push state for global subprog exception path\n");
+                               return PTR_ERR(branch);
+                       }
+                       return process_bpf_exit_full(env, NULL, true);
+               }
+
                /* continue with next insn after call */
                return 0;
        }
@@ -11782,7 +11795,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id)
               is_task_work_add_kfunc(btf_id);
 }
 
-static bool is_bpf_throw_kfunc(struct bpf_insn *insn)
+bool bpf_is_throw_kfunc(struct bpf_insn *insn)
 {
        return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
               insn->imm == special_kfunc_list[KF_bpf_throw];
@@ -12972,8 +12985,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca
 }
 
 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
-static int process_bpf_exit_full(struct bpf_verifier_env *env,
-                                bool *do_print_state, bool exception_exit);
 
 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                            int *insn_idx_p)
@@ -13354,7 +13365,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
        if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie])
                env->prog->call_session_cookie = true;
 
-       if (is_bpf_throw_kfunc(insn))
+       if (bpf_is_throw_kfunc(insn))
                return process_bpf_exit_full(env, NULL, true);
 
        return 0;