bpf: Refactor storage_get_func_atomic to generic non_sleepable flag

Rename the storage_get_func_atomic flag to a more generic non_sleepable
flag that tracks whether a helper or kfunc may be called from a
non-sleepable context. This makes the flag more broadly applicable
beyond just storage_get helpers. See [0] for more context.

The flag is now set unconditionally for all helpers and kfuncs when:
- RCU critical section is active.
- Preemption is disabled.
- IRQs are disabled.
- In a non-sleepable context within a sleepable program (e.g., timer
  callbacks), which is indicated by !in_sleepable().

Previously, the flag was only set for storage_get helpers in these
contexts. With this change, it can be used by any code that needs to
differentiate between sleepable and non-sleepable contexts at the
per-instruction level.

The existing usage in do_misc_fixups() for storage_get helpers is
preserved by checking is_storage_get_function() before using the flag.

  [0]: https://lore.kernel.org/bpf/CAP01T76cbaNi4p-y8E0sjE2NXSra2S=Uja8G4hSQDu_SbXxREQ@mail.gmail.com

Cc: Mykyta Yatsenko <yatsenko@meta.com>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Acked-by: Mykyta Yatsenko <mykyta.yatsenko5@gmail.com>
Link: https://lore.kernel.org/r/20251007220349.3852807-3-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 4c497e839526a40a66c0c9f1054c19763ac9d41a..b57222a25a4ac7d950c81c067136f5f310c42876 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -548,7 +548,7 @@ struct bpf_insn_aux_data {
        bool nospec_result; /* result is unsafe under speculation, nospec must follow */
        bool zext_dst; /* this insn zero extends dst reg */
        bool needs_zext; /* alu op needs to clear upper bits */
-       bool storage_get_func_atomic; /* bpf_*_storage_get() with atomic memory alloc */
+       bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */
        bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */
        bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */
        u8 alu_state; /* used in combination with alu_limit */

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 32123c4b041a8bd906526bee24619554ddf35802..85a9531244124f8ee90d3f549ea314a108ae0690 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -11371,6 +11371,15 @@ static int get_helper_proto(struct bpf_verifier_env *env, int func_id,
        return *ptr && (*ptr)->func ? 0 : -EINVAL;
 }
 
+/* Check if we're in a sleepable context. */
+static inline bool in_sleepable_context(struct bpf_verifier_env *env)
+{
+       return !env->cur_state->active_rcu_lock &&
+              !env->cur_state->active_preempt_locks &&
+              !env->cur_state->active_irq_id &&
+              in_sleepable(env);
+}
+
 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                             int *insn_idx_p)
 {
@@ -11437,9 +11446,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
                                func_id_name(func_id), func_id);
                        return -EINVAL;
                }
-
-               if (is_storage_get_function(func_id))
-                       env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
        }
 
        if (env->cur_state->active_preempt_locks) {
@@ -11448,9 +11454,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
                                func_id_name(func_id), func_id);
                        return -EINVAL;
                }
-
-               if (is_storage_get_function(func_id))
-                       env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
        }
 
        if (env->cur_state->active_irq_id) {
@@ -11459,17 +11462,11 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
                                func_id_name(func_id), func_id);
                        return -EINVAL;
                }
-
-               if (is_storage_get_function(func_id))
-                       env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
        }
 
-       /*
-        * Non-sleepable contexts in sleepable programs (e.g., timer callbacks)
-        * are atomic and must use GFP_ATOMIC for storage_get helpers.
-        */
-       if (!in_sleepable(env) && is_storage_get_function(func_id))
-               env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
+       /* Track non-sleepable context for helpers. */
+       if (!in_sleepable_context(env))
+               env->insn_aux_data[insn_idx].non_sleepable = true;
 
        meta.func_id = func_id;
        /* check args */
@@ -13880,6 +13877,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                return -EACCES;
        }
 
+       /* Track non-sleepable context for kfuncs, same as for helpers. */
+       if (!in_sleepable_context(env))
+               insn_aux->non_sleepable = true;
+
        /* Check the arguments */
        err = check_kfunc_args(env, &meta, insn_idx);
        if (err < 0)
@@ -22502,7 +22503,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
                }
 
                if (is_storage_get_function(insn->imm)) {
-                       if (env->insn_aux_data[i + delta].storage_get_func_atomic)
+                       if (env->insn_aux_data[i + delta].non_sleepable)
                                insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC);
                        else
                                insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL);