bpf: Search and add kfuncs in struct_ops prologue and epilogue

[ Upstream commit d519594ee2445d7cd1ad51f4db4cee58f8213400 ]

Currently, add_kfunc_call() is only invoked once before the main
verification loop. Therefore, the verifier could not find the
bpf_kfunc_btf_tab of a new kfunc call which is not seen in user defined
struct_ops operators but introduced in gen_prologue or gen_epilogue
during do_misc_fixup(). Fix this by searching kfuncs in the patching
instruction buffer and add them to prog->aux->kfunc_tab.

Signed-off-by: Amery Hung <amery.hung@bytedance.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Acked-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20250225233545.285481-1-ameryhung@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index db95b76f5c1397fe0d016646df85189c80f0d732..e2801472c0ae43835d4426398a7d20db2849dee3 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3206,6 +3206,21 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
        return res ? &res->func_model : NULL;
 }
 
+static int add_kfunc_in_insns(struct bpf_verifier_env *env,
+                             struct bpf_insn *insn, int cnt)
+{
+       int i, ret;
+
+       for (i = 0; i < cnt; i++, insn++) {
+               if (bpf_pseudo_kfunc_call(insn)) {
+                       ret = add_kfunc_call(env, insn->imm, insn->off);
+                       if (ret < 0)
+                               return ret;
+               }
+       }
+       return 0;
+}
+
 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
 {
        struct bpf_subprog_info *subprog = env->subprog_info;
@@ -20334,7 +20349,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 {
        struct bpf_subprog_info *subprogs = env->subprog_info;
        const struct bpf_verifier_ops *ops = env->ops;
-       int i, cnt, size, ctx_field_size, delta = 0, epilogue_cnt = 0;
+       int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
        const int insn_cnt = env->prog->len;
        struct bpf_insn *epilogue_buf = env->epilogue_buf;
        struct bpf_insn *insn_buf = env->insn_buf;
@@ -20363,6 +20378,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
                                return -ENOMEM;
                        env->prog = new_prog;
                        delta += cnt - 1;
+
+                       ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
+                       if (ret < 0)
+                               return ret;
                }
        }
 
@@ -20383,6 +20402,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 
                        env->prog = new_prog;
                        delta += cnt - 1;
+
+                       ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
+                       if (ret < 0)
+                               return ret;
                }
        }