bpf: Pass bpf_verifier_env to JIT

Pass bpf_verifier_env to bpf_int_jit_compile(). The follow-up patch will
use env->insn_aux_data in the JIT stage to detect indirect jump targets.

Since bpf_prog_select_runtime() can be called by cbpf and lib/test_bpf.c
code without verifier, introduce helper __bpf_prog_select_runtime()
to accept the env parameter.

Remove the call to bpf_prog_select_runtime() in bpf_prog_load(), and
switch to call __bpf_prog_select_runtime() in the verifier, with env
variable passed. The original bpf_prog_select_runtime() is preserved for
cbpf and lib/test_bpf.c, where env is NULL.

Now all constants blinding calls are moved into the verifier, except
the cbpf and lib/test_bpf.c cases. The instructions arrays are adjusted
by bpf_patch_insn_data() function for normal cases, so there is no need
to call adjust_insn_arrays() in bpf_jit_blind_constants(). Remove it.

Reviewed-by: Anton Protopopov <a.s.protopopov@gmail.com> # v8
Reviewed-by: Emil Tsalapatis <emil@etsalapatis.com> # v12
Acked-by: Hengqi Chen <hengqi.chen@gmail.com> # v14
Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
Link: https://lore.kernel.org/r/20260416064341.151802-3-xukuohai@huaweicloud.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c
index 973ceae48675e7040b74e01361513a9731497e25..639a2736f029ed65e781be24e874a7f4b22158da 100644 (file)
--- a/arch/arc/net/bpf_jit_core.c
+++ b/arch/arc/net/bpf_jit_core.c
@@ -1400,7 +1400,7 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
  * (re)locations involved that their addresses are not known
  * during the first run.
  */
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        vm_dump(prog);
 

diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index e6b1bb2de627410107d74668b5f82727f7c3d4c1..1628b6fc70a419b463c228d86840469d89269b1b 100644 (file)
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -2142,7 +2142,7 @@ bool bpf_jit_needs_zext(void)
        return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        struct bpf_binary_header *header;
        struct jit_ctx ctx;

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index d310d1c351927c8d5517604e5cfd93a6d2787bd2..bd8757952507cdb7bbee8096e02acf572126b840 100644 (file)
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -2000,7 +2000,7 @@ struct arm64_jit_data {
        struct jit_ctx ctx;
 };
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        int image_size, prog_size, extable_size, extable_align, extable_offset;
        struct bpf_binary_header *header;

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index fcc8c0c29fb071054b657b2be4fd2c46d88aedf9..5149ce4cef7efe203adb2ed4760115169838f449 100644 (file)
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1920,7 +1920,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
        return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        bool extra_pass = false;
        u8 *image_ptr, *ro_image_ptr;

diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index d2b6c955f18eb08f36ed6d34b95dea674ec4f542..6ee4abe6a1f705a4a16a8a2291ef2ec86712d2ef 100644 (file)
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -909,7 +909,7 @@ bool bpf_jit_needs_zext(void)
        return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        struct bpf_binary_header *header = NULL;
        struct jit_context ctx;

diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c
index 35dca372b5df95f8c412fc7100b48ebb7068dcf4..172770132440db881ce9891915100f1ef4e53874 100644 (file)
--- a/arch/parisc/net/bpf_jit_core.c
+++ b/arch/parisc/net/bpf_jit_core.c
@@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
        return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        unsigned int prog_size = 0, extable_size = 0;
        bool extra_pass = false;

diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 2bae4699e78fda8d90537bfb58630a72960081fe..53ab97ad6074530f642a472728cdf479147cf63d 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -162,7 +162,7 @@ static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size
        }
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
 {
        u32 proglen;
        u32 alloclen;

diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index 36f0aea8096de0047126b79d2831a4fe1698ef74..4365d07aaf547c40d1a254cf46b0c6440109b824 100644 (file)
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
        return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        unsigned int prog_size = 0, extable_size = 0;
        bool extra_pass = false;

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 2dfc279b1be2c48402342713720f90178bd2c2d9..94128fe6be2334c2ce515e23f69885f1e444ea89 100644 (file)
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -2312,7 +2312,7 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
 /*
  * Compile eBPF program "fp"
  */
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
 {
        struct bpf_binary_header *header;
        struct s390_jit_data *jit_data;

diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index e83e29137566060e2124afe1a93d4790de40d10d..2fa0e93751276e5326c5258df50321f2b9e1fc03 100644 (file)
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -1477,7 +1477,7 @@ struct sparc64_jit_data {
        struct jit_ctx ctx;
 };
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        struct sparc64_jit_data *jit_data;
        struct bpf_binary_header *header;

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 77d00a8dec87bf1866195313fd3ecf9f99ca0482..72d9a5faa230871113a2985b4bda428847cf966a 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3713,7 +3713,7 @@ struct x64_jit_data {
 #define MAX_PASSES 20
 #define PADDING_PASSES (MAX_PASSES - 5)
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        struct bpf_binary_header *rw_header = NULL;
        struct bpf_binary_header *header = NULL;

diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index 5f259577614aff06e6e5b82b94383f2efd85a965..852baf2e4db4d18aea2c4396952af3a94e5f7a87 100644 (file)
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -2518,7 +2518,7 @@ bool bpf_jit_needs_zext(void)
        return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        struct bpf_binary_header *header = NULL;
        int proglen, oldproglen = 0;

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 9fa4d4090093cce89a6bc95121fa8cdd00ee586f..1ec6d5ba64cc4106eb2f1bc1c40a84a85e79ff52 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1108,6 +1108,8 @@ sk_filter_reason(struct sock *sk, struct sk_buff *skb)
        return sk_filter_trim_cap(sk, skb, 1);
 }
 
+struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
+                                          int *err);
 struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
 void bpf_prog_free(struct bpf_prog *fp);
 
@@ -1153,7 +1155,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
        ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
         (void *)__bpf_call_base)
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
 void bpf_jit_compile(struct bpf_prog *prog);
 bool bpf_jit_needs_zext(void);
 bool bpf_jit_inlines_helper_call(s32 imm);
@@ -1188,12 +1190,25 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
 #ifdef CONFIG_BPF_SYSCALL
 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
                                     const struct bpf_insn *patch, u32 len);
+struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env);
+void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
+                              struct bpf_insn_aux_data *orig_insn_aux);
 #else
 static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
                                                   const struct bpf_insn *patch, u32 len)
 {
        return ERR_PTR(-ENOTSUPP);
 }
+
+static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
+{
+       return NULL;
+}
+
+static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
+                                            struct bpf_insn_aux_data *orig_insn_aux)
+{
+}
 #endif /* CONFIG_BPF_SYSCALL */
 
 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index fc9fb3c07866c4e39020911bbd653920274d6c53..79361aa117575ded7b5c8ee216f74d2d5d26cdfb 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1491,23 +1491,6 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
        bpf_prog_clone_free(fp_other);
 }
 
-static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
-{
-#ifdef CONFIG_BPF_SYSCALL
-       struct bpf_map *map;
-       int i;
-
-       if (len <= 1)
-               return;
-
-       for (i = 0; i < prog->aux->used_map_cnt; i++) {
-               map = prog->aux->used_maps[i];
-               if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
-                       bpf_insn_array_adjust(map, off, len);
-       }
-#endif
-}
-
 /*
  * Now this function is used only to blind the main prog and must be invoked only when
  * bpf_prog_need_blind() returns true.
@@ -1580,13 +1563,6 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
 
                if (env)
                        env->prog = clone;
-               else
-                       /*
-                        * Instructions arrays must be updated using absolute xlated offsets.
-                        * The arrays have already been adjusted by bpf_patch_insn_data() when
-                        * env is not NULL.
-                        */
-                       adjust_insn_arrays(clone, i, rewritten);
 
                /* Walk new program and skip insns we just inserted. */
                insn = clone->insnsi + i + insn_delta;
@@ -2555,47 +2531,55 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
        return select_interpreter;
 }
 
-static struct bpf_prog *bpf_prog_jit_compile(struct bpf_prog *prog)
+static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 #ifdef CONFIG_BPF_JIT
        struct bpf_prog *orig_prog;
+       struct bpf_insn_aux_data *orig_insn_aux;
 
        if (!bpf_prog_need_blind(prog))
-               return bpf_int_jit_compile(prog);
+               return bpf_int_jit_compile(env, prog);
+
+       if (env) {
+               /*
+                * If env is not NULL, we are called from the end of bpf_check(), at this
+                * point, only insn_aux_data is used after failure, so it should be restored
+                * on failure.
+                */
+               orig_insn_aux = bpf_dup_insn_aux_data(env);
+               if (!orig_insn_aux)
+                       return prog;
+       }
 
        orig_prog = prog;
-       prog = bpf_jit_blind_constants(NULL, prog);
+       prog = bpf_jit_blind_constants(env, prog);
        /*
         * If blinding was requested and we failed during blinding, we must fall
         * back to the interpreter.
         */
        if (IS_ERR(prog))
-               return orig_prog;
+               goto out_restore;
 
-       prog = bpf_int_jit_compile(prog);
+       prog = bpf_int_jit_compile(env, prog);
        if (prog->jited) {
                bpf_jit_prog_release_other(prog, orig_prog);
+               if (env)
+                       vfree(orig_insn_aux);
                return prog;
        }
 
        bpf_jit_prog_release_other(orig_prog, prog);
+
+out_restore:
        prog = orig_prog;
+       if (env)
+               bpf_restore_insn_aux_data(env, orig_insn_aux);
 #endif
        return prog;
 }
 
-/**
- *     bpf_prog_select_runtime - select exec runtime for BPF program
- *     @fp: bpf_prog populated with BPF program
- *     @err: pointer to error variable
- *
- * Try to JIT eBPF program, if JIT is not available, use interpreter.
- * The BPF program will be executed via bpf_prog_run() function.
- *
- * Return: the &fp argument along with &err set to 0 for success or
- * a negative errno code on failure
- */
-struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
+struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
+                                          int *err)
 {
        /* In case of BPF to BPF calls, verifier did all the prep
         * work with regards to JITing, etc.
@@ -2623,7 +2607,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
                if (*err)
                        return fp;
 
-               fp = bpf_prog_jit_compile(fp);
+               fp = bpf_prog_jit_compile(env, fp);
                bpf_prog_jit_attempt_done(fp);
                if (!fp->jited && jit_needed) {
                        *err = -ENOTSUPP;
@@ -2649,6 +2633,22 @@ finalize:
 
        return fp;
 }
+
+/**
+ *     bpf_prog_select_runtime - select exec runtime for BPF program
+ *     @fp: bpf_prog populated with BPF program
+ *     @err: pointer to error variable
+ *
+ * Try to JIT eBPF program, if JIT is not available, use interpreter.
+ * The BPF program will be executed via bpf_prog_run() function.
+ *
+ * Return: the &fp argument along with &err set to 0 for success or
+ * a negative errno code on failure
+ */
+struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
+{
+       return __bpf_prog_select_runtime(NULL, fp, err);
+}
 EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
 
 static unsigned int __bpf_prog_ret1(const void *ctx,
@@ -3136,7 +3136,7 @@ const struct bpf_func_proto bpf_tail_call_proto = {
  * It is encouraged to implement bpf_int_jit_compile() instead, so that
  * eBPF and implicitly also cBPF can get JITed!
  */
-struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
        return prog;
 }

diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 721b830b5ef2bf0d669ced5c4362874430ea8804..6c86980cc9e8eb38a88564e438e6f233f9792963 100644 (file)
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -993,7 +993,7 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s
        env->subprog_info[env->subprog_cnt].start = env->prog->len;
 }
 
-static struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
+struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
 {
        size_t size;
        void *new_aux;
@@ -1005,8 +1005,8 @@ static struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *
        return new_aux;
 }
 
-static void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
-                                     struct bpf_insn_aux_data *orig_insn_aux)
+void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
+                              struct bpf_insn_aux_data *orig_insn_aux)
 {
        /* the expanded elements are zero-filled, so no special handling is required */
        vfree(env->insn_aux_data);
@@ -1150,7 +1150,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
                func[i]->aux->token = prog->aux->token;
                if (!i)
                        func[i]->aux->exception_boundary = env->seen_exception;
-               func[i] = bpf_int_jit_compile(func[i]);
+               func[i] = bpf_int_jit_compile(env, func[i]);
                if (!func[i]->jited) {
                        err = -ENOTSUPP;
                        goto out_free;
@@ -1194,7 +1194,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
        }
        for (i = 0; i < env->subprog_cnt; i++) {
                old_bpf_func = func[i]->bpf_func;
-               tmp = bpf_int_jit_compile(func[i]);
+               tmp = bpf_int_jit_compile(env, func[i]);
                if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
                        verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
                        err = -ENOTSUPP;

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index b73b25c630734e2699f559f1e6cdbb41f4418341..a3c0214ca9341067eff2f6fc1897497f9cb3a9ec 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3083,10 +3083,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
        if (err < 0)
                goto free_used_maps;
 
-       prog = bpf_prog_select_runtime(prog, &err);
-       if (err < 0)
-               goto free_used_maps;
-
        err = bpf_prog_mark_insn_arrays_ready(prog);
        if (err < 0)
                goto free_used_maps;

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 9e4980128151baeac4ef35e42923a3c36bc5501e..e804e0da350035158a84b2bbbd79cd4b26863e0c 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -20155,6 +20155,14 @@ skip_full_check:
 
        adjust_btf_func(env);
 
+       /* extension progs temporarily inherit the attach_type of their targets
+          for verification purposes, so set it back to zero before returning
+        */
+       if (env->prog->type == BPF_PROG_TYPE_EXT)
+               env->prog->expected_attach_type = 0;
+
+       env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
+
 err_release_maps:
        if (ret)
                release_insn_arrays(env);
@@ -20166,12 +20174,6 @@ err_release_maps:
        if (!env->prog->aux->used_btfs)
                release_btfs(env);
 
-       /* extension progs temporarily inherit the attach_type of their targets
-          for verification purposes, so set it back to zero before returning
-        */
-       if (env->prog->type == BPF_PROG_TYPE_EXT)
-               env->prog->expected_attach_type = 0;
-
        *prog = env->prog;
 
        module_put(env->attach_btf_mod);