bpf, x86: fix freeing of not-finalized bpf_prog_pack

[ Upstream commit 1d5f82d9dd477d5c66e0214a68c3e4f308eadd6d ]

syzbot reported a few issues with bpf_prog_pack [1], [2]. This only happens
with multiple subprogs. In jit_subprogs(), we first call bpf_int_jit_compile()
on each sub program. And then, we call it on each sub program again. jit_data
is not freed in the first call of bpf_int_jit_compile(). Similarly we don't
call bpf_jit_binary_pack_finalize() in the first call of bpf_int_jit_compile().

If bpf_int_jit_compile() failed for one sub program, we will call
bpf_jit_binary_pack_finalize() for this sub program. However, we don't have a
chance to call it for other sub programs. Then we will hit "goto out_free" in
jit_subprogs(), and call bpf_jit_free on some subprograms that haven't got
bpf_jit_binary_pack_finalize() yet.

At this point, bpf_jit_binary_pack_free() is called and the whole 2MB page is
freed erroneously.

Fix this with a custom bpf_jit_free() for x86_64, which calls
bpf_jit_binary_pack_finalize() if necessary. Also, with custom
bpf_jit_free(), bpf_prog_aux->use_bpf_prog_pack is not needed any more,
remove it.

Fixes: 1022a5498f6f ("bpf, x86_64: Use bpf_jit_binary_pack_alloc")
[1] https://syzkaller.appspot.com/bug?extid=2f649ec6d2eea1495a8f
[2] https://syzkaller.appspot.com/bug?extid=87f65c75f4a72db05445
Reported-by: syzbot+2f649ec6d2eea1495a8f@syzkaller.appspotmail.com
Reported-by: syzbot+87f65c75f4a72db05445@syzkaller.appspotmail.com
Signed-off-by: Song Liu <song@kernel.org>
Link: https://lore.kernel.org/r/20220706002612.4013790-1-song@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 124456bb23b92f522cb1b5d20cd2d9f271e529e9..400117f630b814c20a655d53bae0e1e854dc596c 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -2483,3 +2483,28 @@ bool bpf_jit_supports_subprog_tailcalls(void)
 {
        return true;
 }
+
+void bpf_jit_free(struct bpf_prog *prog)
+{
+       if (prog->jited) {
+               struct x64_jit_data *jit_data = prog->aux->jit_data;
+               struct bpf_binary_header *hdr;
+
+               /*
+                * If we fail the final pass of JIT (from jit_subprogs),
+                * the program may not be finalized yet. Call finalize here
+                * before freeing it.
+                */
+               if (jit_data) {
+                       bpf_jit_binary_pack_finalize(prog, jit_data->header,
+                                                    jit_data->rw_header);
+                       kvfree(jit_data->addrs);
+                       kfree(jit_data);
+               }
+               hdr = bpf_jit_binary_pack_hdr(prog);
+               bpf_jit_binary_pack_free(hdr, NULL);
+               WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
+       }
+
+       bpf_prog_unlock_free(prog);
+}

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index b5d19a6f7d247449b7a338a9c0a581bd3794e89c..492e114b4e32a204e4815e5a08b9ecc3c23297d4 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -960,7 +960,6 @@ struct bpf_prog_aux {
        bool sleepable;
        bool tail_call_reachable;
        bool xdp_has_frags;
-       bool use_bpf_prog_pack;
        /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
        const struct btf_type *attach_func_proto;
        /* function name for valid attach_btf_id */

diff --git a/include/linux/filter.h b/include/linux/filter.h
index d9a0db845b508ce2bb1c91a90b56f97923e6c5b7..8fd2e2f58eeb22b4a72e43a78f9edae284eeddb8 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1061,6 +1061,14 @@ u64 bpf_jit_alloc_exec_limit(void);
 void *bpf_jit_alloc_exec(unsigned long size);
 void bpf_jit_free_exec(void *addr);
 void bpf_jit_free(struct bpf_prog *fp);
+struct bpf_binary_header *
+bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);
+
+static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
+{
+       return list_empty(&fp->aux->ksym.lnode) ||
+              fp->aux->ksym.lnode.prev == LIST_POISON2;
+}
 
 struct bpf_binary_header *
 bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image,

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 7a1ce697689b97f559afd7228aa276089f2507af..483bee45ead519a771b5938ed74d5e3977556256 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -649,12 +649,6 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp)
        return fp->jited && !bpf_prog_was_classic(fp);
 }
 
-static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
-{
-       return list_empty(&fp->aux->ksym.lnode) ||
-              fp->aux->ksym.lnode.prev == LIST_POISON2;
-}
-
 void bpf_prog_kallsyms_add(struct bpf_prog *fp)
 {
        if (!bpf_prog_kallsyms_candidate(fp) ||
@@ -1149,7 +1143,6 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog,
                bpf_prog_pack_free(ro_header);
                return PTR_ERR(ptr);
        }
-       prog->aux->use_bpf_prog_pack = true;
        return 0;
 }
 
@@ -1173,17 +1166,23 @@ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header,
        bpf_jit_uncharge_modmem(size);
 }
 
+struct bpf_binary_header *
+bpf_jit_binary_pack_hdr(const struct bpf_prog *fp)
+{
+       unsigned long real_start = (unsigned long)fp->bpf_func;
+       unsigned long addr;
+
+       addr = real_start & BPF_PROG_CHUNK_MASK;
+       return (void *)addr;
+}
+
 static inline struct bpf_binary_header *
 bpf_jit_binary_hdr(const struct bpf_prog *fp)
 {
        unsigned long real_start = (unsigned long)fp->bpf_func;
        unsigned long addr;
 
-       if (fp->aux->use_bpf_prog_pack)
-               addr = real_start & BPF_PROG_CHUNK_MASK;
-       else
-               addr = real_start & PAGE_MASK;
-
+       addr = real_start & PAGE_MASK;
        return (void *)addr;
 }
 
@@ -1196,11 +1195,7 @@ void __weak bpf_jit_free(struct bpf_prog *fp)
        if (fp->jited) {
                struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp);
 
-               if (fp->aux->use_bpf_prog_pack)
-                       bpf_jit_binary_pack_free(hdr, NULL /* rw_buffer */);
-               else
-                       bpf_jit_binary_free(hdr);
-
+               bpf_jit_binary_free(hdr);
                WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
        }