bpf: use realloc in bpf_patch_insn_data

Avoid excessive vzalloc/vfree calls when patching instructions in
do_misc_fixups(). bpf_patch_insn_data() uses vzalloc to allocate new
memory for env->insn_aux_data for each patch as follows:

  struct bpf_prog *bpf_patch_insn_data(env, ...)
  {
    ...
    new_data = vzalloc(... O(program size) ...);
    ...
    adjust_insn_aux_data(env, new_data, ...);
    ...
  }

  void adjust_insn_aux_data(env, new_data, ...)
  {
    ...
    memcpy(new_data, env->insn_aux_data);
    vfree(env->insn_aux_data);
    env->insn_aux_data = new_data;
    ...
  }

The vzalloc/vfree pair is hot in perf report collected for e.g.
pyperf180 test case. It can be replaced with a call to vrealloc in
order to reduce the number of actual memory allocations.

This is a stop-gap solution, as bpf_patch_insn_data is still hot in
the profile. More comprehansive solutions had been discussed before
e.g. as in [1].

[1] https://lore.kernel.org/bpf/CAEf4BzY_E8MSL4mD0UPuuiDcbJhh9e2xQo2=5w+ppRWWiYSGvQ@mail.gmail.com/

Suggested-by: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
Acked-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Tested-by: Anton Protopopov <a.s.protopopov@gmail.com>
Link: https://lore.kernel.org/r/20250807010205.3210608-3-eddyz87@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 69eb2b5c2218b2298640f0964da44617b9bba211..a61d5799669233195100da6c9fdeb129675c45fd 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -20699,12 +20699,11 @@ static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
  * [0, off) and [off, end) to new locations, so the patched range stays zero
  */
 static void adjust_insn_aux_data(struct bpf_verifier_env *env,
-                                struct bpf_insn_aux_data *new_data,
                                 struct bpf_prog *new_prog, u32 off, u32 cnt)
 {
-       struct bpf_insn_aux_data *old_data = env->insn_aux_data;
+       struct bpf_insn_aux_data *data = env->insn_aux_data;
        struct bpf_insn *insn = new_prog->insnsi;
-       u32 old_seen = old_data[off].seen;
+       u32 old_seen = data[off].seen;
        u32 prog_len;
        int i;
 
@@ -20712,22 +20711,20 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
         * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
         * original insn at old prog.
         */
-       old_data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);
+       data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);
 
        if (cnt == 1)
                return;
        prog_len = new_prog->len;
 
-       memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
-       memcpy(new_data + off + cnt - 1, old_data + off,
-              sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
+       memmove(data + off + cnt - 1, data + off,
+               sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
+       memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1));
        for (i = off; i < off + cnt - 1; i++) {
                /* Expand insni[off]'s seen count to the patched range. */
-               new_data[i].seen = old_seen;
-               new_data[i].zext_dst = insn_has_def32(insn + i);
+               data[i].seen = old_seen;
+               data[i].zext_dst = insn_has_def32(insn + i);
        }
-       env->insn_aux_data = new_data;
-       vfree(old_data);
 }
 
 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
@@ -20765,10 +20762,14 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of
        struct bpf_insn_aux_data *new_data = NULL;
 
        if (len > 1) {
-               new_data = vzalloc(array_size(env->prog->len + len - 1,
-                                             sizeof(struct bpf_insn_aux_data)));
+               new_data = vrealloc(env->insn_aux_data,
+                                   array_size(env->prog->len + len - 1,
+                                              sizeof(struct bpf_insn_aux_data)),
+                                   GFP_KERNEL_ACCOUNT | __GFP_ZERO);
                if (!new_data)
                        return NULL;
+
+               env->insn_aux_data = new_data;
        }
 
        new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
@@ -20780,7 +20781,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of
                vfree(new_data);
                return NULL;
        }
-       adjust_insn_aux_data(env, new_data, new_prog, off, len);
+       adjust_insn_aux_data(env, new_prog, off, len);
        adjust_subprog_starts(env, off, len);
        adjust_poke_descs(new_prog, off, len);
        return new_prog;