bpf, x86: inline bpf_get_current_task() for x86_64

Inline bpf_get_current_task() and bpf_get_current_task_btf() for x86_64
to obtain better performance.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20260120070555.233486-2-dongml2@chinatelecom.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index bca0ca82d164b5661f3623e278d20ad8b521cc29..9905ad40f4d365c4b393106f52c617b1e0d0fcf3 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -18130,6 +18130,10 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
        switch (imm) {
 #ifdef CONFIG_X86_64
        case BPF_FUNC_get_smp_processor_id:
+#ifdef CONFIG_SMP
+       case BPF_FUNC_get_current_task_btf:
+       case BPF_FUNC_get_current_task:
+#endif
                return env->prog->jit_requested && bpf_jit_supports_percpu_insn();
 #endif
        default:
@@ -23715,6 +23719,24 @@ patch_map_ops_generic:
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }
+
+               /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */
+               if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) &&
+                   verifier_inlines_helper_call(env, insn->imm)) {
+                       insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&current_task);
+                       insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
+                       insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
+                       cnt = 3;
+
+                       new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
+                       if (!new_prog)
+                               return -ENOMEM;
+
+                       delta    += cnt - 1;
+                       env->prog = prog = new_prog;
+                       insn      = new_prog->insnsi + i + delta;
+                       goto next_insn;
+               }
 #endif
                /* Implement bpf_get_func_arg inline. */
                if (prog_type == BPF_PROG_TYPE_TRACING &&