bpf: Make update_prog_stats() always_inline

The function update_prog_stats() will be called in the bpf trampoline.
In most cases, it will be optimized by the compiler by making it inline.
However, we can't rely on the compiler all the time, and just make it
__always_inline to reduce the possible overhead.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Link: https://lore.kernel.org/r/20250621045501.101187-1-dongml2@chinatelecom.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index c4b1a98ff7265215afd250b6b2dd2a71fec746e1..b1e358c16eebd89d86a73251ab770431fe608d2a 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -911,27 +911,32 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
        return bpf_prog_start_time();
 }
 
-static void notrace update_prog_stats(struct bpf_prog *prog,
-                                     u64 start)
+static void notrace __update_prog_stats(struct bpf_prog *prog, u64 start)
 {
        struct bpf_prog_stats *stats;
+       unsigned long flags;
+       u64 duration;
 
-       if (static_branch_unlikely(&bpf_stats_enabled_key) &&
-           /* static_key could be enabled in __bpf_prog_enter*
-            * and disabled in __bpf_prog_exit*.
-            * And vice versa.
-            * Hence check that 'start' is valid.
-            */
-           start > NO_START_TIME) {
-               u64 duration = sched_clock() - start;
-               unsigned long flags;
-
-               stats = this_cpu_ptr(prog->stats);
-               flags = u64_stats_update_begin_irqsave(&stats->syncp);
-               u64_stats_inc(&stats->cnt);
-               u64_stats_add(&stats->nsecs, duration);
-               u64_stats_update_end_irqrestore(&stats->syncp, flags);
-       }
+       /*
+        * static_key could be enabled in __bpf_prog_enter* and disabled in
+        * __bpf_prog_exit*. And vice versa. Check that 'start' is valid.
+        */
+       if (start <= NO_START_TIME)
+               return;
+
+       duration = sched_clock() - start;
+       stats = this_cpu_ptr(prog->stats);
+       flags = u64_stats_update_begin_irqsave(&stats->syncp);
+       u64_stats_inc(&stats->cnt);
+       u64_stats_add(&stats->nsecs, duration);
+       u64_stats_update_end_irqrestore(&stats->syncp, flags);
+}
+
+static __always_inline void notrace update_prog_stats(struct bpf_prog *prog,
+                                                     u64 start)
+{
+       if (static_branch_unlikely(&bpf_stats_enabled_key))
+               __update_prog_stats(prog, start);
 }
 
 static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,