s390/bpf: Inline smp_processor_id and current_task

Inline these calls in bpf jit:
 - bpf_get_smp_processor_id()
 - bpf_get_current_task()
 - bpf_get_current_task_btf()

s390 has a 8 KiB per-CPU prefix area in the CPU's
virtual address space, called the lowcore. It is a
struct that contains the cpu number and a pointer
to the current task. These are exactly the values
returned by the BPF helpers.

Emit a load from the lowcore instead of a helper
function call.

JIT output for `bpf_get_smp_processor_id`:

 Before:                       After:
---------------               ----------------
brasl   %r14,0x3ffe0385460    ly      %r14,928
lgr     %r14,%r2

JIT output for `bpf_get_current_task`:

 Before:                        After:
---------------                ----------------
brasl   %r14,0x3ffe0362a90     lg      %r14,832
lgr     %r14,%r2

Benchmark using [1] on KVM(virtme-ng).

./benchs/run_bench_trigger.sh glob-arr-inc arr-inc hash-inc

+---------------+--------------------+--------------------+--------------+
|     Name      |       Before       |       After        |   % change   |
|---------------+--------------------+--------------------+--------------|
| glob-arr-inc  | 244.954 ± 0.654M/s | 278.501 ± 0.834M/s |   + 13.70%   |
| arr-inc       | 311.597 ± 1.016M/s | 313.610 ± 0.331M/s |   + 0.65%    |
| hash-inc      | 47.421 ± 0.017M/s  | 47.600 ± 0.004M/s  |   + 0.38%    |
+---------------+--------------------+--------------------+--------------+

[1] https://github.com/anakryiko/linux/commit/8dec900975ef

Signed-off-by: Maxim Khmelevskii <max@linux.ibm.com>
Reviewed-by: Ilya Leoshkevich <iii@linux.ibm.com>
Link: https://lore.kernel.org/r/20260414142930.528751-1-max@linux.ibm.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 94128fe6be2334c2ce515e23f69885f1e444ea89..14eaaa5b21851e82caa4be9c80e43abaeb97b655 100644 (file)
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -27,6 +27,7 @@
 #include <asm/extable.h>
 #include <asm/dis.h>
 #include <asm/facility.h>
+#include <asm/lowcore.h>
 #include <asm/nospec-branch.h>
 #include <asm/set_memory.h>
 #include <asm/text-patching.h>
@@ -1777,6 +1778,30 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
                int j, ret;
                u64 func;
 
+               /* Implement helper call to bpf_get_smp_processor_id() inline */
+               if (insn->src_reg == 0 &&
+                   insn->imm == BPF_FUNC_get_smp_processor_id) {
+                       const u32 *cpu_nr = &get_lowcore()->cpu_nr;
+
+                       /* ly %b0, cpu_nr */
+                       EMIT6_DISP_LH(0xe3000000, 0x0058, BPF_REG_0, REG_0, REG_0,
+                                     (unsigned long)cpu_nr);
+                       break;
+               }
+
+               /* Implement helper call to bpf_get_current_task/_btf() inline */
+               if (insn->src_reg == 0 &&
+                   (insn->imm == BPF_FUNC_get_current_task ||
+                    insn->imm == BPF_FUNC_get_current_task_btf)) {
+                       const u64 *current_task =
+                               &get_lowcore()->current_task;
+
+                       /* lg %b0, current_task */
+                       EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_0, REG_0, REG_0,
+                                     (unsigned long)current_task);
+                       break;
+               }
+
                ret = bpf_jit_get_func_addr(fp, insn, extra_pass,
                                            &func, &func_addr_fixed);
                if (ret < 0)
@@ -3057,3 +3082,15 @@ bool bpf_jit_supports_timed_may_goto(void)
 {
        return true;
 }
+
+bool bpf_jit_inlines_helper_call(s32 imm)
+{
+       switch (imm) {
+       case BPF_FUNC_get_smp_processor_id:
+       case BPF_FUNC_get_current_task:
+       case BPF_FUNC_get_current_task_btf:
+               return true;
+       default:
+               return false;
+       }
+}