riscv, bpf: Optimize cmpxchg insn with Zacas support

Optimize cmpxchg instruction with amocas.w and amocas.d
Zacas instructions.

Signed-off-by: Pu Lehui <pulehui@huawei.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Tested-by: Björn Töpel <bjorn@rivosinc.com>
Reviewed-by: Björn Töpel <bjorn@rivosinc.com>
Acked-by: Björn Töpel <bjorn@kernel.org>
Link: https://lore.kernel.org/bpf/20250719091730.2660197-8-pulehui@huaweicloud.com

diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h
index 2351fba5d3e7c0969dfdeebe7140df7008ca2d83..0790f40b7e9dbf00b8fd1c485e8947a81f29fc81 100644 (file)
--- a/arch/riscv/net/bpf_jit.h
+++ b/arch/riscv/net/bpf_jit.h
@@ -1294,6 +1294,33 @@ out_be:
        emit_mv(rd, RV_REG_T2, ctx);
 }
 
+static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_context *ctx)
+{
+       int jmp_offset;
+
+       if (rv_ext_enabled(ZACAS)) {
+               emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) :
+                    rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx);
+               if (!is64)
+                       emit_zextw(r0, r0, ctx);
+               return;
+       }
+
+       if (is64)
+               emit_mv(RV_REG_T2, r0, ctx);
+       else
+               emit_addiw(RV_REG_T2, r0, 0, ctx);
+       emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
+            rv_lr_w(r0, 0, rd, 0, 0), ctx);
+       jmp_offset = ninsns_rvoff(8);
+       emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
+       emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) :
+            rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
+       jmp_offset = ninsns_rvoff(-6);
+       emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
+       emit_fence_rw_rw(ctx);
+}
+
 #endif /* __riscv_xlen == 64 */
 
 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog);

diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index a6a9fd9193e5e42833f8999767092c1214880ed5..8e813809d3054ee61baa569644d6a445066f5940 100644 (file)
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -599,10 +599,9 @@ static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
 static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
                           struct rv_jit_context *ctx)
 {
-       u8 r0, code = insn->code;
+       u8 code = insn->code;
        s16 off = insn->off;
        s32 imm = insn->imm;
-       int jmp_offset;
        bool is64;
 
        if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) {
@@ -673,20 +672,7 @@ static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
                break;
        /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */
        case BPF_CMPXCHG:
-               r0 = bpf_to_rv_reg(BPF_REG_0, ctx);
-               if (is64)
-                       emit_mv(RV_REG_T2, r0, ctx);
-               else
-                       emit_addiw(RV_REG_T2, r0, 0, ctx);
-               emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
-                    rv_lr_w(r0, 0, rd, 0, 0), ctx);
-               jmp_offset = ninsns_rvoff(8);
-               emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
-               emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) :
-                    rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
-               jmp_offset = ninsns_rvoff(-6);
-               emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
-               emit_fence_rw_rw(ctx);
+               emit_cmpxchg(rd, rs, regmap[BPF_REG_0], is64, ctx);
                break;
        default:
                pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm);