LoongArch: BPF: Add BPF exception tables

Inspired by commit 800834285361("bpf, arm64: Add BPF exception tables"),
do similar to LoongArch to add BPF exception tables.

When a tracing BPF program attempts to read memory without using the
bpf_probe_read() helper, the verifier marks the load instruction with
the BPF_PROBE_MEM flag. Since the LoongArch JIT does not currently
recognize this flag it falls back to the interpreter.

Add support for BPF_PROBE_MEM, by appending an exception table to the
BPF program. If the load instruction causes a data abort, the fixup
infrastructure finds the exception table and fixes up the fault, by
clearing the destination register and jumping over the faulting
instruction.

To keep the compact exception table entry format, inspect the pc in
fixup_exception(). A more generic solution would add a "handler" field
to the table entry, like on x86, s390 and arm64, etc.

Signed-off-by: Youling Tang <tangyouling@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

diff --git a/arch/loongarch/include/asm/asm-extable.h b/arch/loongarch/include/asm/asm-extable.h
index f5502cb50c6efc8952489ced77dcedcc0a4e1a4d..df05005f2b80a22bbb8cb205335e1f0e7d15031e 100644 (file)
--- a/arch/loongarch/include/asm/asm-extable.h
+++ b/arch/loongarch/include/asm/asm-extable.h
@@ -5,6 +5,7 @@
 #define EX_TYPE_NONE                   0
 #define EX_TYPE_FIXUP                  1
 #define EX_TYPE_UACCESS_ERR_ZERO       2
+#define EX_TYPE_BPF                    3
 
 #ifdef __ASSEMBLY__
 

diff --git a/arch/loongarch/include/asm/extable.h b/arch/loongarch/include/asm/extable.h
index 92612b4364a1d46cd0fb994ed3ae30152835d843..5abf29f1bc9196be6aef78ed57243b21da6415fd 100644 (file)
--- a/arch/loongarch/include/asm/extable.h
+++ b/arch/loongarch/include/asm/extable.h
@@ -32,6 +32,16 @@ do {                                                 \
        (b)->data = (tmp).data;                         \
 } while (0)
 
+#ifdef CONFIG_BPF_JIT
+bool ex_handler_bpf(const struct exception_table_entry *ex, struct pt_regs *regs);
+#else
+static inline
+bool ex_handler_bpf(const struct exception_table_entry *ex, struct pt_regs *regs)
+{
+       return false;
+}
+#endif /* !CONFIG_BPF_JIT */
+
 bool fixup_exception(struct pt_regs *regs);
 
 #endif

diff --git a/arch/loongarch/mm/extable.c b/arch/loongarch/mm/extable.c
index 9b0cfd8989408eb84f0917341a7355d8e8e09848..9ab69872dcffea2c6e34a568e4f23aec9041a624 100644 (file)
--- a/arch/loongarch/mm/extable.c
+++ b/arch/loongarch/mm/extable.c
@@ -55,6 +55,8 @@ bool fixup_exception(struct pt_regs *regs)
                return ex_handler_fixup(ex, regs);
        case EX_TYPE_UACCESS_ERR_ZERO:
                return ex_handler_uaccess_err_zero(ex, regs);
+       case EX_TYPE_BPF:
+               return ex_handler_bpf(ex, regs);
        }
 
        BUG();

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index bdcd0c7719a9eedd3145e4d4cbb8a07508806d11..c4b1947ebf768fb222e23fdd9a93796c83690dad 100644 (file)
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -387,6 +387,65 @@ static bool is_signed_bpf_cond(u8 cond)
               cond == BPF_JSGE || cond == BPF_JSLE;
 }
 
+#define BPF_FIXUP_REG_MASK     GENMASK(31, 27)
+#define BPF_FIXUP_OFFSET_MASK  GENMASK(26, 0)
+
+bool ex_handler_bpf(const struct exception_table_entry *ex,
+                   struct pt_regs *regs)
+{
+       int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
+       off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
+
+       regs->regs[dst_reg] = 0;
+       regs->csr_era = (unsigned long)&ex->fixup - offset;
+
+       return true;
+}
+
+/* For accesses to BTF pointers, add an entry to the exception table */
+static int add_exception_handler(const struct bpf_insn *insn,
+                                struct jit_ctx *ctx,
+                                int dst_reg)
+{
+       unsigned long pc;
+       off_t offset;
+       struct exception_table_entry *ex;
+
+       if (!ctx->image || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM)
+               return 0;
+
+       if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
+               return -EINVAL;
+
+       ex = &ctx->prog->aux->extable[ctx->num_exentries];
+       pc = (unsigned long)&ctx->image[ctx->idx - 1];
+
+       offset = pc - (long)&ex->insn;
+       if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
+               return -ERANGE;
+
+       ex->insn = offset;
+
+       /*
+        * Since the extable follows the program, the fixup offset is always
+        * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
+        * to keep things simple, and put the destination register in the upper
+        * bits. We don't need to worry about buildtime or runtime sort
+        * modifying the upper bits because the table is already sorted, and
+        * isn't part of the main exception table.
+        */
+       offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
+       if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
+               return -ERANGE;
+
+       ex->type = EX_TYPE_BPF;
+       ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
+
+       ctx->num_exentries++;
+
+       return 0;
+}
+
 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
 {
        u8 tm = -1;
@@ -816,6 +875,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
        case BPF_LDX | BPF_MEM | BPF_H:
        case BPF_LDX | BPF_MEM | BPF_W:
        case BPF_LDX | BPF_MEM | BPF_DW:
+       case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
+       case BPF_LDX | BPF_PROBE_MEM | BPF_W:
+       case BPF_LDX | BPF_PROBE_MEM | BPF_H:
+       case BPF_LDX | BPF_PROBE_MEM | BPF_B:
                switch (BPF_SIZE(code)) {
                case BPF_B:
                        if (is_signed_imm12(off)) {
@@ -854,6 +917,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
                        }
                        break;
                }
+
+               ret = add_exception_handler(insn, ctx, dst);
+               if (ret)
+                       return ret;
                break;
 
        /* *(size *)(dst + off) = imm */
@@ -1018,6 +1085,9 @@ static int validate_code(struct jit_ctx *ctx)
                        return -1;
        }
 
+       if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
+               return -1;
+
        return 0;
 }
 
@@ -1025,7 +1095,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 {
        bool tmp_blinded = false, extra_pass = false;
        u8 *image_ptr;
-       int image_size;
+       int image_size, prog_size, extable_size;
        struct jit_ctx ctx;
        struct jit_data *jit_data;
        struct bpf_binary_header *header;
@@ -1066,7 +1136,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
                image_ptr = jit_data->image;
                header = jit_data->header;
                extra_pass = true;
-               image_size = sizeof(u32) * ctx.idx;
+               prog_size = sizeof(u32) * ctx.idx;
                goto skip_init_ctx;
        }
 
@@ -1088,12 +1158,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
        ctx.epilogue_offset = ctx.idx;
        build_epilogue(&ctx);
 
+       extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
+
        /* Now we know the actual image size.
         * As each LoongArch instruction is of length 32bit,
         * we are translating number of JITed intructions into
         * the size required to store these JITed code.
         */
-       image_size = sizeof(u32) * ctx.idx;
+       prog_size = sizeof(u32) * ctx.idx;
+       image_size = prog_size + extable_size;
        /* Now we know the size of the structure to make */
        header = bpf_jit_binary_alloc(image_size, &image_ptr,
                                      sizeof(u32), jit_fill_hole);
@@ -1104,9 +1177,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 
        /* 2. Now, the actual pass to generate final JIT code */
        ctx.image = (union loongarch_instruction *)image_ptr;
+       if (extable_size)
+               prog->aux->extable = (void *)image_ptr + prog_size;
 
 skip_init_ctx:
        ctx.idx = 0;
+       ctx.num_exentries = 0;
 
        build_prologue(&ctx);
        if (build_body(&ctx, extra_pass)) {
@@ -1125,7 +1201,7 @@ skip_init_ctx:
 
        /* And we're done */
        if (bpf_jit_enable > 1)
-               bpf_jit_dump(prog->len, image_size, 2, ctx.image);
+               bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
 
        /* Update the icache */
        flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
@@ -1147,7 +1223,7 @@ skip_init_ctx:
                jit_data->header = header;
        }
        prog->jited = 1;
-       prog->jited_len = image_size;
+       prog->jited_len = prog_size;
        prog->bpf_func = (void *)ctx.image;
 
        if (!prog->is_func || extra_pass) {

diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h
index e665ddb0aeb85580d583098c88fd6995e1353da1..ca708024fdd3e61f6859b76c125d2bda64989d21 100644 (file)
--- a/arch/loongarch/net/bpf_jit.h
+++ b/arch/loongarch/net/bpf_jit.h
@@ -4,6 +4,7 @@
  *
  * Copyright (C) 2022 Loongson Technology Corporation Limited
  */
+#include <linux/bitfield.h>
 #include <linux/bpf.h>
 #include <linux/filter.h>
 #include <asm/cacheflush.h>
@@ -15,6 +16,7 @@ struct jit_ctx {
        unsigned int flags;
        unsigned int epilogue_offset;
        u32 *offset;
+       int num_exentries;
        union loongarch_instruction *image;
        u32 stack_size;
 };