bpf: Add gen_epilogue to bpf_verifier_ops

This patch adds a .gen_epilogue to the bpf_verifier_ops. It is similar
to the existing .gen_prologue. Instead of allowing a subsystem
to run code at the beginning of a bpf prog, it allows the subsystem
to run code just before the bpf prog exit.

One of the use case is to allow the upcoming bpf qdisc to ensure that
the skb->dev is the same as the qdisc->dev_queue->dev. The bpf qdisc
struct_ops implementation could either fix it up or drop the skb.
Another use case could be in bpf_tcp_ca.c to enforce snd_cwnd
has sane value (e.g. non zero).

The epilogue can do the useful thing (like checking skb->dev) if it
can access the bpf prog's ctx. Unlike prologue, r1 may not hold the
ctx pointer. This patch saves the r1 in the stack if the .gen_epilogue
has returned some instructions in the "epilogue_buf".

The existing .gen_prologue is done in convert_ctx_accesses().
The new .gen_epilogue is done in the convert_ctx_accesses() also.
When it sees the (BPF_JMP | BPF_EXIT) instruction, it will be patched
with the earlier generated "epilogue_buf". The epilogue patching is
only done for the main prog.

Only one epilogue will be patched to the main program. When the
bpf prog has multiple BPF_EXIT instructions, a BPF_JA is used
to goto the earlier patched epilogue. Majority of the archs
support (BPF_JMP32 | BPF_JA): x86, arm, s390, risv64, loongarch,
powerpc and arc. This patch keeps it simple and always
use (BPF_JMP32 | BPF_JA). A new macro BPF_JMP32_A is added to
generate the (BPF_JMP32 | BPF_JA) insn.

Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20240829210833.388152-4-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index dc63083f76b7c9f06a657db3040b81a14d079e96..6f87fb014fba943daf3cb6c37ed0046876772001 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -974,6 +974,8 @@ struct bpf_verifier_ops {
                                struct bpf_insn_access_aux *info);
        int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
                            const struct bpf_prog *prog);
+       int (*gen_epilogue)(struct bpf_insn *insn, const struct bpf_prog *prog,
+                           s16 ctx_stack_off);
        int (*gen_ld_abs)(const struct bpf_insn *orig,
                          struct bpf_insn *insn_buf);
        u32 (*convert_ctx_access)(enum bpf_access_type type,

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 0ad2d189c546d2a8bda16629c27d94b2dc4de551..2e20207315a9d6cb1617d51adca46f803a386141 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -783,6 +783,7 @@ struct bpf_verifier_env {
         */
        char tmp_str_buf[TMP_STR_BUF_LEN];
        struct bpf_insn insn_buf[INSN_BUF_SIZE];
+       struct bpf_insn epilogue_buf[INSN_BUF_SIZE];
 };
 
 static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)

diff --git a/include/linux/filter.h b/include/linux/filter.h
index b6672ff61407fc8dd56e4555e30f1a3c9dc238dc..99b6fc83825b554fae33a4e2a6d5c72c787c6991 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -437,6 +437,16 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
                .off   = OFF,                                   \
                .imm   = 0 })
 
+/* Unconditional jumps, gotol pc + imm32 */
+
+#define BPF_JMP32_A(IMM)                                       \
+       ((struct bpf_insn) {                                    \
+               .code  = BPF_JMP32 | BPF_JA,                    \
+               .dst_reg = 0,                                   \
+               .src_reg = 0,                                   \
+               .off   = 0,                                     \
+               .imm   = IMM })
+
 /* Relative call */
 
 #define BPF_CALL_REL(TGT)                                      \

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 03e974129c05f7d803b9dcd5cedd9b862b3d15dd..fdb1d39c8b58ae299f6155e7f238fd854417caf9 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -19677,15 +19677,39 @@ apply_patch_buffer:
  */
 static int convert_ctx_accesses(struct bpf_verifier_env *env)
 {
+       struct bpf_subprog_info *subprogs = env->subprog_info;
        const struct bpf_verifier_ops *ops = env->ops;
-       int i, cnt, size, ctx_field_size, delta = 0;
+       int i, cnt, size, ctx_field_size, delta = 0, epilogue_cnt = 0;
        const int insn_cnt = env->prog->len;
+       struct bpf_insn *epilogue_buf = env->epilogue_buf;
        struct bpf_insn *insn_buf = env->insn_buf;
        struct bpf_insn *insn;
        u32 target_size, size_default, off;
        struct bpf_prog *new_prog;
        enum bpf_access_type type;
        bool is_narrower_load;
+       int epilogue_idx = 0;
+
+       if (ops->gen_epilogue) {
+               epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,
+                                                -(subprogs[0].stack_depth + 8));
+               if (epilogue_cnt >= INSN_BUF_SIZE) {
+                       verbose(env, "bpf verifier is misconfigured\n");
+                       return -EINVAL;
+               } else if (epilogue_cnt) {
+                       /* Save the ARG_PTR_TO_CTX for the epilogue to use */
+                       cnt = 0;
+                       subprogs[0].stack_depth += 8;
+                       insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
+                                                     -subprogs[0].stack_depth);
+                       insn_buf[cnt++] = env->prog->insnsi[0];
+                       new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
+                       if (!new_prog)
+                               return -ENOMEM;
+                       env->prog = new_prog;
+                       delta += cnt - 1;
+               }
+       }
 
        if (ops->gen_prologue || env->seen_direct_write) {
                if (!ops->gen_prologue) {
@@ -19742,6 +19766,25 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
                        insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code);
                        env->prog->aux->num_exentries++;
                        continue;
+               } else if (insn->code == (BPF_JMP | BPF_EXIT) &&
+                          epilogue_cnt &&
+                          i + delta < subprogs[1].start) {
+                       /* Generate epilogue for the main prog */
+                       if (epilogue_idx) {
+                               /* jump back to the earlier generated epilogue */
+                               insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1);
+                               cnt = 1;
+                       } else {
+                               memcpy(insn_buf, epilogue_buf,
+                                      epilogue_cnt * sizeof(*epilogue_buf));
+                               cnt = epilogue_cnt;
+                               /* epilogue_idx cannot be 0. It must have at
+                                * least one ctx ptr saving insn before the
+                                * epilogue.
+                                */
+                                epilogue_idx = i + delta;
+                       }
+                       goto patch_insn_buf;
                } else {
                        continue;
                }
@@ -19878,6 +19921,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
                                                       insn->dst_reg, insn->dst_reg,
                                                       size * 8, 0);
 
+patch_insn_buf:
                new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                if (!new_prog)
                        return -ENOMEM;