io_uring/bpf-ops: implement loop_step with BPF struct_ops

Introduce io_uring BPF struct ops implementing the loop_step callback,
which will allow BPF to overwrite the default io_uring event loop logic.

The callback takes an io_uring context, the main role of which is to be
passed to io_uring kfuncs. The other argument is a struct iou_loop_params,
which BPF can use to request CQ waiting and communicate other parameters.
See the event loop description in the previous patch for more details.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/98db437651ce64e9cbeb611c60bf5887259db09f.1772109579.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/Kconfig b/io_uring/Kconfig
index a7ae23cf103579f310e771ab9ff99f8fe0cce28a..a283d9e53787fdb1b0146bde8ff715d45d0c720a 100644 (file)
--- a/io_uring/Kconfig
+++ b/io_uring/Kconfig
@@ -14,3 +14,8 @@ config IO_URING_BPF
        def_bool y
        depends on BPF
        depends on NET
+
+config IO_URING_BPF_OPS
+       def_bool y
+       depends on IO_URING
+       depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF

diff --git a/io_uring/Makefile b/io_uring/Makefile
index 1c1f47de32a4cbb666346631bd21a819f7229eea..c54e328d141049845738b806ef61cd861435f715 100644 (file)
--- a/io_uring/Makefile
+++ b/io_uring/Makefile
@@ -25,3 +25,4 @@ obj-$(CONFIG_NET) += net.o cmd_net.o
 obj-$(CONFIG_PROC_FS) += fdinfo.o
 obj-$(CONFIG_IO_URING_MOCK_FILE) += mock_file.o
 obj-$(CONFIG_IO_URING_BPF) += bpf_filter.o
+obj-$(CONFIG_IO_URING_BPF_OPS) += bpf-ops.o

diff --git a/io_uring/bpf-ops.c b/io_uring/bpf-ops.c
new file mode 100644 (file)
index 0000000..975db5a
--- /dev/null
+++ b/io_uring/bpf-ops.c
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/mutex.h>
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+
+#include "io_uring.h"
+#include "register.h"
+#include "bpf-ops.h"
+#include "loop.h"
+
+static const struct btf_type *loop_params_type;
+
+static int io_bpf_ops__loop_step(struct io_ring_ctx *ctx,
+                                struct iou_loop_params *lp)
+{
+       return IOU_LOOP_STOP;
+}
+
+static struct io_uring_bpf_ops io_bpf_ops_stubs = {
+       .loop_step = io_bpf_ops__loop_step,
+};
+
+static bool bpf_io_is_valid_access(int off, int size,
+                                   enum bpf_access_type type,
+                                   const struct bpf_prog *prog,
+                                   struct bpf_insn_access_aux *info)
+{
+       if (type != BPF_READ)
+               return false;
+       if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
+               return false;
+       if (off % size != 0)
+               return false;
+
+       return btf_ctx_access(off, size, type, prog, info);
+}
+
+static int bpf_io_btf_struct_access(struct bpf_verifier_log *log,
+                                   const struct bpf_reg_state *reg, int off,
+                                   int size)
+{
+       const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
+
+       if (t == loop_params_type) {
+               if (off + size <= offsetofend(struct iou_loop_params, cq_wait_idx))
+                       return SCALAR_VALUE;
+       }
+
+       return -EACCES;
+}
+
+static const struct bpf_verifier_ops bpf_io_verifier_ops = {
+       .get_func_proto = bpf_base_func_proto,
+       .is_valid_access = bpf_io_is_valid_access,
+       .btf_struct_access = bpf_io_btf_struct_access,
+};
+
+static const struct btf_type *
+io_lookup_struct_type(struct btf *btf, const char *name)
+{
+       s32 type_id;
+
+       type_id = btf_find_by_name_kind(btf, name, BTF_KIND_STRUCT);
+       if (type_id < 0)
+               return NULL;
+       return btf_type_by_id(btf, type_id);
+}
+
+static int bpf_io_init(struct btf *btf)
+{
+       loop_params_type = io_lookup_struct_type(btf, "iou_loop_params");
+       if (!loop_params_type) {
+               pr_err("io_uring: Failed to locate iou_loop_params\n");
+               return -EINVAL;
+       }
+
+       return 0;
+}
+
+static int bpf_io_check_member(const struct btf_type *t,
+                               const struct btf_member *member,
+                               const struct bpf_prog *prog)
+{
+       return 0;
+}
+
+static int bpf_io_init_member(const struct btf_type *t,
+                              const struct btf_member *member,
+                              void *kdata, const void *udata)
+{
+       return 0;
+}
+
+static int bpf_io_reg(void *kdata, struct bpf_link *link)
+{
+       return -EOPNOTSUPP;
+}
+
+static void bpf_io_unreg(void *kdata, struct bpf_link *link)
+{
+}
+
+static struct bpf_struct_ops bpf_ring_ops = {
+       .verifier_ops = &bpf_io_verifier_ops,
+       .reg = bpf_io_reg,
+       .unreg = bpf_io_unreg,
+       .check_member = bpf_io_check_member,
+       .init_member = bpf_io_init_member,
+       .init = bpf_io_init,
+       .cfi_stubs = &io_bpf_ops_stubs,
+       .name = "io_uring_bpf_ops",
+       .owner = THIS_MODULE,
+};
+
+static int __init io_uring_bpf_init(void)
+{
+       int ret;
+
+       ret = register_bpf_struct_ops(&bpf_ring_ops, io_uring_bpf_ops);
+       if (ret) {
+               pr_err("io_uring: Failed to register struct_ops (%d)\n", ret);
+               return ret;
+       }
+
+       return 0;
+}
+__initcall(io_uring_bpf_init);

diff --git a/io_uring/bpf-ops.h b/io_uring/bpf-ops.h
new file mode 100644 (file)
index 0000000..e8a08ae
--- /dev/null
+++ b/io_uring/bpf-ops.h
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef IOU_BPF_OPS_H
+#define IOU_BPF_OPS_H
+
+#include <linux/io_uring_types.h>
+
+struct io_uring_bpf_ops {
+       int (*loop_step)(struct io_ring_ctx *ctx, struct iou_loop_params *lp);
+
+       __u32 ring_fd;
+       void *priv;
+};
+
+#endif /* IOU_BPF_OPS_H */

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 960d36c49ffe426c69b17fa3a3ba80206710056f..0a80c8e6e6331e173f9f95f7adb3030befe6e0fe 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -87,6 +87,7 @@
 #include "msg_ring.h"
 #include "memmap.h"
 #include "zcrx.h"
+#include "bpf-ops.h"
 
 #include "timeout.h"
 #include "poll.h"