bpf: Add multi kprobe link

Adding new link type BPF_LINK_TYPE_KPROBE_MULTI that attaches kprobe
program through fprobe API.

The fprobe API allows to attach probe on multiple functions at once
very fast, because it works on top of ftrace. On the other hand this
limits the probe point to the function entry or return.

The kprobe program gets the same pt_regs input ctx as when it's attached
through the perf API.

Adding new attach type BPF_TRACE_KPROBE_MULTI that allows attachment
kprobe to multiple function with new link.

User provides array of addresses or symbols with count to attach the
kprobe program to. The new link_create uapi interface looks like:

  struct {
          __u32           flags;
          __u32           cnt;
          __aligned_u64   syms;
          __aligned_u64   addrs;
  } kprobe_multi;

The flags field allows single BPF_TRACE_KPROBE_MULTI bit to create
return multi kprobe.

Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20220316122419.933957-4-jolsa@kernel.org

diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
index 48a91c51c015d7fa786c85f528ec38b5707992af..3e24ad0c4b3c1ae9cc9f77d0e8c464c476d1d232 100644 (file)
--- a/include/linux/bpf_types.h
+++ b/include/linux/bpf_types.h
@@ -140,3 +140,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp)
 #ifdef CONFIG_PERF_EVENTS
 BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)
 #endif
+BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)

diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
index dcea51fb60e2761c9ce468b1a2eddb78ed0919dd..8f0e9e7cb4937c000a592249a1312efb06a424b1 100644 (file)
--- a/include/linux/trace_events.h
+++ b/include/linux/trace_events.h
@@ -15,6 +15,7 @@ struct array_buffer;
 struct tracer;
 struct dentry;
 struct bpf_prog;
+union bpf_attr;
 
 const char *trace_print_flags_seq(struct trace_seq *p, const char *delim,
                                  unsigned long flags,
@@ -738,6 +739,7 @@ void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);
 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
                            u32 *fd_type, const char **buf,
                            u64 *probe_offset, u64 *probe_addr);
+int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
 #else
 static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
 {
@@ -779,6 +781,11 @@ static inline int bpf_get_perf_event_info(const struct perf_event *event,
 {
        return -EOPNOTSUPP;
 }
+static inline int
+bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
+{
+       return -EOPNOTSUPP;
+}
 #endif
 
 enum {

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 99fab54ae9c07b51b41443bc22e92dfa3d91088f..d77f47af77525ea6c8454643d1b15559b8b83c01 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -997,6 +997,7 @@ enum bpf_attach_type {
        BPF_SK_REUSEPORT_SELECT,
        BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
        BPF_PERF_EVENT,
+       BPF_TRACE_KPROBE_MULTI,
        __MAX_BPF_ATTACH_TYPE
 };
 
@@ -1011,6 +1012,7 @@ enum bpf_link_type {
        BPF_LINK_TYPE_NETNS = 5,
        BPF_LINK_TYPE_XDP = 6,
        BPF_LINK_TYPE_PERF_EVENT = 7,
+       BPF_LINK_TYPE_KPROBE_MULTI = 8,
 
        MAX_BPF_LINK_TYPE,
 };
@@ -1118,6 +1120,11 @@ enum bpf_link_type {
  */
 #define BPF_F_XDP_HAS_FRAGS    (1U << 5)
 
+/* link_create.kprobe_multi.flags used in LINK_CREATE command for
+ * BPF_TRACE_KPROBE_MULTI attach type to create return probe.
+ */
+#define BPF_F_KPROBE_MULTI_RETURN      (1U << 0)
+
 /* When BPF ldimm64's insn[0].src_reg != 0 then this can have
  * the following extensions:
  *
@@ -1475,6 +1482,12 @@ union bpf_attr {
                                 */
                                __u64           bpf_cookie;
                        } perf_event;
+                       struct {
+                               __u32           flags;
+                               __u32           cnt;
+                               __aligned_u64   syms;
+                               __aligned_u64   addrs;
+                       } kprobe_multi;
                };
        } link_create;
 

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 9beb585be5a695910148dc168009d79786a827f7..b8bb67ee6c57b86a282a95ecc57e113f8a30236d 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -32,6 +32,7 @@
 #include <linux/bpf-netns.h>
 #include <linux/rcupdate_trace.h>
 #include <linux/memcontrol.h>
+#include <linux/trace_events.h>
 
 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
                          (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
@@ -3022,6 +3023,11 @@ out_put_file:
        fput(perf_file);
        return err;
 }
+#else
+static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
+{
+       return -EOPNOTSUPP;
+}
 #endif /* CONFIG_PERF_EVENTS */
 
 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
@@ -4255,7 +4261,7 @@ static int tracing_bpf_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
        return -EINVAL;
 }
 
-#define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len
+#define BPF_LINK_CREATE_LAST_FIELD link_create.kprobe_multi.addrs
 static int link_create(union bpf_attr *attr, bpfptr_t uattr)
 {
        enum bpf_prog_type ptype;
@@ -4279,7 +4285,6 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
                ret = tracing_bpf_link_attach(attr, uattr, prog);
                goto out;
        case BPF_PROG_TYPE_PERF_EVENT:
-       case BPF_PROG_TYPE_KPROBE:
        case BPF_PROG_TYPE_TRACEPOINT:
                if (attr->link_create.attach_type != BPF_PERF_EVENT) {
                        ret = -EINVAL;
@@ -4287,6 +4292,14 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
                }
                ptype = prog->type;
                break;
+       case BPF_PROG_TYPE_KPROBE:
+               if (attr->link_create.attach_type != BPF_PERF_EVENT &&
+                   attr->link_create.attach_type != BPF_TRACE_KPROBE_MULTI) {
+                       ret = -EINVAL;
+                       goto out;
+               }
+               ptype = prog->type;
+               break;
        default:
                ptype = attach_type_to_prog_type(attr->link_create.attach_type);
                if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) {
@@ -4318,13 +4331,16 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
                ret = bpf_xdp_link_attach(attr, prog);
                break;
 #endif
-#ifdef CONFIG_PERF_EVENTS
        case BPF_PROG_TYPE_PERF_EVENT:
        case BPF_PROG_TYPE_TRACEPOINT:
-       case BPF_PROG_TYPE_KPROBE:
                ret = bpf_perf_link_attach(attr, prog);
                break;
-#endif
+       case BPF_PROG_TYPE_KPROBE:
+               if (attr->link_create.attach_type == BPF_PERF_EVENT)
+                       ret = bpf_perf_link_attach(attr, prog);
+               else
+                       ret = bpf_kprobe_multi_link_attach(attr, prog);
+               break;
        default:
                ret = -EINVAL;
        }

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index a2024ba32a20dc06a0cc6f914d330cace3ea7281..fffa2171fae41321ee47c086e69fd2ad0990aa5a 100644 (file)
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -17,6 +17,7 @@
 #include <linux/error-injection.h>
 #include <linux/btf_ids.h>
 #include <linux/bpf_lsm.h>
+#include <linux/fprobe.h>
 
 #include <net/bpf_sk_storage.h>
 
@@ -2181,3 +2182,213 @@ static int __init bpf_event_init(void)
 
 fs_initcall(bpf_event_init);
 #endif /* CONFIG_MODULES */
+
+#ifdef CONFIG_FPROBE
+struct bpf_kprobe_multi_link {
+       struct bpf_link link;
+       struct fprobe fp;
+       unsigned long *addrs;
+};
+
+static void bpf_kprobe_multi_link_release(struct bpf_link *link)
+{
+       struct bpf_kprobe_multi_link *kmulti_link;
+
+       kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
+       unregister_fprobe(&kmulti_link->fp);
+}
+
+static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link)
+{
+       struct bpf_kprobe_multi_link *kmulti_link;
+
+       kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
+       kvfree(kmulti_link->addrs);
+       kfree(kmulti_link);
+}
+
+static const struct bpf_link_ops bpf_kprobe_multi_link_lops = {
+       .release = bpf_kprobe_multi_link_release,
+       .dealloc = bpf_kprobe_multi_link_dealloc,
+};
+
+static int
+kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,
+                          struct pt_regs *regs)
+{
+       int err;
+
+       if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
+               err = 0;
+               goto out;
+       }
+
+       migrate_disable();
+       rcu_read_lock();
+       err = bpf_prog_run(link->link.prog, regs);
+       rcu_read_unlock();
+       migrate_enable();
+
+ out:
+       __this_cpu_dec(bpf_prog_active);
+       return err;
+}
+
+static void
+kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip,
+                         struct pt_regs *regs)
+{
+       unsigned long saved_ip = instruction_pointer(regs);
+       struct bpf_kprobe_multi_link *link;
+
+       /*
+        * Because fprobe's regs->ip is set to the next instruction of
+        * dynamic-ftrace instruction, correct entry ip must be set, so
+        * that the bpf program can access entry address via regs as same
+        * as kprobes.
+        *
+        * Both kprobe and kretprobe see the entry ip of traced function
+        * as instruction pointer.
+        */
+       instruction_pointer_set(regs, entry_ip);
+
+       link = container_of(fp, struct bpf_kprobe_multi_link, fp);
+       kprobe_multi_link_prog_run(link, regs);
+
+       instruction_pointer_set(regs, saved_ip);
+}
+
+static int
+kprobe_multi_resolve_syms(const void *usyms, u32 cnt,
+                         unsigned long *addrs)
+{
+       unsigned long addr, size;
+       const char **syms;
+       int err = -ENOMEM;
+       unsigned int i;
+       char *func;
+
+       size = cnt * sizeof(*syms);
+       syms = kvzalloc(size, GFP_KERNEL);
+       if (!syms)
+               return -ENOMEM;
+
+       func = kmalloc(KSYM_NAME_LEN, GFP_KERNEL);
+       if (!func)
+               goto error;
+
+       if (copy_from_user(syms, usyms, size)) {
+               err = -EFAULT;
+               goto error;
+       }
+
+       for (i = 0; i < cnt; i++) {
+               err = strncpy_from_user(func, syms[i], KSYM_NAME_LEN);
+               if (err == KSYM_NAME_LEN)
+                       err = -E2BIG;
+               if (err < 0)
+                       goto error;
+               err = -EINVAL;
+               addr = kallsyms_lookup_name(func);
+               if (!addr)
+                       goto error;
+               if (!kallsyms_lookup_size_offset(addr, &size, NULL))
+                       goto error;
+               addr = ftrace_location_range(addr, addr + size - 1);
+               if (!addr)
+                       goto error;
+               addrs[i] = addr;
+       }
+
+       err = 0;
+error:
+       kvfree(syms);
+       kfree(func);
+       return err;
+}
+
+int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
+{
+       struct bpf_kprobe_multi_link *link = NULL;
+       struct bpf_link_primer link_primer;
+       unsigned long *addrs;
+       u32 flags, cnt, size;
+       void __user *uaddrs;
+       void __user *usyms;
+       int err;
+
+       /* no support for 32bit archs yet */
+       if (sizeof(u64) != sizeof(void *))
+               return -EOPNOTSUPP;
+
+       if (prog->expected_attach_type != BPF_TRACE_KPROBE_MULTI)
+               return -EINVAL;
+
+       flags = attr->link_create.kprobe_multi.flags;
+       if (flags & ~BPF_F_KPROBE_MULTI_RETURN)
+               return -EINVAL;
+
+       uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs);
+       usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms);
+       if (!!uaddrs == !!usyms)
+               return -EINVAL;
+
+       cnt = attr->link_create.kprobe_multi.cnt;
+       if (!cnt)
+               return -EINVAL;
+
+       size = cnt * sizeof(*addrs);
+       addrs = kvmalloc(size, GFP_KERNEL);
+       if (!addrs)
+               return -ENOMEM;
+
+       if (uaddrs) {
+               if (copy_from_user(addrs, uaddrs, size)) {
+                       err = -EFAULT;
+                       goto error;
+               }
+       } else {
+               err = kprobe_multi_resolve_syms(usyms, cnt, addrs);
+               if (err)
+                       goto error;
+       }
+
+       link = kzalloc(sizeof(*link), GFP_KERNEL);
+       if (!link) {
+               err = -ENOMEM;
+               goto error;
+       }
+
+       bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI,
+                     &bpf_kprobe_multi_link_lops, prog);
+
+       err = bpf_link_prime(&link->link, &link_primer);
+       if (err)
+               goto error;
+
+       if (flags & BPF_F_KPROBE_MULTI_RETURN)
+               link->fp.exit_handler = kprobe_multi_link_handler;
+       else
+               link->fp.entry_handler = kprobe_multi_link_handler;
+
+       link->addrs = addrs;
+
+       err = register_fprobe_ips(&link->fp, addrs, cnt);
+       if (err) {
+               bpf_link_cleanup(&link_primer);
+               return err;
+       }
+
+       return bpf_link_settle(&link_primer);
+
+error:
+       kfree(link);
+       kvfree(addrs);
+       return err;
+}
+#else /* !CONFIG_FPROBE */
+int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
+{
+       return -EOPNOTSUPP;
+}
+#endif

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 99fab54ae9c07b51b41443bc22e92dfa3d91088f..d77f47af77525ea6c8454643d1b15559b8b83c01 100644 (file)
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -997,6 +997,7 @@ enum bpf_attach_type {
        BPF_SK_REUSEPORT_SELECT,
        BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
        BPF_PERF_EVENT,
+       BPF_TRACE_KPROBE_MULTI,
        __MAX_BPF_ATTACH_TYPE
 };
 
@@ -1011,6 +1012,7 @@ enum bpf_link_type {
        BPF_LINK_TYPE_NETNS = 5,
        BPF_LINK_TYPE_XDP = 6,
        BPF_LINK_TYPE_PERF_EVENT = 7,
+       BPF_LINK_TYPE_KPROBE_MULTI = 8,
 
        MAX_BPF_LINK_TYPE,
 };
@@ -1118,6 +1120,11 @@ enum bpf_link_type {
  */
 #define BPF_F_XDP_HAS_FRAGS    (1U << 5)
 
+/* link_create.kprobe_multi.flags used in LINK_CREATE command for
+ * BPF_TRACE_KPROBE_MULTI attach type to create return probe.
+ */
+#define BPF_F_KPROBE_MULTI_RETURN      (1U << 0)
+
 /* When BPF ldimm64's insn[0].src_reg != 0 then this can have
  * the following extensions:
  *
@@ -1475,6 +1482,12 @@ union bpf_attr {
                                 */
                                __u64           bpf_cookie;
                        } perf_event;
+                       struct {
+                               __u32           flags;
+                               __u32           cnt;
+                               __aligned_u64   syms;
+                               __aligned_u64   addrs;
+                       } kprobe_multi;
                };
        } link_create;