x86/ibt,ftrace: Search for __fentry__ location

commit aebfd12521d9c7d0b502cf6d06314cfbcdccfe3b upstream.

Currently a lot of ftrace code assumes __fentry__ is at sym+0. However
with Intel IBT enabled the first instruction of a function will most
likely be ENDBR.

Change ftrace_location() to not only return the __fentry__ location
when called for the __fentry__ location, but also when called for the
sym+0 location.

Then audit/update all callsites of this function to consistently use
these new semantics.

Suggested-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Masami Hiramatsu <mhiramat@kernel.org>
Acked-by: Josh Poimboeuf <jpoimboe@redhat.com>
Link: https://lore.kernel.org/r/20220308154318.227581603@infradead.org
Stable-dep-of: e60b613df8b6 ("ftrace: Fix possible use-after-free issue in ftrace_location()")
[Shivani: Modified to apply on v5.10.y]
Signed-off-by: Shivani Agarwal <shivani.agarwal@broadcom.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index e7edc9e4c6cd999cc49a8d154cfb6604ab3c8c4f..6d59c8e7719b1f182dec975cd360aedbda532b32 100644 (file)
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -194,17 +194,10 @@ static unsigned long
 __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
 {
        struct kprobe *kp;
-       unsigned long faddr;
+       bool faddr;
 
        kp = get_kprobe((void *)addr);
-       faddr = ftrace_location(addr);
-       /*
-        * Addresses inside the ftrace location are refused by
-        * arch_check_ftrace_location(). Something went terribly wrong
-        * if such an address is checked here.
-        */
-       if (WARN_ON(faddr && faddr != addr))
-               return 0UL;
+       faddr = ftrace_location(addr) == addr;
        /*
         * Use the current code if it is not modified by Kprobe
         * and it cannot be modified by ftrace.

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 87becf77cc759f1155cf36fe6ebad225866519ed..0a14f14d83fe2a4bbb9472e2ff0cbc10e4c8d4c9 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -87,18 +87,6 @@ out:
        return tr;
 }
 
-static int is_ftrace_location(void *ip)
-{
-       long addr;
-
-       addr = ftrace_location((long)ip);
-       if (!addr)
-               return 0;
-       if (WARN_ON_ONCE(addr != (long)ip))
-               return -EFAULT;
-       return 1;
-}
-
 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
 {
        void *ip = tr->func.addr;
@@ -127,12 +115,12 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_ad
 static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
 {
        void *ip = tr->func.addr;
+       unsigned long faddr;
        int ret;
 
-       ret = is_ftrace_location(ip);
-       if (ret < 0)
-               return ret;
-       tr->func.ftrace_managed = ret;
+       faddr = ftrace_location((unsigned long)ip);
+       if (faddr)
+               tr->func.ftrace_managed = true;
 
        if (tr->func.ftrace_managed)
                ret = register_ftrace_direct((long)ip, (long)new_addr);

diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index c8e62458d323f205cba9211693a9497614012390..551ac118159f9e36c24b8ded07b36d3d60a3a107 100644 (file)
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -1609,14 +1609,10 @@ static inline int check_kprobe_rereg(struct kprobe *p)
 
 int __weak arch_check_ftrace_location(struct kprobe *p)
 {
-       unsigned long ftrace_addr;
+       unsigned long addr = (unsigned long)p->addr;
 
-       ftrace_addr = ftrace_location((unsigned long)p->addr);
-       if (ftrace_addr) {
+       if (ftrace_location(addr) == addr) {
 #ifdef CONFIG_KPROBES_ON_FTRACE
-               /* Given address is not on the instruction boundary */
-               if ((unsigned long)p->addr != ftrace_addr)
-                       return -EILSEQ;
                p->flags |= KPROBE_FLAG_FTRACE;
 #else  /* !CONFIG_KPROBES_ON_FTRACE */
                return -EINVAL;

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 31fec924b7c48c82b506b1cfe7ba2f0eef3fd8a1..a781733b2a0120432a6c6cdac75aa86f89b29e15 100644 (file)
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -1575,17 +1575,34 @@ unsigned long ftrace_location_range(unsigned long start, unsigned long end)
 }
 
 /**
- * ftrace_location - return true if the ip giving is a traced location
+ * ftrace_location - return the ftrace location
  * @ip: the instruction pointer to check
  *
- * Returns rec->ip if @ip given is a pointer to a ftrace location.
- * That is, the instruction that is either a NOP or call to
- * the function tracer. It checks the ftrace internal tables to
- * determine if the address belongs or not.
+ * If @ip matches the ftrace location, return @ip.
+ * If @ip matches sym+0, return sym's ftrace location.
+ * Otherwise, return 0.
  */
 unsigned long ftrace_location(unsigned long ip)
 {
-       return ftrace_location_range(ip, ip);
+       struct dyn_ftrace *rec;
+       unsigned long offset;
+       unsigned long size;
+
+       rec = lookup_rec(ip, ip);
+       if (!rec) {
+               if (!kallsyms_lookup_size_offset(ip, &size, &offset))
+                       goto out;
+
+               /* map sym+0 to __fentry__ */
+               if (!offset)
+                       rec = lookup_rec(ip, ip + size - 1);
+       }
+
+       if (rec)
+               return rec->ip;
+
+out:
+       return 0;
 }
 
 /**
@@ -4948,7 +4965,8 @@ ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
 {
        struct ftrace_func_entry *entry;
 
-       if (!ftrace_location(ip))
+       ip = ftrace_location(ip);
+       if (!ip)
                return -EINVAL;
 
        if (remove) {
@@ -5096,11 +5114,16 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr)
        struct ftrace_func_entry *entry;
        struct ftrace_hash *free_hash = NULL;
        struct dyn_ftrace *rec;
-       int ret = -EBUSY;
+       int ret = -ENODEV;
 
        mutex_lock(&direct_mutex);
 
+       ip = ftrace_location(ip);
+       if (!ip)
+               goto out_unlock;
+
        /* See if there's a direct function at @ip already */
+       ret = -EBUSY;
        if (ftrace_find_rec_direct(ip))
                goto out_unlock;
 
@@ -5229,6 +5252,10 @@ int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
 
        mutex_lock(&direct_mutex);
 
+       ip = ftrace_location(ip);
+       if (!ip)
+               goto out_unlock;
+
        entry = find_direct_entry(&ip, NULL);
        if (!entry)
                goto out_unlock;
@@ -5360,6 +5387,11 @@ int modify_ftrace_direct(unsigned long ip,
        mutex_lock(&direct_mutex);
 
        mutex_lock(&ftrace_lock);
+
+       ip = ftrace_location(ip);
+       if (!ip)
+               goto out_unlock;
+
        entry = find_direct_entry(&ip, &rec);
        if (!entry)
                goto out_unlock;