unwind_user/x86: Teach FP unwind about start of function

When userspace is interrupted at the start of a function, before we
get a chance to complete the frame, unwind will miss one caller.

X86 has a uprobe specific fixup for this, add bits to the generic
unwinder to support this.

Suggested-by: Jens Remus <jremus@linux.ibm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20251024145156.GM4068168@noisy.programming.kicks-ass.net

diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 745caa6c15a32d0d2c02f0cfe32ed41700ba4138..0cf68ad9dcd0b411984287801c4a957d59320be6 100644 (file)
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -2845,46 +2845,6 @@ static unsigned long get_segment_base(unsigned int segment)
        return get_desc_base(desc);
 }
 
-#ifdef CONFIG_UPROBES
-/*
- * Heuristic-based check if uprobe is installed at the function entry.
- *
- * Under assumption of user code being compiled with frame pointers,
- * `push %rbp/%ebp` is a good indicator that we indeed are.
- *
- * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern.
- * If we get this wrong, captured stack trace might have one extra bogus
- * entry, but the rest of stack trace will still be meaningful.
- */
-static bool is_uprobe_at_func_entry(struct pt_regs *regs)
-{
-       struct arch_uprobe *auprobe;
-
-       if (!current->utask)
-               return false;
-
-       auprobe = current->utask->auprobe;
-       if (!auprobe)
-               return false;
-
-       /* push %rbp/%ebp */
-       if (auprobe->insn[0] == 0x55)
-               return true;
-
-       /* endbr64 (64-bit only) */
-       if (user_64bit_mode(regs) && is_endbr((u32 *)auprobe->insn))
-               return true;
-
-       return false;
-}
-
-#else
-static bool is_uprobe_at_func_entry(struct pt_regs *regs)
-{
-       return false;
-}
-#endif /* CONFIG_UPROBES */
-
 #ifdef CONFIG_IA32_EMULATION
 
 #include <linux/compat.h>

diff --git a/arch/x86/include/asm/unwind_user.h b/arch/x86/include/asm/unwind_user.h
index b166e102d4445ba4d3412619562fcf4247797da2..c4f1ff8874d67439060020f5a5fd1311e4961a38 100644 (file)
--- a/arch/x86/include/asm/unwind_user.h
+++ b/arch/x86/include/asm/unwind_user.h
@@ -3,6 +3,7 @@
 #define _ASM_X86_UNWIND_USER_H
 
 #include <asm/ptrace.h>
+#include <asm/uprobes.h>
 
 #define ARCH_INIT_USER_FP_FRAME(ws)                    \
        .cfa_off        =  2*(ws),                      \
@@ -10,6 +11,12 @@
        .fp_off         = -2*(ws),                      \
        .use_fp         = true,
 
+#define ARCH_INIT_USER_FP_ENTRY_FRAME(ws)              \
+       .cfa_off        =  1*(ws),                      \
+       .ra_off         = -1*(ws),                      \
+       .fp_off         = 0,                            \
+       .use_fp         = false,
+
 static inline int unwind_user_word_size(struct pt_regs *regs)
 {
        /* We can't unwind VM86 stacks */
@@ -22,4 +29,9 @@ static inline int unwind_user_word_size(struct pt_regs *regs)
        return sizeof(long);
 }
 
+static inline bool unwind_user_at_function_start(struct pt_regs *regs)
+{
+       return is_uprobe_at_func_entry(regs);
+}
+
 #endif /* _ASM_X86_UNWIND_USER_H */

diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index 1ee2e5115955cd99371c38009ac0f1f6d12d51cc..362210c7999872c8e6b97b950ee76d4731553a21 100644 (file)
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -62,4 +62,13 @@ struct arch_uprobe_task {
        unsigned int                    saved_tf;
 };
 
+#ifdef CONFIG_UPROBES
+extern bool is_uprobe_at_func_entry(struct pt_regs *regs);
+#else
+static bool is_uprobe_at_func_entry(struct pt_regs *regs)
+{
+       return false;
+}
+#endif /* CONFIG_UPROBES */
+
 #endif /* _ASM_UPROBES_H */

diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index a563e90832d7a632090dd4dd2b97efeba6dfe27f..7be8e361ca55b8e32b7f5d5567311b9510ad53e5 100644 (file)
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -1791,3 +1791,35 @@ bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
        else
                return regs->sp <= ret->stack;
 }
+
+/*
+ * Heuristic-based check if uprobe is installed at the function entry.
+ *
+ * Under assumption of user code being compiled with frame pointers,
+ * `push %rbp/%ebp` is a good indicator that we indeed are.
+ *
+ * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern.
+ * If we get this wrong, captured stack trace might have one extra bogus
+ * entry, but the rest of stack trace will still be meaningful.
+ */
+bool is_uprobe_at_func_entry(struct pt_regs *regs)
+{
+       struct arch_uprobe *auprobe;
+
+       if (!current->utask)
+               return false;
+
+       auprobe = current->utask->auprobe;
+       if (!auprobe)
+               return false;
+
+       /* push %rbp/%ebp */
+       if (auprobe->insn[0] == 0x55)
+               return true;
+
+       /* endbr64 (64-bit only) */
+       if (user_64bit_mode(regs) && is_endbr((u32 *)auprobe->insn))
+               return true;
+
+       return false;
+}

diff --git a/include/linux/unwind_user_types.h b/include/linux/unwind_user_types.h
index 938f7e6233327d2488dba7830584fc7774ca309e..412729a269bcc491e5557b7e0f0b730b8d9e79d9 100644 (file)
--- a/include/linux/unwind_user_types.h
+++ b/include/linux/unwind_user_types.h
@@ -39,6 +39,7 @@ struct unwind_user_state {
        unsigned int                            ws;
        enum unwind_user_type                   current_type;
        unsigned int                            available_types;
+       bool                                    topmost;
        bool                                    done;
 };
 

diff --git a/kernel/unwind/user.c b/kernel/unwind/user.c
index 642871527a1320c85a293691856ca6fd03d44d44..39e2707894447b68d25baa6fc26e4db9492e3e3d 100644 (file)
--- a/kernel/unwind/user.c
+++ b/kernel/unwind/user.c
@@ -26,14 +26,12 @@ get_user_word(unsigned long *word, unsigned long base, int off, unsigned int ws)
        return get_user(*word, addr);
 }
 
-static int unwind_user_next_fp(struct unwind_user_state *state)
+static int unwind_user_next_common(struct unwind_user_state *state,
+                                  const struct unwind_user_frame *frame)
 {
-       const struct unwind_user_frame frame = {
-               ARCH_INIT_USER_FP_FRAME(state->ws)
-       };
        unsigned long cfa, fp, ra;
 
-       if (frame.use_fp) {
+       if (frame->use_fp) {
                if (state->fp < state->sp)
                        return -EINVAL;
                cfa = state->fp;
@@ -42,7 +40,7 @@ static int unwind_user_next_fp(struct unwind_user_state *state)
        }
 
        /* Get the Canonical Frame Address (CFA) */
-       cfa += frame.cfa_off;
+       cfa += frame->cfa_off;
 
        /* stack going in wrong direction? */
        if (cfa <= state->sp)
@@ -53,19 +51,41 @@ static int unwind_user_next_fp(struct unwind_user_state *state)
                return -EINVAL;
 
        /* Find the Return Address (RA) */
-       if (get_user_word(&ra, cfa, frame.ra_off, state->ws))
+       if (get_user_word(&ra, cfa, frame->ra_off, state->ws))
                return -EINVAL;
 
-       if (frame.fp_off && get_user_word(&fp, cfa, frame.fp_off, state->ws))
+       if (frame->fp_off && get_user_word(&fp, cfa, frame->fp_off, state->ws))
                return -EINVAL;
 
        state->ip = ra;
        state->sp = cfa;
-       if (frame.fp_off)
+       if (frame->fp_off)
                state->fp = fp;
+       state->topmost = false;
        return 0;
 }
 
+static int unwind_user_next_fp(struct unwind_user_state *state)
+{
+#ifdef CONFIG_HAVE_UNWIND_USER_FP
+       struct pt_regs *regs = task_pt_regs(current);
+
+       if (state->topmost && unwind_user_at_function_start(regs)) {
+               const struct unwind_user_frame fp_entry_frame = {
+                       ARCH_INIT_USER_FP_ENTRY_FRAME(state->ws)
+               };
+               return unwind_user_next_common(state, &fp_entry_frame);
+       }
+
+       const struct unwind_user_frame fp_frame = {
+               ARCH_INIT_USER_FP_FRAME(state->ws)
+       };
+       return unwind_user_next_common(state, &fp_frame);
+#else
+       return -EINVAL;
+#endif
+}
+
 static int unwind_user_next(struct unwind_user_state *state)
 {
        unsigned long iter_mask = state->available_types;
@@ -118,6 +138,7 @@ static int unwind_user_start(struct unwind_user_state *state)
                state->done = true;
                return -EINVAL;
        }
+       state->topmost = true;
 
        return 0;
 }