x86/fgraph,bpf: Fix stack ORC unwind from kprobe_multi return probe

Currently we don't get stack trace via ORC unwinder on top of fgraph exit
handler. We can see that when generating stacktrace from kretprobe_multi
bpf program which is based on fprobe/fgraph.

The reason is that the ORC unwind code won't get pass the return_to_handler
callback installed by fgraph return probe machinery.

Solving this by creating stack frame in return_to_handler expected by
ftrace_graph_ret_addr function to recover original return address and
continue with the unwind.

Also updating the pt_regs data with cs/flags/rsp which are needed for
successful stack retrieval from ebpf bpf_get_stackid helper.
 - in get_perf_callchain we check user_mode(regs) so CS has to be set
 - in perf_callchain_kernel we call perf_hw_regs(regs), so EFLAGS/FIXED
    has to be unset

Acked-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20251104215405.168643-3-jolsa@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>

diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index 93156ac4ffe09826f25d6cb77552c2795d1cef0c..b08c95872eed9fc43d1791173fe4068ae8cb5bb8 100644 (file)
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -56,6 +56,11 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs)
        return &arch_ftrace_regs(fregs)->regs;
 }
 
+#define arch_ftrace_partial_regs(regs) do {    \
+       regs->flags &= ~X86_EFLAGS_FIXED;       \
+       regs->cs = __KERNEL_CS;                 \
+} while (0)
+
 #define arch_ftrace_fill_perf_regs(fregs, _regs) do {  \
                (_regs)->ip = arch_ftrace_regs(fregs)->regs.ip;         \
                (_regs)->sp = arch_ftrace_regs(fregs)->regs.sp;         \

diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index 367da36381678f1a05d781963aedac861548d453..823dbdd0eb41093f47ed71a19ba9b6a01bca5af4 100644 (file)
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -354,12 +354,17 @@ SYM_CODE_START(return_to_handler)
        UNWIND_HINT_UNDEFINED
        ANNOTATE_NOENDBR
 
+       /* Restore return_to_handler value that got eaten by previous ret instruction. */
+       subq $8, %rsp
+       UNWIND_HINT_FUNC
+
        /* Save ftrace_regs for function exit context  */
        subq $(FRAME_SIZE), %rsp
 
        movq %rax, RAX(%rsp)
        movq %rdx, RDX(%rsp)
        movq %rbp, RBP(%rsp)
+       movq %rsp, RSP(%rsp)
        movq %rsp, %rdi
 
        call ftrace_return_to_handler
@@ -368,7 +373,8 @@ SYM_CODE_START(return_to_handler)
        movq RDX(%rsp), %rdx
        movq RAX(%rsp), %rax
 
-       addq $(FRAME_SIZE), %rsp
+       addq $(FRAME_SIZE) + 8, %rsp
+
        /*
         * Jump back to the old return address. This cannot be JMP_NOSPEC rdi
         * since IBT would demand that contain ENDBR, which simply isn't so for

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 7ded7df6e9b504f1162f8c85a851c894420a17f5..07f8c309e432732375367a33223f89f60fc5b40b 100644 (file)
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -193,6 +193,10 @@ static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs
 #if !defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS) || \
        defined(CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS)
 
+#ifndef arch_ftrace_partial_regs
+#define arch_ftrace_partial_regs(regs) do {} while (0)
+#endif
+
 static __always_inline struct pt_regs *
 ftrace_partial_regs(struct ftrace_regs *fregs, struct pt_regs *regs)
 {
@@ -202,7 +206,11 @@ ftrace_partial_regs(struct ftrace_regs *fregs, struct pt_regs *regs)
         * Since arch_ftrace_get_regs() will check some members and may return
         * NULL, we can not use it.
         */
-       return &arch_ftrace_regs(fregs)->regs;
+       regs = &arch_ftrace_regs(fregs)->regs;
+
+       /* Allow arch specific updates to regs. */
+       arch_ftrace_partial_regs(regs);
+       return regs;
 }
 
 #endif /* !CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS || CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS */