arm64: rethook: Add arm64 rethook implementation

Add rethook arm64 implementation. Most of the code has been copied from
kretprobes on arm64.

Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Tested-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/164735287344.1084943.9787335632585653418.stgit@devnote2

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 09b885cc4db531fd18debbb2d3cd43e5aa5cdd58..62cff9d0a5bbbef88fdee1f70a7fca815e3e3e81 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -201,6 +201,7 @@ config ARM64
        select HAVE_SYSCALL_TRACEPOINTS
        select HAVE_KPROBES
        select HAVE_KRETPROBES
+       select HAVE_RETHOOK
        select HAVE_GENERIC_VDSO
        select IOMMU_DMA if IOMMU_SUPPORT
        select IRQ_DOMAIN

diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h
index e77cdef9ca29bba1ca37954c6880d2a48354d9cf..bf04107da97c7ceb5a89daa1ac2b088e58e3611e 100644 (file)
--- a/arch/arm64/include/asm/stacktrace.h
+++ b/arch/arm64/include/asm/stacktrace.h
@@ -58,7 +58,7 @@ struct stackframe {
        DECLARE_BITMAP(stacks_done, __NR_STACK_TYPES);
        unsigned long prev_fp;
        enum stack_type prev_type;
-#ifdef CONFIG_KRETPROBES
+#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK)
        struct llist_node *kr_cur;
 #endif
 };

diff --git a/arch/arm64/kernel/probes/Makefile b/arch/arm64/kernel/probes/Makefile
index 8e4be92e25b17017176b7f4e8a68acb95e934308..24e689f44c32df5db5d802614c040fe894023dd2 100644 (file)
--- a/arch/arm64/kernel/probes/Makefile
+++ b/arch/arm64/kernel/probes/Makefile
@@ -4,3 +4,4 @@ obj-$(CONFIG_KPROBES)           += kprobes.o decode-insn.o      \
                                   simulate-insn.o
 obj-$(CONFIG_UPROBES)          += uprobes.o decode-insn.o      \
                                   simulate-insn.o
+obj-$(CONFIG_RETHOOK)          += rethook.o rethook_trampoline.o

diff --git a/arch/arm64/kernel/probes/rethook.c b/arch/arm64/kernel/probes/rethook.c
new file mode 100644 (file)
index 0000000..edc6b80
--- /dev/null
+++ b/arch/arm64/kernel/probes/rethook.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Generic return hook for arm64.
+ * Most of the code is copied from arch/arm64/kernel/probes/kprobes.c
+ */
+
+#include <linux/kprobes.h>
+#include <linux/rethook.h>
+
+/* This is called from arch_rethook_trampoline() */
+unsigned long __used arch_rethook_trampoline_callback(struct pt_regs *regs)
+{
+       return rethook_trampoline_handler(regs, regs->regs[29]);
+}
+NOKPROBE_SYMBOL(arch_rethook_trampoline_callback);
+
+void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount)
+{
+       rhn->ret_addr = regs->regs[30];
+       rhn->frame = regs->regs[29];
+
+       /* replace return addr (x30) with trampoline */
+       regs->regs[30] = (u64)arch_rethook_trampoline;
+}
+NOKPROBE_SYMBOL(arch_rethook_prepare);

diff --git a/arch/arm64/kernel/probes/rethook_trampoline.S b/arch/arm64/kernel/probes/rethook_trampoline.S
new file mode 100644 (file)
index 0000000..610f520
--- /dev/null
+++ b/arch/arm64/kernel/probes/rethook_trampoline.S
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * trampoline entry and return code for rethook.
+ * Copied from arch/arm64/kernel/probes/kprobes_trampoline.S
+ */
+
+#include <linux/linkage.h>
+#include <asm/asm-offsets.h>
+#include <asm/assembler.h>
+
+       .text
+
+       .macro  save_all_base_regs
+       stp x0, x1, [sp, #S_X0]
+       stp x2, x3, [sp, #S_X2]
+       stp x4, x5, [sp, #S_X4]
+       stp x6, x7, [sp, #S_X6]
+       stp x8, x9, [sp, #S_X8]
+       stp x10, x11, [sp, #S_X10]
+       stp x12, x13, [sp, #S_X12]
+       stp x14, x15, [sp, #S_X14]
+       stp x16, x17, [sp, #S_X16]
+       stp x18, x19, [sp, #S_X18]
+       stp x20, x21, [sp, #S_X20]
+       stp x22, x23, [sp, #S_X22]
+       stp x24, x25, [sp, #S_X24]
+       stp x26, x27, [sp, #S_X26]
+       stp x28, x29, [sp, #S_X28]
+       add x0, sp, #PT_REGS_SIZE
+       stp lr, x0, [sp, #S_LR]
+       /*
+        * Construct a useful saved PSTATE
+        */
+       mrs x0, nzcv
+       mrs x1, daif
+       orr x0, x0, x1
+       mrs x1, CurrentEL
+       orr x0, x0, x1
+       mrs x1, SPSel
+       orr x0, x0, x1
+       stp xzr, x0, [sp, #S_PC]
+       .endm
+
+       .macro  restore_all_base_regs
+       ldr x0, [sp, #S_PSTATE]
+       and x0, x0, #(PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT)
+       msr nzcv, x0
+       ldp x0, x1, [sp, #S_X0]
+       ldp x2, x3, [sp, #S_X2]
+       ldp x4, x5, [sp, #S_X4]
+       ldp x6, x7, [sp, #S_X6]
+       ldp x8, x9, [sp, #S_X8]
+       ldp x10, x11, [sp, #S_X10]
+       ldp x12, x13, [sp, #S_X12]
+       ldp x14, x15, [sp, #S_X14]
+       ldp x16, x17, [sp, #S_X16]
+       ldp x18, x19, [sp, #S_X18]
+       ldp x20, x21, [sp, #S_X20]
+       ldp x22, x23, [sp, #S_X22]
+       ldp x24, x25, [sp, #S_X24]
+       ldp x26, x27, [sp, #S_X26]
+       ldp x28, x29, [sp, #S_X28]
+       .endm
+
+SYM_CODE_START(arch_rethook_trampoline)
+       sub sp, sp, #PT_REGS_SIZE
+
+       save_all_base_regs
+
+       /* Setup a frame pointer. */
+       add x29, sp, #S_FP
+
+       mov x0, sp
+       bl arch_rethook_trampoline_callback
+       /*
+        * Replace trampoline address in lr with actual orig_ret_addr return
+        * address.
+        */
+       mov lr, x0
+
+       /* The frame pointer (x29) is restored with other registers. */
+       restore_all_base_regs
+
+       add sp, sp, #PT_REGS_SIZE
+       ret
+
+SYM_CODE_END(arch_rethook_trampoline)

diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index e4103e085681143e5f285c91d428f152d7013a6d..efb5b85b57c26ae58e9122808bf5b71ab7580b3a 100644 (file)
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -8,6 +8,7 @@
 #include <linux/export.h>
 #include <linux/ftrace.h>
 #include <linux/kprobes.h>
+#include <linux/rethook.h>
 #include <linux/sched.h>
 #include <linux/sched/debug.h>
 #include <linux/sched/task_stack.h>
@@ -38,7 +39,7 @@ static notrace void start_backtrace(struct stackframe *frame, unsigned long fp,
 {
        frame->fp = fp;
        frame->pc = pc;
-#ifdef CONFIG_KRETPROBES
+#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK)
        frame->kr_cur = NULL;
 #endif
 
@@ -138,6 +139,10 @@ static int notrace unwind_frame(struct task_struct *tsk,
        if (is_kretprobe_trampoline(frame->pc))
                frame->pc = kretprobe_find_ret_addr(tsk, (void *)frame->fp, &frame->kr_cur);
 #endif
+#ifdef CONFIG_RETHOOK
+       if (is_rethook_trampoline(frame->pc))
+               frame->pc = rethook_find_ret_addr(tsk, frame->fp, &frame->kr_cur);
+#endif
 
        return 0;
 }