powerpc/ftrace: Add support for DYNAMIC_FTRACE_WITH_CALL_OPS

Implement support for DYNAMIC_FTRACE_WITH_CALL_OPS similar to the
arm64 implementation.

This works by patching-in a pointer to an associated ftrace_ops
structure before each traceable function. If multiple ftrace_ops are
associated with a call site, then a special ftrace_list_ops is used to
enable iterating over all the registered ftrace_ops. If no ftrace_ops
are associated with a call site, then a special ftrace_nop_ops structure
is used to render the ftrace call as a no-op. ftrace trampoline can then
read the associated ftrace_ops for a call site by loading from an offset
from the LR, and branch directly to the associated function.

The primary advantage with this approach is that we don't have to
iterate over all the registered ftrace_ops for call sites that have a
single ftrace_ops registered. This is the equivalent of implementing
support for dynamic ftrace trampolines, which set up a special ftrace
trampoline for each registered ftrace_ops and have individual call sites
branch into those directly.

A secondary advantage is that this gives us a way to add support for
direct ftrace callers without having to resort to using stubs. The
address of the direct call trampoline can be loaded from the ftrace_ops
structure.

To support this, we reserve a nop before each function on 32-bit
powerpc. For 64-bit powerpc, two nops are reserved before each
out-of-line stub. During ftrace activation, we update this location with
the associated ftrace_ops pointer. Then, on ftrace entry, we load from
this location and call into ftrace_ops->func().

For 64-bit powerpc, we ensure that the out-of-line stub area is
doubleword aligned so that ftrace_ops address can be updated atomically.

Signed-off-by: Naveen N Rao <naveen@kernel.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://patch.msgid.link/20241030070850.1361304-15-hbathini@linux.ibm.com

diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 50d418f72ee8e500bf3b54b8bdcf59a6afa955fe..79bd5a375527d9f981fa599eedd99c5f3f87d692 100644 (file)
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -234,6 +234,7 @@ config PPC
        select HAVE_DEBUG_STACKOVERFLOW
        select HAVE_DYNAMIC_FTRACE
        select HAVE_DYNAMIC_FTRACE_WITH_ARGS    if ARCH_USING_PATCHABLE_FUNCTION_ENTRY || MPROFILE_KERNEL || PPC32
+       select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if PPC_FTRACE_OUT_OF_LINE || (PPC32 && ARCH_USING_PATCHABLE_FUNCTION_ENTRY)
        select HAVE_DYNAMIC_FTRACE_WITH_REGS    if ARCH_USING_PATCHABLE_FUNCTION_ENTRY || MPROFILE_KERNEL || PPC32
        select HAVE_EBPF_JIT
        select HAVE_EFFICIENT_UNALIGNED_ACCESS

diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile
index a52167830e8b66c55c87a104a4e18662250cde90..99af7953e8442dc33330edea5f816c868fd55df1 100644 (file)
--- a/arch/powerpc/Makefile
+++ b/arch/powerpc/Makefile
@@ -151,8 +151,12 @@ KBUILD_CPPFLAGS    += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
 ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
 CC_FLAGS_FTRACE := -fpatchable-function-entry=1
 else
+ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS # PPC32 only
+CC_FLAGS_FTRACE := -fpatchable-function-entry=3,1
+else
 CC_FLAGS_FTRACE := -fpatchable-function-entry=2
 endif
+endif
 else
 CC_FLAGS_FTRACE := -pg
 ifdef CONFIG_MPROFILE_KERNEL

diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h
index 28f3590ca78089148c33c6ef5845b03619c4d999..1ad1328cf4e31468219420dd498f2a2d82c3da36 100644 (file)
--- a/arch/powerpc/include/asm/ftrace.h
+++ b/arch/powerpc/include/asm/ftrace.h
@@ -136,8 +136,11 @@ static inline u8 this_cpu_get_ftrace_enabled(void) { return 1; }
 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[];
 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
 struct ftrace_ool_stub {
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+       struct ftrace_ops *ftrace_op;
+#endif
        u32     insn[4];
-};
+} __aligned(sizeof(unsigned long));
 extern struct ftrace_ool_stub ftrace_ool_stub_text_end[], ftrace_ool_stub_text[],
                              ftrace_ool_stub_inittext[];
 extern unsigned int ftrace_ool_stub_text_end_count, ftrace_ool_stub_text_count,

diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 340f69bfcaa7f4ebda49c7377f82afb291e94850..318349f78820e371bb3ff02defcc79a4f02ff321 100644 (file)
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -679,5 +679,9 @@ int main(void)
        DEFINE(FTRACE_OOL_STUB_SIZE, sizeof(struct ftrace_ool_stub));
 #endif
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+       OFFSET(FTRACE_OPS_FUNC, ftrace_ops, func);
+#endif
+
        return 0;
 }

diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c
index bee2c54a8c047d927d8bcd9bceccc3e5ab3d9ad5..9090d1a21600e96d8c0449b3d8e728e63962d515 100644 (file)
--- a/arch/powerpc/kernel/trace/ftrace.c
+++ b/arch/powerpc/kernel/trace/ftrace.c
@@ -38,8 +38,11 @@ unsigned long ftrace_call_adjust(unsigned long addr)
                return 0;
 
        if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY) &&
-           !IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
+           !IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
                addr += MCOUNT_INSN_SIZE;
+               if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
+                       addr += MCOUNT_INSN_SIZE;
+       }
 
        return addr;
 }
@@ -264,6 +267,46 @@ static int ftrace_init_ool_stub(struct module *mod, struct dyn_ftrace *rec)
 #endif
 }
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+static const struct ftrace_ops *powerpc_rec_get_ops(struct dyn_ftrace *rec)
+{
+       const struct ftrace_ops *ops = NULL;
+
+       if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
+               ops = ftrace_find_unique_ops(rec);
+               WARN_ON_ONCE(!ops);
+       }
+
+       if (!ops)
+               ops = &ftrace_list_ops;
+
+       return ops;
+}
+
+static int ftrace_rec_set_ops(struct dyn_ftrace *rec, const struct ftrace_ops *ops)
+{
+       if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
+               return patch_ulong((void *)(ftrace_get_ool_stub(rec) - sizeof(unsigned long)),
+                                  (unsigned long)ops);
+       else
+               return patch_ulong((void *)(rec->ip - MCOUNT_INSN_SIZE - sizeof(unsigned long)),
+                                  (unsigned long)ops);
+}
+
+static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec)
+{
+       return ftrace_rec_set_ops(rec, &ftrace_nop_ops);
+}
+
+static int ftrace_rec_update_ops(struct dyn_ftrace *rec)
+{
+       return ftrace_rec_set_ops(rec, powerpc_rec_get_ops(rec));
+}
+#else
+static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; }
+static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; }
+#endif
+
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr)
 {
@@ -294,6 +337,10 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
        if (!ret)
                ret = ftrace_modify_code(ip, old, new);
 
+       ret = ftrace_rec_update_ops(rec);
+       if (ret)
+               return ret;
+
        if (!ret && IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
                ret = ftrace_modify_code(rec->ip, ppc_inst(PPC_RAW_NOP()),
                         ppc_inst(PPC_RAW_BRANCH((long)ftrace_get_ool_stub(rec) - (long)rec->ip)));
@@ -345,16 +392,19 @@ void ftrace_replace_code(int enable)
                case FTRACE_UPDATE_MODIFY_CALL:
                        ret = ftrace_get_call_inst(rec, new_addr, &new_call_inst);
                        ret |= ftrace_get_call_inst(rec, addr, &call_inst);
+                       ret |= ftrace_rec_update_ops(rec);
                        old = call_inst;
                        new = new_call_inst;
                        break;
                case FTRACE_UPDATE_MAKE_NOP:
                        ret = ftrace_get_call_inst(rec, addr, &call_inst);
+                       ret |= ftrace_rec_set_nop_ops(rec);
                        old = call_inst;
                        new = nop_inst;
                        break;
                case FTRACE_UPDATE_MAKE_CALL:
                        ret = ftrace_get_call_inst(rec, new_addr, &call_inst);
+                       ret |= ftrace_rec_update_ops(rec);
                        old = nop_inst;
                        new = call_inst;
                        break;
@@ -470,6 +520,13 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
        ppc_inst_t old, new;
        int ret;
 
+       /*
+        * When using CALL_OPS, the function to call is associated with the
+        * call site, and we don't have a global function pointer to update.
+        */
+       if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
+               return 0;
+
        old = ppc_inst_read((u32 *)&ftrace_call);
        new = ftrace_create_branch_inst(ip, ppc_function_entry(func), 1);
        ret = ftrace_modify_code(ip, old, new);

diff --git a/arch/powerpc/kernel/trace/ftrace_entry.S b/arch/powerpc/kernel/trace/ftrace_entry.S
index a6bf7f841040375750f5f2af61873407d401941a..ff376c9903081b80bd5f6c238560f209776da32a 100644 (file)
--- a/arch/powerpc/kernel/trace/ftrace_entry.S
+++ b/arch/powerpc/kernel/trace/ftrace_entry.S
@@ -85,11 +85,21 @@
        /* Save callee's TOC in the ABI compliant location */
        std     r2, STK_GOT(r1)
        LOAD_PACA_TOC()         /* get kernel TOC in r2 */
+#endif
+
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+       /* r7 points to the instruction following the call to ftrace */
+       PPC_LL  r5, -(MCOUNT_INSN_SIZE*2 + SZL)(r7)
+       PPC_LL  r12, FTRACE_OPS_FUNC(r5)
+       mtctr   r12
+#else /* !CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS */
+#ifdef CONFIG_PPC64
        LOAD_REG_ADDR(r3, function_trace_op)
        ld      r5,0(r3)
 #else
        lis     r3,function_trace_op@ha
        lwz     r5,function_trace_op@l(r3)
+#endif
 #endif
 
        /* Save special regs */
@@ -205,20 +215,30 @@
 #endif
 .endm
 
-_GLOBAL(ftrace_regs_caller)
-       ftrace_regs_entry 1
-       /* ftrace_call(r3, r4, r5, r6) */
+.macro ftrace_regs_func allregs
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+       bctrl
+#else
+       .if \allregs == 1
 .globl ftrace_regs_call
 ftrace_regs_call:
+       .else
+.globl ftrace_call
+ftrace_call:
+       .endif
+       /* ftrace_call(r3, r4, r5, r6) */
        bl      ftrace_stub
+#endif
+.endm
+
+_GLOBAL(ftrace_regs_caller)
+       ftrace_regs_entry 1
+       ftrace_regs_func 1
        ftrace_regs_exit 1
 
 _GLOBAL(ftrace_caller)
        ftrace_regs_entry 0
-       /* ftrace_call(r3, r4, r5, r6) */
-.globl ftrace_call
-ftrace_call:
-       bl      ftrace_stub
+       ftrace_regs_func 0
        ftrace_regs_exit 0
 
 _GLOBAL(ftrace_stub)
@@ -377,7 +397,7 @@ _GLOBAL(return_to_handler)
 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
 SYM_DATA(ftrace_ool_stub_text_count, .long CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE)
 
-SYM_CODE_START(ftrace_ool_stub_text)
+SYM_START(ftrace_ool_stub_text, SYM_L_GLOBAL, .balign SZL)
        .space CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE * FTRACE_OOL_STUB_SIZE
 SYM_CODE_END(ftrace_ool_stub_text)
 #endif

diff --git a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh
index 6a201df83524e180362cbb1e42c067740c4c89b1..950a7778324b6842048c33b4b668f05e9de15d40 100755 (executable)
--- a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh
+++ b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh
@@ -28,12 +28,13 @@ fi
 
 cat > "$arch_vmlinux_S" <<EOF
 #include <asm/asm-offsets.h>
+#include <asm/ppc_asm.h>
 #include <linux/linkage.h>
 
 .pushsection .tramp.ftrace.text,"aw"
 SYM_DATA(ftrace_ool_stub_text_end_count, .long $num_ool_stubs_text_end)
 
-SYM_CODE_START(ftrace_ool_stub_text_end)
+SYM_START(ftrace_ool_stub_text_end, SYM_L_GLOBAL, .balign SZL)
 #if $num_ool_stubs_text_end
        .space $num_ool_stubs_text_end * FTRACE_OOL_STUB_SIZE
 #endif
@@ -43,7 +44,7 @@ SYM_CODE_END(ftrace_ool_stub_text_end)
 .pushsection .tramp.ftrace.init,"aw"
 SYM_DATA(ftrace_ool_stub_inittext_count, .long $num_ool_stubs_inittext)
 
-SYM_CODE_START(ftrace_ool_stub_inittext)
+SYM_START(ftrace_ool_stub_inittext, SYM_L_GLOBAL, .balign SZL)
        .space $num_ool_stubs_inittext * FTRACE_OOL_STUB_SIZE
 SYM_CODE_END(ftrace_ool_stub_inittext)
 .popsection