1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
4 #include <linux/ftrace.h>
5 #include <linux/uaccess.h>
6 #include <linux/stop_machine.h>
7 #include <asm/cacheflush.h>
9 #ifdef CONFIG_DYNAMIC_FTRACE
12 #define NOP32_HI 0xc400
13 #define NOP32_LO 0x4820
14 #define PUSH_LR 0x14d0
15 #define MOVIH_LINK 0xea3a
16 #define ORI_LINK 0xef5a
17 #define JSR_LINK 0xe8fa
18 #define BSR_LINK 0xe000
21 * Gcc-csky with -pg will insert stub in function prologue:
27 * If the (callee - current_pc) is less then 64MB, we'll use bsr:
32 * else we'll use (movih + ori + jsr):
38 * (r26 is our reserved link-reg)
41 static inline void make_jbsr(unsigned long callee
, unsigned long pc
,
42 uint16_t *call
, bool nolr
)
46 call
[0] = nolr
? NOP
: PUSH_LR
;
48 offset
= (long) callee
- (long) pc
;
50 if (unlikely(offset
< -67108864 || offset
> 67108864)) {
52 call
[2] = callee
>> 16;
54 call
[4] = callee
& 0xffff;
61 ((uint16_t)((unsigned long) offset
>> 16) & 0x3ff);
62 call
[2] = (uint16_t)((unsigned long) offset
& 0xffff);
63 call
[3] = call
[5] = NOP32_HI
;
64 call
[4] = call
[6] = NOP32_LO
;
68 static uint16_t nops
[7] = {NOP
, NOP32_HI
, NOP32_LO
, NOP32_HI
, NOP32_LO
,
70 static int ftrace_check_current_nop(unsigned long hook
)
73 unsigned long hook_pos
= hook
- 2;
75 if (probe_kernel_read((void *)olds
, (void *)hook_pos
, sizeof(nops
)))
78 if (memcmp((void *)nops
, (void *)olds
, sizeof(nops
))) {
79 pr_err("%p: nop but get (%04x %04x %04x %04x %04x %04x %04x)\n",
81 olds
[0], olds
[1], olds
[2], olds
[3], olds
[4], olds
[5],
90 static int ftrace_modify_code(unsigned long hook
, unsigned long target
,
91 bool enable
, bool nolr
)
95 unsigned long hook_pos
= hook
- 2;
98 make_jbsr(target
, hook
, call
, nolr
);
100 ret
= probe_kernel_write((void *)hook_pos
, enable
? call
: nops
,
105 flush_icache_range(hook_pos
, hook_pos
+ MCOUNT_INSN_SIZE
);
110 int ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
112 int ret
= ftrace_check_current_nop(rec
->ip
);
117 return ftrace_modify_code(rec
->ip
, addr
, true, false);
120 int ftrace_make_nop(struct module
*mod
, struct dyn_ftrace
*rec
,
123 return ftrace_modify_code(rec
->ip
, addr
, false, false);
126 int ftrace_update_ftrace_func(ftrace_func_t func
)
128 int ret
= ftrace_modify_code((unsigned long)&ftrace_call
,
129 (unsigned long)func
, true, true);
131 ret
= ftrace_modify_code((unsigned long)&ftrace_regs_call
,
132 (unsigned long)func
, true, true);
136 int __init
ftrace_dyn_arch_init(void)
140 #endif /* CONFIG_DYNAMIC_FTRACE */
142 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
143 int ftrace_modify_call(struct dyn_ftrace
*rec
, unsigned long old_addr
,
146 return ftrace_modify_code(rec
->ip
, addr
, true, true);
150 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
151 void prepare_ftrace_return(unsigned long *parent
, unsigned long self_addr
,
152 unsigned long frame_pointer
)
154 unsigned long return_hooker
= (unsigned long)&return_to_handler
;
157 if (unlikely(atomic_read(¤t
->tracing_graph_pause
)))
162 if (!function_graph_enter(old
, self_addr
,
163 *(unsigned long *)frame_pointer
, parent
)) {
165 * For csky-gcc function has sub-call:
169 * st.w r15, (sp, 0x4)
172 * We only need set *parent for resume
174 * For csky-gcc function has no sub-call:
180 * We need set *parent and *(frame_pointer + 4) for resume,
181 * because lr is resumed twice.
183 *parent
= return_hooker
;
185 if (*(unsigned long *)frame_pointer
== old
)
186 *(unsigned long *)frame_pointer
= return_hooker
;
190 #ifdef CONFIG_DYNAMIC_FTRACE
191 int ftrace_enable_ftrace_graph_caller(void)
193 return ftrace_modify_code((unsigned long)&ftrace_graph_call
,
194 (unsigned long)&ftrace_graph_caller
, true, true);
197 int ftrace_disable_ftrace_graph_caller(void)
199 return ftrace_modify_code((unsigned long)&ftrace_graph_call
,
200 (unsigned long)&ftrace_graph_caller
, false, true);
202 #endif /* CONFIG_DYNAMIC_FTRACE */
203 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
205 #ifdef CONFIG_DYNAMIC_FTRACE
206 #ifndef CONFIG_CPU_HAS_ICACHE_INS
207 struct ftrace_modify_param
{
212 static int __ftrace_modify_code(void *data
)
214 struct ftrace_modify_param
*param
= data
;
216 if (atomic_inc_return(¶m
->cpu_count
) == 1) {
217 ftrace_modify_all_code(param
->command
);
218 atomic_inc(¶m
->cpu_count
);
220 while (atomic_read(¶m
->cpu_count
) <= num_online_cpus())
222 local_icache_inv_all(NULL
);
228 void arch_ftrace_update_code(int command
)
230 struct ftrace_modify_param param
= { command
, ATOMIC_INIT(0) };
232 stop_machine(__ftrace_modify_code
, ¶m
, cpu_online_mask
);
235 #endif /* CONFIG_DYNAMIC_FTRACE */
237 /* _mcount is defined in abi's mcount.S */
238 EXPORT_SYMBOL(_mcount
);