1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
12 #include <objtool/builtin.h>
13 #include <objtool/cfi.h>
14 #include <objtool/arch.h>
15 #include <objtool/check.h>
16 #include <objtool/special.h>
17 #include <objtool/warn.h>
18 #include <objtool/endianness.h>
20 #include <linux/objtool_types.h>
21 #include <linux/hashtable.h>
22 #include <linux/kernel.h>
23 #include <linux/static_call_types.h>
26 struct alternative
*next
;
27 struct instruction
*insn
;
31 static unsigned long nr_cfi
, nr_cfi_reused
, nr_cfi_cache
;
33 static struct cfi_init_state initial_func_cfi
;
34 static struct cfi_state init_cfi
;
35 static struct cfi_state func_cfi
;
37 struct instruction
*find_insn(struct objtool_file
*file
,
38 struct section
*sec
, unsigned long offset
)
40 struct instruction
*insn
;
42 hash_for_each_possible(file
->insn_hash
, insn
, hash
, sec_offset_hash(sec
, offset
)) {
43 if (insn
->sec
== sec
&& insn
->offset
== offset
)
50 struct instruction
*next_insn_same_sec(struct objtool_file
*file
,
51 struct instruction
*insn
)
53 if (insn
->idx
== INSN_CHUNK_MAX
)
54 return find_insn(file
, insn
->sec
, insn
->offset
+ insn
->len
);
63 static struct instruction
*next_insn_same_func(struct objtool_file
*file
,
64 struct instruction
*insn
)
66 struct instruction
*next
= next_insn_same_sec(file
, insn
);
67 struct symbol
*func
= insn_func(insn
);
72 if (next
&& insn_func(next
) == func
)
75 /* Check if we're already in the subfunction: */
76 if (func
== func
->cfunc
)
79 /* Move to the subfunction: */
80 return find_insn(file
, func
->cfunc
->sec
, func
->cfunc
->offset
);
83 static struct instruction
*prev_insn_same_sec(struct objtool_file
*file
,
84 struct instruction
*insn
)
88 return find_insn(file
, insn
->sec
, insn
->offset
- insn
->prev_len
);
95 static struct instruction
*prev_insn_same_sym(struct objtool_file
*file
,
96 struct instruction
*insn
)
98 struct instruction
*prev
= prev_insn_same_sec(file
, insn
);
100 if (prev
&& insn_func(prev
) == insn_func(insn
))
106 #define for_each_insn(file, insn) \
107 for (struct section *__sec, *__fake = (struct section *)1; \
108 __fake; __fake = NULL) \
109 for_each_sec(file, __sec) \
110 sec_for_each_insn(file, __sec, insn)
112 #define func_for_each_insn(file, func, insn) \
113 for (insn = find_insn(file, func->sec, func->offset); \
115 insn = next_insn_same_func(file, insn))
117 #define sym_for_each_insn(file, sym, insn) \
118 for (insn = find_insn(file, sym->sec, sym->offset); \
119 insn && insn->offset < sym->offset + sym->len; \
120 insn = next_insn_same_sec(file, insn))
122 #define sym_for_each_insn_continue_reverse(file, sym, insn) \
123 for (insn = prev_insn_same_sec(file, insn); \
124 insn && insn->offset >= sym->offset; \
125 insn = prev_insn_same_sec(file, insn))
127 #define sec_for_each_insn_from(file, insn) \
128 for (; insn; insn = next_insn_same_sec(file, insn))
130 #define sec_for_each_insn_continue(file, insn) \
131 for (insn = next_insn_same_sec(file, insn); insn; \
132 insn = next_insn_same_sec(file, insn))
134 static inline struct symbol
*insn_call_dest(struct instruction
*insn
)
136 if (insn
->type
== INSN_JUMP_DYNAMIC
||
137 insn
->type
== INSN_CALL_DYNAMIC
)
140 return insn
->_call_dest
;
143 static inline struct reloc
*insn_jump_table(struct instruction
*insn
)
145 if (insn
->type
== INSN_JUMP_DYNAMIC
||
146 insn
->type
== INSN_CALL_DYNAMIC
)
147 return insn
->_jump_table
;
152 static bool is_jump_table_jump(struct instruction
*insn
)
154 struct alt_group
*alt_group
= insn
->alt_group
;
156 if (insn_jump_table(insn
))
159 /* Retpoline alternative for a jump table? */
160 return alt_group
&& alt_group
->orig_group
&&
161 insn_jump_table(alt_group
->orig_group
->first_insn
);
164 static bool is_sibling_call(struct instruction
*insn
)
167 * Assume only STT_FUNC calls have jump-tables.
169 if (insn_func(insn
)) {
170 /* An indirect jump is either a sibling call or a jump to a table. */
171 if (insn
->type
== INSN_JUMP_DYNAMIC
)
172 return !is_jump_table_jump(insn
);
175 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
176 return (is_static_jump(insn
) && insn_call_dest(insn
));
180 * This checks to see if the given function is a "noreturn" function.
182 * For global functions which are outside the scope of this object file, we
183 * have to keep a manual list of them.
185 * For local functions, we have to detect them manually by simply looking for
186 * the lack of a return instruction.
188 static bool __dead_end_function(struct objtool_file
*file
, struct symbol
*func
,
192 struct instruction
*insn
;
196 * Unfortunately these have to be hard coded because the noreturn
197 * attribute isn't provided in ELF data. Keep 'em sorted.
199 static const char * const global_noreturns
[] = {
201 "__module_put_and_kthread_exit",
204 "__ubsan_handle_builtin_unreachable",
205 "arch_call_rest_init",
206 "arch_cpu_idle_dead",
207 "cpu_bringup_and_idle",
212 "ex_handler_msr_mce",
214 "kthread_complete_and_exit",
216 "kunit_try_catch_throw",
218 "machine_real_restart",
220 "nmi_panic_self_stop",
222 "panic_smp_self_stop",
224 "rewind_stack_and_make_dead",
230 "x86_64_start_kernel",
231 "x86_64_start_reservations",
232 "xen_cpu_bringup_again",
239 if (func
->bind
== STB_WEAK
)
242 if (func
->bind
== STB_GLOBAL
)
243 for (i
= 0; i
< ARRAY_SIZE(global_noreturns
); i
++)
244 if (!strcmp(func
->name
, global_noreturns
[i
]))
250 insn
= find_insn(file
, func
->sec
, func
->offset
);
251 if (!insn
|| !insn_func(insn
))
254 func_for_each_insn(file
, func
, insn
) {
257 if (insn
->type
== INSN_RETURN
)
265 * A function can have a sibling call instead of a return. In that
266 * case, the function's dead-end status depends on whether the target
267 * of the sibling call returns.
269 func_for_each_insn(file
, func
, insn
) {
270 if (is_sibling_call(insn
)) {
271 struct instruction
*dest
= insn
->jump_dest
;
274 /* sibling call to another file */
277 /* local sibling call */
278 if (recursion
== 5) {
280 * Infinite recursion: two functions have
281 * sibling calls to each other. This is a very
282 * rare case. It means they aren't dead ends.
287 return __dead_end_function(file
, insn_func(dest
), recursion
+1);
294 static bool dead_end_function(struct objtool_file
*file
, struct symbol
*func
)
296 return __dead_end_function(file
, func
, 0);
299 static void init_cfi_state(struct cfi_state
*cfi
)
303 for (i
= 0; i
< CFI_NUM_REGS
; i
++) {
304 cfi
->regs
[i
].base
= CFI_UNDEFINED
;
305 cfi
->vals
[i
].base
= CFI_UNDEFINED
;
307 cfi
->cfa
.base
= CFI_UNDEFINED
;
308 cfi
->drap_reg
= CFI_UNDEFINED
;
309 cfi
->drap_offset
= -1;
312 static void init_insn_state(struct objtool_file
*file
, struct insn_state
*state
,
315 memset(state
, 0, sizeof(*state
));
316 init_cfi_state(&state
->cfi
);
319 * We need the full vmlinux for noinstr validation, otherwise we can
320 * not correctly determine insn_call_dest(insn)->sec (external symbols
321 * do not have a section).
323 if (opts
.link
&& opts
.noinstr
&& sec
)
324 state
->noinstr
= sec
->noinstr
;
327 static struct cfi_state
*cfi_alloc(void)
329 struct cfi_state
*cfi
= calloc(sizeof(struct cfi_state
), 1);
331 WARN("calloc failed");
339 static struct hlist_head
*cfi_hash
;
341 static inline bool cficmp(struct cfi_state
*cfi1
, struct cfi_state
*cfi2
)
343 return memcmp((void *)cfi1
+ sizeof(cfi1
->hash
),
344 (void *)cfi2
+ sizeof(cfi2
->hash
),
345 sizeof(struct cfi_state
) - sizeof(struct hlist_node
));
348 static inline u32
cfi_key(struct cfi_state
*cfi
)
350 return jhash((void *)cfi
+ sizeof(cfi
->hash
),
351 sizeof(*cfi
) - sizeof(cfi
->hash
), 0);
354 static struct cfi_state
*cfi_hash_find_or_add(struct cfi_state
*cfi
)
356 struct hlist_head
*head
= &cfi_hash
[hash_min(cfi_key(cfi
), cfi_bits
)];
357 struct cfi_state
*obj
;
359 hlist_for_each_entry(obj
, head
, hash
) {
360 if (!cficmp(cfi
, obj
)) {
368 hlist_add_head(&obj
->hash
, head
);
373 static void cfi_hash_add(struct cfi_state
*cfi
)
375 struct hlist_head
*head
= &cfi_hash
[hash_min(cfi_key(cfi
), cfi_bits
)];
377 hlist_add_head(&cfi
->hash
, head
);
380 static void *cfi_hash_alloc(unsigned long size
)
382 cfi_bits
= max(10, ilog2(size
));
383 cfi_hash
= mmap(NULL
, sizeof(struct hlist_head
) << cfi_bits
,
384 PROT_READ
|PROT_WRITE
,
385 MAP_PRIVATE
|MAP_ANON
, -1, 0);
386 if (cfi_hash
== (void *)-1L) {
387 WARN("mmap fail cfi_hash");
389 } else if (opts
.stats
) {
390 printf("cfi_bits: %d\n", cfi_bits
);
396 static unsigned long nr_insns
;
397 static unsigned long nr_insns_visited
;
400 * Call the arch-specific instruction decoder for all the instructions and add
401 * them to the global instruction list.
403 static int decode_instructions(struct objtool_file
*file
)
407 unsigned long offset
;
408 struct instruction
*insn
;
411 for_each_sec(file
, sec
) {
412 struct instruction
*insns
= NULL
;
416 if (!(sec
->sh
.sh_flags
& SHF_EXECINSTR
))
419 if (strcmp(sec
->name
, ".altinstr_replacement") &&
420 strcmp(sec
->name
, ".altinstr_aux") &&
421 strncmp(sec
->name
, ".discard.", 9))
424 if (!strcmp(sec
->name
, ".noinstr.text") ||
425 !strcmp(sec
->name
, ".entry.text") ||
426 !strcmp(sec
->name
, ".cpuidle.text") ||
427 !strncmp(sec
->name
, ".text.__x86.", 12))
431 * .init.text code is ran before userspace and thus doesn't
432 * strictly need retpolines, except for modules which are
433 * loaded late, they very much do need retpoline in their
436 if (!strcmp(sec
->name
, ".init.text") && !opts
.module
)
439 for (offset
= 0; offset
< sec
->sh
.sh_size
; offset
+= insn
->len
) {
440 if (!insns
|| idx
== INSN_CHUNK_MAX
) {
441 insns
= calloc(sizeof(*insn
), INSN_CHUNK_SIZE
);
443 WARN("malloc failed");
453 INIT_LIST_HEAD(&insn
->call_node
);
455 insn
->offset
= offset
;
456 insn
->prev_len
= prev_len
;
458 ret
= arch_decode_instruction(file
, sec
, offset
,
459 sec
->sh
.sh_size
- offset
,
464 prev_len
= insn
->len
;
467 * By default, "ud2" is a dead end unless otherwise
468 * annotated, because GCC 7 inserts it for certain
469 * divide-by-zero cases.
471 if (insn
->type
== INSN_BUG
)
472 insn
->dead_end
= true;
474 hash_add(file
->insn_hash
, &insn
->hash
, sec_offset_hash(sec
, insn
->offset
));
478 // printf("%s: last chunk used: %d\n", sec->name, (int)idx);
480 sec_for_each_sym(sec
, func
) {
481 if (func
->type
!= STT_NOTYPE
&& func
->type
!= STT_FUNC
)
484 if (func
->offset
== sec
->sh
.sh_size
) {
485 /* Heuristic: likely an "end" symbol */
486 if (func
->type
== STT_NOTYPE
)
488 WARN("%s(): STT_FUNC at end of section",
493 if (func
->return_thunk
|| func
->alias
!= func
)
496 if (!find_insn(file
, sec
, func
->offset
)) {
497 WARN("%s(): can't find starting instruction",
502 sym_for_each_insn(file
, func
, insn
) {
504 if (func
->type
== STT_FUNC
&&
505 insn
->type
== INSN_ENDBR
&&
506 list_empty(&insn
->call_node
)) {
507 if (insn
->offset
== func
->offset
) {
508 list_add_tail(&insn
->call_node
, &file
->endbr_list
);
511 file
->nr_endbr_int
++;
519 printf("nr_insns: %lu\n", nr_insns
);
525 * Read the pv_ops[] .data table to find the static initialized values.
527 static int add_pv_ops(struct objtool_file
*file
, const char *symname
)
529 struct symbol
*sym
, *func
;
530 unsigned long off
, end
;
534 sym
= find_symbol_by_name(file
->elf
, symname
);
539 end
= off
+ sym
->len
;
541 rel
= find_reloc_by_dest_range(file
->elf
, sym
->sec
, off
, end
- off
);
546 if (func
->type
== STT_SECTION
)
547 func
= find_symbol_by_offset(rel
->sym
->sec
, rel
->addend
);
549 idx
= (rel
->offset
- sym
->offset
) / sizeof(unsigned long);
551 objtool_pv_add(file
, idx
, func
);
553 off
= rel
->offset
+ 1;
562 * Allocate and initialize file->pv_ops[].
564 static int init_pv_ops(struct objtool_file
*file
)
566 static const char *pv_ops_tables
[] = {
582 sym
= find_symbol_by_name(file
->elf
, "pv_ops");
586 nr
= sym
->len
/ sizeof(unsigned long);
587 file
->pv_ops
= calloc(sizeof(struct pv_state
), nr
);
591 for (idx
= 0; idx
< nr
; idx
++)
592 INIT_LIST_HEAD(&file
->pv_ops
[idx
].targets
);
594 for (idx
= 0; (pv_ops
= pv_ops_tables
[idx
]); idx
++)
595 add_pv_ops(file
, pv_ops
);
600 static struct instruction
*find_last_insn(struct objtool_file
*file
,
603 struct instruction
*insn
= NULL
;
605 unsigned int end
= (sec
->sh
.sh_size
> 10) ? sec
->sh
.sh_size
- 10 : 0;
607 for (offset
= sec
->sh
.sh_size
- 1; offset
>= end
&& !insn
; offset
--)
608 insn
= find_insn(file
, sec
, offset
);
614 * Mark "ud2" instructions and manually annotated dead ends.
616 static int add_dead_ends(struct objtool_file
*file
)
620 struct instruction
*insn
;
623 * Check for manually annotated dead ends.
625 sec
= find_section_by_name(file
->elf
, ".rela.discard.unreachable");
629 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
630 if (reloc
->sym
->type
!= STT_SECTION
) {
631 WARN("unexpected relocation symbol type in %s", sec
->name
);
634 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
636 insn
= prev_insn_same_sec(file
, insn
);
637 else if (reloc
->addend
== reloc
->sym
->sec
->sh
.sh_size
) {
638 insn
= find_last_insn(file
, reloc
->sym
->sec
);
640 WARN("can't find unreachable insn at %s+0x%" PRIx64
,
641 reloc
->sym
->sec
->name
, reloc
->addend
);
645 WARN("can't find unreachable insn at %s+0x%" PRIx64
,
646 reloc
->sym
->sec
->name
, reloc
->addend
);
650 insn
->dead_end
= true;
655 * These manually annotated reachable checks are needed for GCC 4.4,
656 * where the Linux unreachable() macro isn't supported. In that case
657 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
660 sec
= find_section_by_name(file
->elf
, ".rela.discard.reachable");
664 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
665 if (reloc
->sym
->type
!= STT_SECTION
) {
666 WARN("unexpected relocation symbol type in %s", sec
->name
);
669 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
671 insn
= prev_insn_same_sec(file
, insn
);
672 else if (reloc
->addend
== reloc
->sym
->sec
->sh
.sh_size
) {
673 insn
= find_last_insn(file
, reloc
->sym
->sec
);
675 WARN("can't find reachable insn at %s+0x%" PRIx64
,
676 reloc
->sym
->sec
->name
, reloc
->addend
);
680 WARN("can't find reachable insn at %s+0x%" PRIx64
,
681 reloc
->sym
->sec
->name
, reloc
->addend
);
685 insn
->dead_end
= false;
691 static int create_static_call_sections(struct objtool_file
*file
)
694 struct static_call_site
*site
;
695 struct instruction
*insn
;
696 struct symbol
*key_sym
;
697 char *key_name
, *tmp
;
700 sec
= find_section_by_name(file
->elf
, ".static_call_sites");
702 INIT_LIST_HEAD(&file
->static_call_list
);
703 WARN("file already has .static_call_sites section, skipping");
707 if (list_empty(&file
->static_call_list
))
711 list_for_each_entry(insn
, &file
->static_call_list
, call_node
)
714 sec
= elf_create_section(file
->elf
, ".static_call_sites", SHF_WRITE
,
715 sizeof(struct static_call_site
), idx
);
720 list_for_each_entry(insn
, &file
->static_call_list
, call_node
) {
722 site
= (struct static_call_site
*)sec
->data
->d_buf
+ idx
;
723 memset(site
, 0, sizeof(struct static_call_site
));
725 /* populate reloc for 'addr' */
726 if (elf_add_reloc_to_insn(file
->elf
, sec
,
727 idx
* sizeof(struct static_call_site
),
729 insn
->sec
, insn
->offset
))
732 /* find key symbol */
733 key_name
= strdup(insn_call_dest(insn
)->name
);
738 if (strncmp(key_name
, STATIC_CALL_TRAMP_PREFIX_STR
,
739 STATIC_CALL_TRAMP_PREFIX_LEN
)) {
740 WARN("static_call: trampoline name malformed: %s", key_name
);
744 tmp
= key_name
+ STATIC_CALL_TRAMP_PREFIX_LEN
- STATIC_CALL_KEY_PREFIX_LEN
;
745 memcpy(tmp
, STATIC_CALL_KEY_PREFIX_STR
, STATIC_CALL_KEY_PREFIX_LEN
);
747 key_sym
= find_symbol_by_name(file
->elf
, tmp
);
750 WARN("static_call: can't find static_call_key symbol: %s", tmp
);
756 * For modules(), the key might not be exported, which
757 * means the module can make static calls but isn't
758 * allowed to change them.
760 * In that case we temporarily set the key to be the
761 * trampoline address. This is fixed up in
762 * static_call_add_module().
764 key_sym
= insn_call_dest(insn
);
768 /* populate reloc for 'key' */
769 if (elf_add_reloc(file
->elf
, sec
,
770 idx
* sizeof(struct static_call_site
) + 4,
771 R_X86_64_PC32
, key_sym
,
772 is_sibling_call(insn
) * STATIC_CALL_SITE_TAIL
))
781 static int create_retpoline_sites_sections(struct objtool_file
*file
)
783 struct instruction
*insn
;
787 sec
= find_section_by_name(file
->elf
, ".retpoline_sites");
789 WARN("file already has .retpoline_sites, skipping");
794 list_for_each_entry(insn
, &file
->retpoline_call_list
, call_node
)
800 sec
= elf_create_section(file
->elf
, ".retpoline_sites", 0,
803 WARN("elf_create_section: .retpoline_sites");
808 list_for_each_entry(insn
, &file
->retpoline_call_list
, call_node
) {
810 int *site
= (int *)sec
->data
->d_buf
+ idx
;
813 if (elf_add_reloc_to_insn(file
->elf
, sec
,
816 insn
->sec
, insn
->offset
)) {
817 WARN("elf_add_reloc_to_insn: .retpoline_sites");
827 static int create_return_sites_sections(struct objtool_file
*file
)
829 struct instruction
*insn
;
833 sec
= find_section_by_name(file
->elf
, ".return_sites");
835 WARN("file already has .return_sites, skipping");
840 list_for_each_entry(insn
, &file
->return_thunk_list
, call_node
)
846 sec
= elf_create_section(file
->elf
, ".return_sites", 0,
849 WARN("elf_create_section: .return_sites");
854 list_for_each_entry(insn
, &file
->return_thunk_list
, call_node
) {
856 int *site
= (int *)sec
->data
->d_buf
+ idx
;
859 if (elf_add_reloc_to_insn(file
->elf
, sec
,
862 insn
->sec
, insn
->offset
)) {
863 WARN("elf_add_reloc_to_insn: .return_sites");
873 static int create_ibt_endbr_seal_sections(struct objtool_file
*file
)
875 struct instruction
*insn
;
879 sec
= find_section_by_name(file
->elf
, ".ibt_endbr_seal");
881 WARN("file already has .ibt_endbr_seal, skipping");
886 list_for_each_entry(insn
, &file
->endbr_list
, call_node
)
890 printf("ibt: ENDBR at function start: %d\n", file
->nr_endbr
);
891 printf("ibt: ENDBR inside functions: %d\n", file
->nr_endbr_int
);
892 printf("ibt: superfluous ENDBR: %d\n", idx
);
898 sec
= elf_create_section(file
->elf
, ".ibt_endbr_seal", 0,
901 WARN("elf_create_section: .ibt_endbr_seal");
906 list_for_each_entry(insn
, &file
->endbr_list
, call_node
) {
908 int *site
= (int *)sec
->data
->d_buf
+ idx
;
909 struct symbol
*sym
= insn
->sym
;
912 if (opts
.module
&& sym
&& sym
->type
== STT_FUNC
&&
913 insn
->offset
== sym
->offset
&&
914 (!strcmp(sym
->name
, "init_module") ||
915 !strcmp(sym
->name
, "cleanup_module")))
916 WARN("%s(): not an indirect call target", sym
->name
);
918 if (elf_add_reloc_to_insn(file
->elf
, sec
,
921 insn
->sec
, insn
->offset
)) {
922 WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
932 static int create_cfi_sections(struct objtool_file
*file
)
939 sec
= find_section_by_name(file
->elf
, ".cfi_sites");
941 INIT_LIST_HEAD(&file
->call_list
);
942 WARN("file already has .cfi_sites section, skipping");
947 for_each_sym(file
, sym
) {
948 if (sym
->type
!= STT_FUNC
)
951 if (strncmp(sym
->name
, "__cfi_", 6))
957 sec
= elf_create_section(file
->elf
, ".cfi_sites", 0, sizeof(unsigned int), idx
);
962 for_each_sym(file
, sym
) {
963 if (sym
->type
!= STT_FUNC
)
966 if (strncmp(sym
->name
, "__cfi_", 6))
969 loc
= (unsigned int *)sec
->data
->d_buf
+ idx
;
970 memset(loc
, 0, sizeof(unsigned int));
972 if (elf_add_reloc_to_insn(file
->elf
, sec
,
973 idx
* sizeof(unsigned int),
975 sym
->sec
, sym
->offset
))
984 static int create_mcount_loc_sections(struct objtool_file
*file
)
986 int addrsize
= elf_class_addrsize(file
->elf
);
987 struct instruction
*insn
;
991 sec
= find_section_by_name(file
->elf
, "__mcount_loc");
993 INIT_LIST_HEAD(&file
->mcount_loc_list
);
994 WARN("file already has __mcount_loc section, skipping");
998 if (list_empty(&file
->mcount_loc_list
))
1002 list_for_each_entry(insn
, &file
->mcount_loc_list
, call_node
)
1005 sec
= elf_create_section(file
->elf
, "__mcount_loc", 0, addrsize
, idx
);
1009 sec
->sh
.sh_addralign
= addrsize
;
1012 list_for_each_entry(insn
, &file
->mcount_loc_list
, call_node
) {
1015 loc
= sec
->data
->d_buf
+ idx
;
1016 memset(loc
, 0, addrsize
);
1018 if (elf_add_reloc_to_insn(file
->elf
, sec
, idx
,
1019 addrsize
== sizeof(u64
) ? R_ABS64
: R_ABS32
,
1020 insn
->sec
, insn
->offset
))
1029 static int create_direct_call_sections(struct objtool_file
*file
)
1031 struct instruction
*insn
;
1032 struct section
*sec
;
1036 sec
= find_section_by_name(file
->elf
, ".call_sites");
1038 INIT_LIST_HEAD(&file
->call_list
);
1039 WARN("file already has .call_sites section, skipping");
1043 if (list_empty(&file
->call_list
))
1047 list_for_each_entry(insn
, &file
->call_list
, call_node
)
1050 sec
= elf_create_section(file
->elf
, ".call_sites", 0, sizeof(unsigned int), idx
);
1055 list_for_each_entry(insn
, &file
->call_list
, call_node
) {
1057 loc
= (unsigned int *)sec
->data
->d_buf
+ idx
;
1058 memset(loc
, 0, sizeof(unsigned int));
1060 if (elf_add_reloc_to_insn(file
->elf
, sec
,
1061 idx
* sizeof(unsigned int),
1063 insn
->sec
, insn
->offset
))
1073 * Warnings shouldn't be reported for ignored functions.
1075 static void add_ignores(struct objtool_file
*file
)
1077 struct instruction
*insn
;
1078 struct section
*sec
;
1079 struct symbol
*func
;
1080 struct reloc
*reloc
;
1082 sec
= find_section_by_name(file
->elf
, ".rela.discard.func_stack_frame_non_standard");
1086 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
1087 switch (reloc
->sym
->type
) {
1093 func
= find_func_by_offset(reloc
->sym
->sec
, reloc
->addend
);
1099 WARN("unexpected relocation symbol type in %s: %d", sec
->name
, reloc
->sym
->type
);
1103 func_for_each_insn(file
, func
, insn
)
1104 insn
->ignore
= true;
1109 * This is a whitelist of functions that is allowed to be called with AC set.
1110 * The list is meant to be minimal and only contains compiler instrumentation
1111 * ABI and a few functions used to implement *_{to,from}_user() functions.
1113 * These functions must not directly change AC, but may PUSHF/POPF.
1115 static const char *uaccess_safe_builtin
[] = {
1118 "kasan_check_range",
1119 /* KASAN out-of-line */
1120 "__asan_loadN_noabort",
1121 "__asan_load1_noabort",
1122 "__asan_load2_noabort",
1123 "__asan_load4_noabort",
1124 "__asan_load8_noabort",
1125 "__asan_load16_noabort",
1126 "__asan_storeN_noabort",
1127 "__asan_store1_noabort",
1128 "__asan_store2_noabort",
1129 "__asan_store4_noabort",
1130 "__asan_store8_noabort",
1131 "__asan_store16_noabort",
1132 "__kasan_check_read",
1133 "__kasan_check_write",
1135 "__asan_report_load_n_noabort",
1136 "__asan_report_load1_noabort",
1137 "__asan_report_load2_noabort",
1138 "__asan_report_load4_noabort",
1139 "__asan_report_load8_noabort",
1140 "__asan_report_load16_noabort",
1141 "__asan_report_store_n_noabort",
1142 "__asan_report_store1_noabort",
1143 "__asan_report_store2_noabort",
1144 "__asan_report_store4_noabort",
1145 "__asan_report_store8_noabort",
1146 "__asan_report_store16_noabort",
1148 "__kcsan_check_access",
1153 "kcsan_found_watchpoint",
1154 "kcsan_setup_watchpoint",
1155 "kcsan_check_scoped_accesses",
1156 "kcsan_disable_current",
1157 "kcsan_enable_current_nowarn",
1159 "__tsan_func_entry",
1161 "__tsan_read_range",
1162 "__tsan_write_range",
1173 "__tsan_read_write1",
1174 "__tsan_read_write2",
1175 "__tsan_read_write4",
1176 "__tsan_read_write8",
1177 "__tsan_read_write16",
1178 "__tsan_volatile_read1",
1179 "__tsan_volatile_read2",
1180 "__tsan_volatile_read4",
1181 "__tsan_volatile_read8",
1182 "__tsan_volatile_read16",
1183 "__tsan_volatile_write1",
1184 "__tsan_volatile_write2",
1185 "__tsan_volatile_write4",
1186 "__tsan_volatile_write8",
1187 "__tsan_volatile_write16",
1188 "__tsan_atomic8_load",
1189 "__tsan_atomic16_load",
1190 "__tsan_atomic32_load",
1191 "__tsan_atomic64_load",
1192 "__tsan_atomic8_store",
1193 "__tsan_atomic16_store",
1194 "__tsan_atomic32_store",
1195 "__tsan_atomic64_store",
1196 "__tsan_atomic8_exchange",
1197 "__tsan_atomic16_exchange",
1198 "__tsan_atomic32_exchange",
1199 "__tsan_atomic64_exchange",
1200 "__tsan_atomic8_fetch_add",
1201 "__tsan_atomic16_fetch_add",
1202 "__tsan_atomic32_fetch_add",
1203 "__tsan_atomic64_fetch_add",
1204 "__tsan_atomic8_fetch_sub",
1205 "__tsan_atomic16_fetch_sub",
1206 "__tsan_atomic32_fetch_sub",
1207 "__tsan_atomic64_fetch_sub",
1208 "__tsan_atomic8_fetch_and",
1209 "__tsan_atomic16_fetch_and",
1210 "__tsan_atomic32_fetch_and",
1211 "__tsan_atomic64_fetch_and",
1212 "__tsan_atomic8_fetch_or",
1213 "__tsan_atomic16_fetch_or",
1214 "__tsan_atomic32_fetch_or",
1215 "__tsan_atomic64_fetch_or",
1216 "__tsan_atomic8_fetch_xor",
1217 "__tsan_atomic16_fetch_xor",
1218 "__tsan_atomic32_fetch_xor",
1219 "__tsan_atomic64_fetch_xor",
1220 "__tsan_atomic8_fetch_nand",
1221 "__tsan_atomic16_fetch_nand",
1222 "__tsan_atomic32_fetch_nand",
1223 "__tsan_atomic64_fetch_nand",
1224 "__tsan_atomic8_compare_exchange_strong",
1225 "__tsan_atomic16_compare_exchange_strong",
1226 "__tsan_atomic32_compare_exchange_strong",
1227 "__tsan_atomic64_compare_exchange_strong",
1228 "__tsan_atomic8_compare_exchange_weak",
1229 "__tsan_atomic16_compare_exchange_weak",
1230 "__tsan_atomic32_compare_exchange_weak",
1231 "__tsan_atomic64_compare_exchange_weak",
1232 "__tsan_atomic8_compare_exchange_val",
1233 "__tsan_atomic16_compare_exchange_val",
1234 "__tsan_atomic32_compare_exchange_val",
1235 "__tsan_atomic64_compare_exchange_val",
1236 "__tsan_atomic_thread_fence",
1237 "__tsan_atomic_signal_fence",
1238 "__tsan_unaligned_read16",
1239 "__tsan_unaligned_write16",
1243 "__sanitizer_cov_trace_pc",
1244 "__sanitizer_cov_trace_const_cmp1",
1245 "__sanitizer_cov_trace_const_cmp2",
1246 "__sanitizer_cov_trace_const_cmp4",
1247 "__sanitizer_cov_trace_const_cmp8",
1248 "__sanitizer_cov_trace_cmp1",
1249 "__sanitizer_cov_trace_cmp2",
1250 "__sanitizer_cov_trace_cmp4",
1251 "__sanitizer_cov_trace_cmp8",
1252 "__sanitizer_cov_trace_switch",
1254 "kmsan_copy_to_user",
1256 "kmsan_unpoison_entry_regs",
1257 "kmsan_unpoison_memory",
1258 "__msan_chain_origin",
1259 "__msan_get_context_state",
1260 "__msan_instrument_asm_store",
1261 "__msan_metadata_ptr_for_load_1",
1262 "__msan_metadata_ptr_for_load_2",
1263 "__msan_metadata_ptr_for_load_4",
1264 "__msan_metadata_ptr_for_load_8",
1265 "__msan_metadata_ptr_for_load_n",
1266 "__msan_metadata_ptr_for_store_1",
1267 "__msan_metadata_ptr_for_store_2",
1268 "__msan_metadata_ptr_for_store_4",
1269 "__msan_metadata_ptr_for_store_8",
1270 "__msan_metadata_ptr_for_store_n",
1271 "__msan_poison_alloca",
1274 "ubsan_type_mismatch_common",
1275 "__ubsan_handle_type_mismatch",
1276 "__ubsan_handle_type_mismatch_v1",
1277 "__ubsan_handle_shift_out_of_bounds",
1278 "__ubsan_handle_load_invalid_value",
1280 "stackleak_track_stack",
1282 "csum_partial_copy_generic",
1284 "copy_mc_fragile_handle_tail",
1285 "copy_mc_enhanced_fast_string",
1286 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1288 "clear_user_rep_good",
1289 "clear_user_original",
1293 static void add_uaccess_safe(struct objtool_file
*file
)
1295 struct symbol
*func
;
1301 for (name
= uaccess_safe_builtin
; *name
; name
++) {
1302 func
= find_symbol_by_name(file
->elf
, *name
);
1306 func
->uaccess_safe
= true;
1311 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1312 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1313 * But it at least allows objtool to understand the control flow *around* the
1316 static int add_ignore_alternatives(struct objtool_file
*file
)
1318 struct section
*sec
;
1319 struct reloc
*reloc
;
1320 struct instruction
*insn
;
1322 sec
= find_section_by_name(file
->elf
, ".rela.discard.ignore_alts");
1326 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
1327 if (reloc
->sym
->type
!= STT_SECTION
) {
1328 WARN("unexpected relocation symbol type in %s", sec
->name
);
1332 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
1334 WARN("bad .discard.ignore_alts entry");
1338 insn
->ignore_alts
= true;
1344 __weak
bool arch_is_retpoline(struct symbol
*sym
)
1349 __weak
bool arch_is_rethunk(struct symbol
*sym
)
1354 static struct reloc
*insn_reloc(struct objtool_file
*file
, struct instruction
*insn
)
1356 struct reloc
*reloc
;
1364 reloc
= find_reloc_by_dest_range(file
->elf
, insn
->sec
,
1365 insn
->offset
, insn
->len
);
1374 static void remove_insn_ops(struct instruction
*insn
)
1376 struct stack_op
*op
, *next
;
1378 for (op
= insn
->stack_ops
; op
; op
= next
) {
1382 insn
->stack_ops
= NULL
;
1385 static void annotate_call_site(struct objtool_file
*file
,
1386 struct instruction
*insn
, bool sibling
)
1388 struct reloc
*reloc
= insn_reloc(file
, insn
);
1389 struct symbol
*sym
= insn_call_dest(insn
);
1395 * Alternative replacement code is just template code which is
1396 * sometimes copied to the original instruction. For now, don't
1397 * annotate it. (In the future we might consider annotating the
1398 * original instruction if/when it ever makes sense to do so.)
1400 if (!strcmp(insn
->sec
->name
, ".altinstr_replacement"))
1403 if (sym
->static_call_tramp
) {
1404 list_add_tail(&insn
->call_node
, &file
->static_call_list
);
1408 if (sym
->retpoline_thunk
) {
1409 list_add_tail(&insn
->call_node
, &file
->retpoline_call_list
);
1414 * Many compilers cannot disable KCOV or sanitizer calls with a function
1415 * attribute so they need a little help, NOP out any such calls from
1418 if (opts
.hack_noinstr
&& insn
->sec
->noinstr
&& sym
->profiling_func
) {
1420 reloc
->type
= R_NONE
;
1421 elf_write_reloc(file
->elf
, reloc
);
1424 elf_write_insn(file
->elf
, insn
->sec
,
1425 insn
->offset
, insn
->len
,
1426 sibling
? arch_ret_insn(insn
->len
)
1427 : arch_nop_insn(insn
->len
));
1429 insn
->type
= sibling
? INSN_RETURN
: INSN_NOP
;
1433 * We've replaced the tail-call JMP insn by two new
1434 * insn: RET; INT3, except we only have a single struct
1435 * insn here. Mark it retpoline_safe to avoid the SLS
1436 * warning, instead of adding another insn.
1438 insn
->retpoline_safe
= true;
1444 if (opts
.mcount
&& sym
->fentry
) {
1446 WARN_INSN(insn
, "tail call to __fentry__ !?!?");
1449 reloc
->type
= R_NONE
;
1450 elf_write_reloc(file
->elf
, reloc
);
1453 elf_write_insn(file
->elf
, insn
->sec
,
1454 insn
->offset
, insn
->len
,
1455 arch_nop_insn(insn
->len
));
1457 insn
->type
= INSN_NOP
;
1460 list_add_tail(&insn
->call_node
, &file
->mcount_loc_list
);
1464 if (insn
->type
== INSN_CALL
&& !insn
->sec
->init
)
1465 list_add_tail(&insn
->call_node
, &file
->call_list
);
1467 if (!sibling
&& dead_end_function(file
, sym
))
1468 insn
->dead_end
= true;
1471 static void add_call_dest(struct objtool_file
*file
, struct instruction
*insn
,
1472 struct symbol
*dest
, bool sibling
)
1474 insn
->_call_dest
= dest
;
1479 * Whatever stack impact regular CALLs have, should be undone
1480 * by the RETURN of the called function.
1482 * Annotated intra-function calls retain the stack_ops but
1483 * are converted to JUMP, see read_intra_function_calls().
1485 remove_insn_ops(insn
);
1487 annotate_call_site(file
, insn
, sibling
);
1490 static void add_retpoline_call(struct objtool_file
*file
, struct instruction
*insn
)
1493 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1494 * so convert them accordingly.
1496 switch (insn
->type
) {
1498 insn
->type
= INSN_CALL_DYNAMIC
;
1500 case INSN_JUMP_UNCONDITIONAL
:
1501 insn
->type
= INSN_JUMP_DYNAMIC
;
1503 case INSN_JUMP_CONDITIONAL
:
1504 insn
->type
= INSN_JUMP_DYNAMIC_CONDITIONAL
;
1510 insn
->retpoline_safe
= true;
1513 * Whatever stack impact regular CALLs have, should be undone
1514 * by the RETURN of the called function.
1516 * Annotated intra-function calls retain the stack_ops but
1517 * are converted to JUMP, see read_intra_function_calls().
1519 remove_insn_ops(insn
);
1521 annotate_call_site(file
, insn
, false);
1524 static void add_return_call(struct objtool_file
*file
, struct instruction
*insn
, bool add
)
1527 * Return thunk tail calls are really just returns in disguise,
1528 * so convert them accordingly.
1530 insn
->type
= INSN_RETURN
;
1531 insn
->retpoline_safe
= true;
1534 list_add_tail(&insn
->call_node
, &file
->return_thunk_list
);
1537 static bool is_first_func_insn(struct objtool_file
*file
,
1538 struct instruction
*insn
, struct symbol
*sym
)
1540 if (insn
->offset
== sym
->offset
)
1543 /* Allow direct CALL/JMP past ENDBR */
1545 struct instruction
*prev
= prev_insn_same_sym(file
, insn
);
1547 if (prev
&& prev
->type
== INSN_ENDBR
&&
1548 insn
->offset
== sym
->offset
+ prev
->len
)
1556 * A sibling call is a tail-call to another symbol -- to differentiate from a
1557 * recursive tail-call which is to the same symbol.
1559 static bool jump_is_sibling_call(struct objtool_file
*file
,
1560 struct instruction
*from
, struct instruction
*to
)
1562 struct symbol
*fs
= from
->sym
;
1563 struct symbol
*ts
= to
->sym
;
1565 /* Not a sibling call if from/to a symbol hole */
1569 /* Not a sibling call if not targeting the start of a symbol. */
1570 if (!is_first_func_insn(file
, to
, ts
))
1573 /* Disallow sibling calls into STT_NOTYPE */
1574 if (ts
->type
== STT_NOTYPE
)
1577 /* Must not be self to be a sibling */
1578 return fs
->pfunc
!= ts
->pfunc
;
1582 * Find the destination instructions for all jumps.
1584 static int add_jump_destinations(struct objtool_file
*file
)
1586 struct instruction
*insn
, *jump_dest
;
1587 struct reloc
*reloc
;
1588 struct section
*dest_sec
;
1589 unsigned long dest_off
;
1591 for_each_insn(file
, insn
) {
1592 if (insn
->jump_dest
) {
1594 * handle_group_alt() may have previously set
1595 * 'jump_dest' for some alternatives.
1599 if (!is_static_jump(insn
))
1602 reloc
= insn_reloc(file
, insn
);
1604 dest_sec
= insn
->sec
;
1605 dest_off
= arch_jump_destination(insn
);
1606 } else if (reloc
->sym
->type
== STT_SECTION
) {
1607 dest_sec
= reloc
->sym
->sec
;
1608 dest_off
= arch_dest_reloc_offset(reloc
->addend
);
1609 } else if (reloc
->sym
->retpoline_thunk
) {
1610 add_retpoline_call(file
, insn
);
1612 } else if (reloc
->sym
->return_thunk
) {
1613 add_return_call(file
, insn
, true);
1615 } else if (insn_func(insn
)) {
1617 * External sibling call or internal sibling call with
1620 add_call_dest(file
, insn
, reloc
->sym
, true);
1622 } else if (reloc
->sym
->sec
->idx
) {
1623 dest_sec
= reloc
->sym
->sec
;
1624 dest_off
= reloc
->sym
->sym
.st_value
+
1625 arch_dest_reloc_offset(reloc
->addend
);
1627 /* non-func asm code jumping to another file */
1631 jump_dest
= find_insn(file
, dest_sec
, dest_off
);
1633 struct symbol
*sym
= find_symbol_by_offset(dest_sec
, dest_off
);
1636 * This is a special case for zen_untrain_ret().
1637 * It jumps to __x86_return_thunk(), but objtool
1638 * can't find the thunk's starting RET
1639 * instruction, because the RET is also in the
1640 * middle of another instruction. Objtool only
1641 * knows about the outer instruction.
1643 if (sym
&& sym
->return_thunk
) {
1644 add_return_call(file
, insn
, false);
1648 WARN_INSN(insn
, "can't find jump dest instruction at %s+0x%lx",
1649 dest_sec
->name
, dest_off
);
1654 * Cross-function jump.
1656 if (insn_func(insn
) && insn_func(jump_dest
) &&
1657 insn_func(insn
) != insn_func(jump_dest
)) {
1660 * For GCC 8+, create parent/child links for any cold
1661 * subfunctions. This is _mostly_ redundant with a
1662 * similar initialization in read_symbols().
1664 * If a function has aliases, we want the *first* such
1665 * function in the symbol table to be the subfunction's
1666 * parent. In that case we overwrite the
1667 * initialization done in read_symbols().
1669 * However this code can't completely replace the
1670 * read_symbols() code because this doesn't detect the
1671 * case where the parent function's only reference to a
1672 * subfunction is through a jump table.
1674 if (!strstr(insn_func(insn
)->name
, ".cold") &&
1675 strstr(insn_func(jump_dest
)->name
, ".cold")) {
1676 insn_func(insn
)->cfunc
= insn_func(jump_dest
);
1677 insn_func(jump_dest
)->pfunc
= insn_func(insn
);
1681 if (jump_is_sibling_call(file
, insn
, jump_dest
)) {
1683 * Internal sibling call without reloc or with
1684 * STT_SECTION reloc.
1686 add_call_dest(file
, insn
, insn_func(jump_dest
), true);
1690 insn
->jump_dest
= jump_dest
;
1696 static struct symbol
*find_call_destination(struct section
*sec
, unsigned long offset
)
1698 struct symbol
*call_dest
;
1700 call_dest
= find_func_by_offset(sec
, offset
);
1702 call_dest
= find_symbol_by_offset(sec
, offset
);
1708 * Find the destination instructions for all calls.
1710 static int add_call_destinations(struct objtool_file
*file
)
1712 struct instruction
*insn
;
1713 unsigned long dest_off
;
1714 struct symbol
*dest
;
1715 struct reloc
*reloc
;
1717 for_each_insn(file
, insn
) {
1718 if (insn
->type
!= INSN_CALL
)
1721 reloc
= insn_reloc(file
, insn
);
1723 dest_off
= arch_jump_destination(insn
);
1724 dest
= find_call_destination(insn
->sec
, dest_off
);
1726 add_call_dest(file
, insn
, dest
, false);
1731 if (!insn_call_dest(insn
)) {
1732 WARN_INSN(insn
, "unannotated intra-function call");
1736 if (insn_func(insn
) && insn_call_dest(insn
)->type
!= STT_FUNC
) {
1737 WARN_INSN(insn
, "unsupported call to non-function");
1741 } else if (reloc
->sym
->type
== STT_SECTION
) {
1742 dest_off
= arch_dest_reloc_offset(reloc
->addend
);
1743 dest
= find_call_destination(reloc
->sym
->sec
, dest_off
);
1745 WARN_INSN(insn
, "can't find call dest symbol at %s+0x%lx",
1746 reloc
->sym
->sec
->name
, dest_off
);
1750 add_call_dest(file
, insn
, dest
, false);
1752 } else if (reloc
->sym
->retpoline_thunk
) {
1753 add_retpoline_call(file
, insn
);
1756 add_call_dest(file
, insn
, reloc
->sym
, false);
1763 * The .alternatives section requires some extra special care over and above
1764 * other special sections because alternatives are patched in place.
1766 static int handle_group_alt(struct objtool_file
*file
,
1767 struct special_alt
*special_alt
,
1768 struct instruction
*orig_insn
,
1769 struct instruction
**new_insn
)
1771 struct instruction
*last_new_insn
= NULL
, *insn
, *nop
= NULL
;
1772 struct alt_group
*orig_alt_group
, *new_alt_group
;
1773 unsigned long dest_off
;
1775 orig_alt_group
= orig_insn
->alt_group
;
1776 if (!orig_alt_group
) {
1777 struct instruction
*last_orig_insn
= NULL
;
1779 orig_alt_group
= malloc(sizeof(*orig_alt_group
));
1780 if (!orig_alt_group
) {
1781 WARN("malloc failed");
1784 orig_alt_group
->cfi
= calloc(special_alt
->orig_len
,
1785 sizeof(struct cfi_state
*));
1786 if (!orig_alt_group
->cfi
) {
1787 WARN("calloc failed");
1792 sec_for_each_insn_from(file
, insn
) {
1793 if (insn
->offset
>= special_alt
->orig_off
+ special_alt
->orig_len
)
1796 insn
->alt_group
= orig_alt_group
;
1797 last_orig_insn
= insn
;
1799 orig_alt_group
->orig_group
= NULL
;
1800 orig_alt_group
->first_insn
= orig_insn
;
1801 orig_alt_group
->last_insn
= last_orig_insn
;
1802 orig_alt_group
->nop
= NULL
;
1804 if (orig_alt_group
->last_insn
->offset
+ orig_alt_group
->last_insn
->len
-
1805 orig_alt_group
->first_insn
->offset
!= special_alt
->orig_len
) {
1806 WARN_INSN(orig_insn
, "weirdly overlapping alternative! %ld != %d",
1807 orig_alt_group
->last_insn
->offset
+
1808 orig_alt_group
->last_insn
->len
-
1809 orig_alt_group
->first_insn
->offset
,
1810 special_alt
->orig_len
);
1815 new_alt_group
= malloc(sizeof(*new_alt_group
));
1816 if (!new_alt_group
) {
1817 WARN("malloc failed");
1821 if (special_alt
->new_len
< special_alt
->orig_len
) {
1823 * Insert a fake nop at the end to make the replacement
1824 * alt_group the same size as the original. This is needed to
1825 * allow propagate_alt_cfi() to do its magic. When the last
1826 * instruction affects the stack, the instruction after it (the
1827 * nop) will propagate the new state to the shared CFI array.
1829 nop
= malloc(sizeof(*nop
));
1831 WARN("malloc failed");
1834 memset(nop
, 0, sizeof(*nop
));
1836 nop
->sec
= special_alt
->new_sec
;
1837 nop
->offset
= special_alt
->new_off
+ special_alt
->new_len
;
1838 nop
->len
= special_alt
->orig_len
- special_alt
->new_len
;
1839 nop
->type
= INSN_NOP
;
1840 nop
->sym
= orig_insn
->sym
;
1841 nop
->alt_group
= new_alt_group
;
1842 nop
->ignore
= orig_insn
->ignore_alts
;
1845 if (!special_alt
->new_len
) {
1851 sec_for_each_insn_from(file
, insn
) {
1852 struct reloc
*alt_reloc
;
1854 if (insn
->offset
>= special_alt
->new_off
+ special_alt
->new_len
)
1857 last_new_insn
= insn
;
1859 insn
->ignore
= orig_insn
->ignore_alts
;
1860 insn
->sym
= orig_insn
->sym
;
1861 insn
->alt_group
= new_alt_group
;
1864 * Since alternative replacement code is copy/pasted by the
1865 * kernel after applying relocations, generally such code can't
1866 * have relative-address relocation references to outside the
1867 * .altinstr_replacement section, unless the arch's
1868 * alternatives code can adjust the relative offsets
1871 alt_reloc
= insn_reloc(file
, insn
);
1872 if (alt_reloc
&& arch_pc_relative_reloc(alt_reloc
) &&
1873 !arch_support_alt_relocation(special_alt
, insn
, alt_reloc
)) {
1875 WARN_INSN(insn
, "unsupported relocation in alternatives section");
1879 if (!is_static_jump(insn
))
1882 if (!insn
->immediate
)
1885 dest_off
= arch_jump_destination(insn
);
1886 if (dest_off
== special_alt
->new_off
+ special_alt
->new_len
) {
1887 insn
->jump_dest
= next_insn_same_sec(file
, orig_alt_group
->last_insn
);
1888 if (!insn
->jump_dest
) {
1889 WARN_INSN(insn
, "can't find alternative jump destination");
1895 if (!last_new_insn
) {
1896 WARN_FUNC("can't find last new alternative instruction",
1897 special_alt
->new_sec
, special_alt
->new_off
);
1902 new_alt_group
->orig_group
= orig_alt_group
;
1903 new_alt_group
->first_insn
= *new_insn
;
1904 new_alt_group
->last_insn
= last_new_insn
;
1905 new_alt_group
->nop
= nop
;
1906 new_alt_group
->cfi
= orig_alt_group
->cfi
;
1911 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1912 * If the original instruction is a jump, make the alt entry an effective nop
1913 * by just skipping the original instruction.
1915 static int handle_jump_alt(struct objtool_file
*file
,
1916 struct special_alt
*special_alt
,
1917 struct instruction
*orig_insn
,
1918 struct instruction
**new_insn
)
1920 if (orig_insn
->type
!= INSN_JUMP_UNCONDITIONAL
&&
1921 orig_insn
->type
!= INSN_NOP
) {
1923 WARN_INSN(orig_insn
, "unsupported instruction at jump label");
1927 if (opts
.hack_jump_label
&& special_alt
->key_addend
& 2) {
1928 struct reloc
*reloc
= insn_reloc(file
, orig_insn
);
1931 reloc
->type
= R_NONE
;
1932 elf_write_reloc(file
->elf
, reloc
);
1934 elf_write_insn(file
->elf
, orig_insn
->sec
,
1935 orig_insn
->offset
, orig_insn
->len
,
1936 arch_nop_insn(orig_insn
->len
));
1937 orig_insn
->type
= INSN_NOP
;
1940 if (orig_insn
->type
== INSN_NOP
) {
1941 if (orig_insn
->len
== 2)
1942 file
->jl_nop_short
++;
1944 file
->jl_nop_long
++;
1949 if (orig_insn
->len
== 2)
1954 *new_insn
= next_insn_same_sec(file
, orig_insn
);
1959 * Read all the special sections which have alternate instructions which can be
1960 * patched in or redirected to at runtime. Each instruction having alternate
1961 * instruction(s) has them added to its insn->alts list, which will be
1962 * traversed in validate_branch().
1964 static int add_special_section_alts(struct objtool_file
*file
)
1966 struct list_head special_alts
;
1967 struct instruction
*orig_insn
, *new_insn
;
1968 struct special_alt
*special_alt
, *tmp
;
1969 struct alternative
*alt
;
1972 ret
= special_get_alts(file
->elf
, &special_alts
);
1976 list_for_each_entry_safe(special_alt
, tmp
, &special_alts
, list
) {
1978 orig_insn
= find_insn(file
, special_alt
->orig_sec
,
1979 special_alt
->orig_off
);
1981 WARN_FUNC("special: can't find orig instruction",
1982 special_alt
->orig_sec
, special_alt
->orig_off
);
1988 if (!special_alt
->group
|| special_alt
->new_len
) {
1989 new_insn
= find_insn(file
, special_alt
->new_sec
,
1990 special_alt
->new_off
);
1992 WARN_FUNC("special: can't find new instruction",
1993 special_alt
->new_sec
,
1994 special_alt
->new_off
);
2000 if (special_alt
->group
) {
2001 if (!special_alt
->orig_len
) {
2002 WARN_INSN(orig_insn
, "empty alternative entry");
2006 ret
= handle_group_alt(file
, special_alt
, orig_insn
,
2010 } else if (special_alt
->jump_or_nop
) {
2011 ret
= handle_jump_alt(file
, special_alt
, orig_insn
,
2017 alt
= malloc(sizeof(*alt
));
2019 WARN("malloc failed");
2024 alt
->insn
= new_insn
;
2025 alt
->skip_orig
= special_alt
->skip_orig
;
2026 orig_insn
->ignore_alts
|= special_alt
->skip_alt
;
2027 alt
->next
= orig_insn
->alts
;
2028 orig_insn
->alts
= alt
;
2030 list_del(&special_alt
->list
);
2035 printf("jl\\\tNOP\tJMP\n");
2036 printf("short:\t%ld\t%ld\n", file
->jl_nop_short
, file
->jl_short
);
2037 printf("long:\t%ld\t%ld\n", file
->jl_nop_long
, file
->jl_long
);
2044 static int add_jump_table(struct objtool_file
*file
, struct instruction
*insn
,
2045 struct reloc
*table
)
2047 struct reloc
*reloc
= table
;
2048 struct instruction
*dest_insn
;
2049 struct alternative
*alt
;
2050 struct symbol
*pfunc
= insn_func(insn
)->pfunc
;
2051 unsigned int prev_offset
= 0;
2054 * Each @reloc is a switch table relocation which points to the target
2057 list_for_each_entry_from(reloc
, &table
->sec
->reloc_list
, list
) {
2059 /* Check for the end of the table: */
2060 if (reloc
!= table
&& reloc
->jump_table_start
)
2063 /* Make sure the table entries are consecutive: */
2064 if (prev_offset
&& reloc
->offset
!= prev_offset
+ 8)
2067 /* Detect function pointers from contiguous objects: */
2068 if (reloc
->sym
->sec
== pfunc
->sec
&&
2069 reloc
->addend
== pfunc
->offset
)
2072 dest_insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
2076 /* Make sure the destination is in the same function: */
2077 if (!insn_func(dest_insn
) || insn_func(dest_insn
)->pfunc
!= pfunc
)
2080 alt
= malloc(sizeof(*alt
));
2082 WARN("malloc failed");
2086 alt
->insn
= dest_insn
;
2087 alt
->next
= insn
->alts
;
2089 prev_offset
= reloc
->offset
;
2093 WARN_INSN(insn
, "can't find switch jump table");
2101 * find_jump_table() - Given a dynamic jump, find the switch jump table
2102 * associated with it.
2104 static struct reloc
*find_jump_table(struct objtool_file
*file
,
2105 struct symbol
*func
,
2106 struct instruction
*insn
)
2108 struct reloc
*table_reloc
;
2109 struct instruction
*dest_insn
, *orig_insn
= insn
;
2112 * Backward search using the @first_jump_src links, these help avoid
2113 * much of the 'in between' code. Which avoids us getting confused by
2117 insn
&& insn_func(insn
) && insn_func(insn
)->pfunc
== func
;
2118 insn
= insn
->first_jump_src
?: prev_insn_same_sym(file
, insn
)) {
2120 if (insn
!= orig_insn
&& insn
->type
== INSN_JUMP_DYNAMIC
)
2123 /* allow small jumps within the range */
2124 if (insn
->type
== INSN_JUMP_UNCONDITIONAL
&&
2126 (insn
->jump_dest
->offset
<= insn
->offset
||
2127 insn
->jump_dest
->offset
> orig_insn
->offset
))
2130 table_reloc
= arch_find_switch_table(file
, insn
);
2133 dest_insn
= find_insn(file
, table_reloc
->sym
->sec
, table_reloc
->addend
);
2134 if (!dest_insn
|| !insn_func(dest_insn
) || insn_func(dest_insn
)->pfunc
!= func
)
2144 * First pass: Mark the head of each jump table so that in the next pass,
2145 * we know when a given jump table ends and the next one starts.
2147 static void mark_func_jump_tables(struct objtool_file
*file
,
2148 struct symbol
*func
)
2150 struct instruction
*insn
, *last
= NULL
;
2151 struct reloc
*reloc
;
2153 func_for_each_insn(file
, func
, insn
) {
2158 * Store back-pointers for unconditional forward jumps such
2159 * that find_jump_table() can back-track using those and
2160 * avoid some potentially confusing code.
2162 if (insn
->type
== INSN_JUMP_UNCONDITIONAL
&& insn
->jump_dest
&&
2163 insn
->offset
> last
->offset
&&
2164 insn
->jump_dest
->offset
> insn
->offset
&&
2165 !insn
->jump_dest
->first_jump_src
) {
2167 insn
->jump_dest
->first_jump_src
= insn
;
2168 last
= insn
->jump_dest
;
2171 if (insn
->type
!= INSN_JUMP_DYNAMIC
)
2174 reloc
= find_jump_table(file
, func
, insn
);
2176 reloc
->jump_table_start
= true;
2177 insn
->_jump_table
= reloc
;
2182 static int add_func_jump_tables(struct objtool_file
*file
,
2183 struct symbol
*func
)
2185 struct instruction
*insn
;
2188 func_for_each_insn(file
, func
, insn
) {
2189 if (!insn_jump_table(insn
))
2192 ret
= add_jump_table(file
, insn
, insn_jump_table(insn
));
2201 * For some switch statements, gcc generates a jump table in the .rodata
2202 * section which contains a list of addresses within the function to jump to.
2203 * This finds these jump tables and adds them to the insn->alts lists.
2205 static int add_jump_table_alts(struct objtool_file
*file
)
2207 struct symbol
*func
;
2213 for_each_sym(file
, func
) {
2214 if (func
->type
!= STT_FUNC
)
2217 mark_func_jump_tables(file
, func
);
2218 ret
= add_func_jump_tables(file
, func
);
2226 static void set_func_state(struct cfi_state
*state
)
2228 state
->cfa
= initial_func_cfi
.cfa
;
2229 memcpy(&state
->regs
, &initial_func_cfi
.regs
,
2230 CFI_NUM_REGS
* sizeof(struct cfi_reg
));
2231 state
->stack_size
= initial_func_cfi
.cfa
.offset
;
2232 state
->type
= UNWIND_HINT_TYPE_CALL
;
2235 static int read_unwind_hints(struct objtool_file
*file
)
2237 struct cfi_state cfi
= init_cfi
;
2238 struct section
*sec
, *relocsec
;
2239 struct unwind_hint
*hint
;
2240 struct instruction
*insn
;
2241 struct reloc
*reloc
;
2244 sec
= find_section_by_name(file
->elf
, ".discard.unwind_hints");
2248 relocsec
= sec
->reloc
;
2250 WARN("missing .rela.discard.unwind_hints section");
2254 if (sec
->sh
.sh_size
% sizeof(struct unwind_hint
)) {
2255 WARN("struct unwind_hint size mismatch");
2261 for (i
= 0; i
< sec
->sh
.sh_size
/ sizeof(struct unwind_hint
); i
++) {
2262 hint
= (struct unwind_hint
*)sec
->data
->d_buf
+ i
;
2264 reloc
= find_reloc_by_dest(file
->elf
, sec
, i
* sizeof(*hint
));
2266 WARN("can't find reloc for unwind_hints[%d]", i
);
2270 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
2272 WARN("can't find insn for unwind_hints[%d]", i
);
2278 if (hint
->type
== UNWIND_HINT_TYPE_SAVE
) {
2284 if (hint
->type
== UNWIND_HINT_TYPE_RESTORE
) {
2285 insn
->restore
= true;
2289 if (hint
->type
== UNWIND_HINT_TYPE_REGS_PARTIAL
) {
2290 struct symbol
*sym
= find_symbol_by_offset(insn
->sec
, insn
->offset
);
2292 if (sym
&& sym
->bind
== STB_GLOBAL
) {
2293 if (opts
.ibt
&& insn
->type
!= INSN_ENDBR
&& !insn
->noendbr
) {
2294 WARN_INSN(insn
, "UNWIND_HINT_IRET_REGS without ENDBR");
2299 if (hint
->type
== UNWIND_HINT_TYPE_FUNC
) {
2300 insn
->cfi
= &func_cfi
;
2307 if (arch_decode_hint_reg(hint
->sp_reg
, &cfi
.cfa
.base
)) {
2308 WARN_INSN(insn
, "unsupported unwind_hint sp base reg %d", hint
->sp_reg
);
2312 cfi
.cfa
.offset
= bswap_if_needed(file
->elf
, hint
->sp_offset
);
2313 cfi
.type
= hint
->type
;
2314 cfi
.signal
= hint
->signal
;
2316 insn
->cfi
= cfi_hash_find_or_add(&cfi
);
2322 static int read_noendbr_hints(struct objtool_file
*file
)
2324 struct section
*sec
;
2325 struct instruction
*insn
;
2326 struct reloc
*reloc
;
2328 sec
= find_section_by_name(file
->elf
, ".rela.discard.noendbr");
2332 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
2333 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->sym
->offset
+ reloc
->addend
);
2335 WARN("bad .discard.noendbr entry");
2345 static int read_retpoline_hints(struct objtool_file
*file
)
2347 struct section
*sec
;
2348 struct instruction
*insn
;
2349 struct reloc
*reloc
;
2351 sec
= find_section_by_name(file
->elf
, ".rela.discard.retpoline_safe");
2355 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
2356 if (reloc
->sym
->type
!= STT_SECTION
) {
2357 WARN("unexpected relocation symbol type in %s", sec
->name
);
2361 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
2363 WARN("bad .discard.retpoline_safe entry");
2367 if (insn
->type
!= INSN_JUMP_DYNAMIC
&&
2368 insn
->type
!= INSN_CALL_DYNAMIC
&&
2369 insn
->type
!= INSN_RETURN
&&
2370 insn
->type
!= INSN_NOP
) {
2371 WARN_INSN(insn
, "retpoline_safe hint not an indirect jump/call/ret/nop");
2375 insn
->retpoline_safe
= true;
2381 static int read_instr_hints(struct objtool_file
*file
)
2383 struct section
*sec
;
2384 struct instruction
*insn
;
2385 struct reloc
*reloc
;
2387 sec
= find_section_by_name(file
->elf
, ".rela.discard.instr_end");
2391 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
2392 if (reloc
->sym
->type
!= STT_SECTION
) {
2393 WARN("unexpected relocation symbol type in %s", sec
->name
);
2397 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
2399 WARN("bad .discard.instr_end entry");
2406 sec
= find_section_by_name(file
->elf
, ".rela.discard.instr_begin");
2410 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
2411 if (reloc
->sym
->type
!= STT_SECTION
) {
2412 WARN("unexpected relocation symbol type in %s", sec
->name
);
2416 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
2418 WARN("bad .discard.instr_begin entry");
2428 static int read_validate_unret_hints(struct objtool_file
*file
)
2430 struct section
*sec
;
2431 struct instruction
*insn
;
2432 struct reloc
*reloc
;
2434 sec
= find_section_by_name(file
->elf
, ".rela.discard.validate_unret");
2438 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
2439 if (reloc
->sym
->type
!= STT_SECTION
) {
2440 WARN("unexpected relocation symbol type in %s", sec
->name
);
2444 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
2446 WARN("bad .discard.instr_end entry");
2456 static int read_intra_function_calls(struct objtool_file
*file
)
2458 struct instruction
*insn
;
2459 struct section
*sec
;
2460 struct reloc
*reloc
;
2462 sec
= find_section_by_name(file
->elf
, ".rela.discard.intra_function_calls");
2466 list_for_each_entry(reloc
, &sec
->reloc_list
, list
) {
2467 unsigned long dest_off
;
2469 if (reloc
->sym
->type
!= STT_SECTION
) {
2470 WARN("unexpected relocation symbol type in %s",
2475 insn
= find_insn(file
, reloc
->sym
->sec
, reloc
->addend
);
2477 WARN("bad .discard.intra_function_call entry");
2481 if (insn
->type
!= INSN_CALL
) {
2482 WARN_INSN(insn
, "intra_function_call not a direct call");
2487 * Treat intra-function CALLs as JMPs, but with a stack_op.
2488 * See add_call_destinations(), which strips stack_ops from
2491 insn
->type
= INSN_JUMP_UNCONDITIONAL
;
2493 dest_off
= arch_jump_destination(insn
);
2494 insn
->jump_dest
= find_insn(file
, insn
->sec
, dest_off
);
2495 if (!insn
->jump_dest
) {
2496 WARN_INSN(insn
, "can't find call dest at %s+0x%lx",
2497 insn
->sec
->name
, dest_off
);
2506 * Return true if name matches an instrumentation function, where calls to that
2507 * function from noinstr code can safely be removed, but compilers won't do so.
2509 static bool is_profiling_func(const char *name
)
2512 * Many compilers cannot disable KCOV with a function attribute.
2514 if (!strncmp(name
, "__sanitizer_cov_", 16))
2518 * Some compilers currently do not remove __tsan_func_entry/exit nor
2519 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2520 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2521 * minimum Clang version is 14.0, this can be removed.
2523 if (!strncmp(name
, "__tsan_func_", 12) ||
2524 !strcmp(name
, "__tsan_atomic_signal_fence"))
2530 static int classify_symbols(struct objtool_file
*file
)
2532 struct symbol
*func
;
2534 for_each_sym(file
, func
) {
2535 if (func
->bind
!= STB_GLOBAL
)
2538 if (!strncmp(func
->name
, STATIC_CALL_TRAMP_PREFIX_STR
,
2539 strlen(STATIC_CALL_TRAMP_PREFIX_STR
)))
2540 func
->static_call_tramp
= true;
2542 if (arch_is_retpoline(func
))
2543 func
->retpoline_thunk
= true;
2545 if (arch_is_rethunk(func
))
2546 func
->return_thunk
= true;
2548 if (arch_ftrace_match(func
->name
))
2549 func
->fentry
= true;
2551 if (is_profiling_func(func
->name
))
2552 func
->profiling_func
= true;
2558 static void mark_rodata(struct objtool_file
*file
)
2560 struct section
*sec
;
2564 * Search for the following rodata sections, each of which can
2565 * potentially contain jump tables:
2567 * - .rodata: can contain GCC switch tables
2568 * - .rodata.<func>: same, if -fdata-sections is being used
2569 * - .rodata..c_jump_table: contains C annotated jump tables
2571 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2573 for_each_sec(file
, sec
) {
2574 if (!strncmp(sec
->name
, ".rodata", 7) &&
2575 !strstr(sec
->name
, ".str1.")) {
2581 file
->rodata
= found
;
2584 static int decode_sections(struct objtool_file
*file
)
2590 ret
= init_pv_ops(file
);
2595 * Must be before add_{jump_call}_destination.
2597 ret
= classify_symbols(file
);
2601 ret
= decode_instructions(file
);
2606 add_uaccess_safe(file
);
2608 ret
= add_ignore_alternatives(file
);
2613 * Must be before read_unwind_hints() since that needs insn->noendbr.
2615 ret
= read_noendbr_hints(file
);
2620 * Must be before add_jump_destinations(), which depends on 'func'
2621 * being set for alternatives, to enable proper sibling call detection.
2623 if (opts
.stackval
|| opts
.orc
|| opts
.uaccess
|| opts
.noinstr
) {
2624 ret
= add_special_section_alts(file
);
2629 ret
= add_jump_destinations(file
);
2634 * Must be before add_call_destination(); it changes INSN_CALL to
2637 ret
= read_intra_function_calls(file
);
2641 ret
= add_call_destinations(file
);
2646 * Must be after add_call_destinations() such that it can override
2647 * dead_end_function() marks.
2649 ret
= add_dead_ends(file
);
2653 ret
= add_jump_table_alts(file
);
2657 ret
= read_unwind_hints(file
);
2661 ret
= read_retpoline_hints(file
);
2665 ret
= read_instr_hints(file
);
2669 ret
= read_validate_unret_hints(file
);
2676 static bool is_fentry_call(struct instruction
*insn
)
2678 if (insn
->type
== INSN_CALL
&&
2679 insn_call_dest(insn
) &&
2680 insn_call_dest(insn
)->fentry
)
2686 static bool has_modified_stack_frame(struct instruction
*insn
, struct insn_state
*state
)
2688 struct cfi_state
*cfi
= &state
->cfi
;
2691 if (cfi
->cfa
.base
!= initial_func_cfi
.cfa
.base
|| cfi
->drap
)
2694 if (cfi
->cfa
.offset
!= initial_func_cfi
.cfa
.offset
)
2697 if (cfi
->stack_size
!= initial_func_cfi
.cfa
.offset
)
2700 for (i
= 0; i
< CFI_NUM_REGS
; i
++) {
2701 if (cfi
->regs
[i
].base
!= initial_func_cfi
.regs
[i
].base
||
2702 cfi
->regs
[i
].offset
!= initial_func_cfi
.regs
[i
].offset
)
2709 static bool check_reg_frame_pos(const struct cfi_reg
*reg
,
2710 int expected_offset
)
2712 return reg
->base
== CFI_CFA
&&
2713 reg
->offset
== expected_offset
;
2716 static bool has_valid_stack_frame(struct insn_state
*state
)
2718 struct cfi_state
*cfi
= &state
->cfi
;
2720 if (cfi
->cfa
.base
== CFI_BP
&&
2721 check_reg_frame_pos(&cfi
->regs
[CFI_BP
], -cfi
->cfa
.offset
) &&
2722 check_reg_frame_pos(&cfi
->regs
[CFI_RA
], -cfi
->cfa
.offset
+ 8))
2725 if (cfi
->drap
&& cfi
->regs
[CFI_BP
].base
== CFI_BP
)
2731 static int update_cfi_state_regs(struct instruction
*insn
,
2732 struct cfi_state
*cfi
,
2733 struct stack_op
*op
)
2735 struct cfi_reg
*cfa
= &cfi
->cfa
;
2737 if (cfa
->base
!= CFI_SP
&& cfa
->base
!= CFI_SP_INDIRECT
)
2741 if (op
->dest
.type
== OP_DEST_PUSH
|| op
->dest
.type
== OP_DEST_PUSHF
)
2745 if (op
->src
.type
== OP_SRC_POP
|| op
->src
.type
== OP_SRC_POPF
)
2748 /* add immediate to sp */
2749 if (op
->dest
.type
== OP_DEST_REG
&& op
->src
.type
== OP_SRC_ADD
&&
2750 op
->dest
.reg
== CFI_SP
&& op
->src
.reg
== CFI_SP
)
2751 cfa
->offset
-= op
->src
.offset
;
2756 static void save_reg(struct cfi_state
*cfi
, unsigned char reg
, int base
, int offset
)
2758 if (arch_callee_saved_reg(reg
) &&
2759 cfi
->regs
[reg
].base
== CFI_UNDEFINED
) {
2760 cfi
->regs
[reg
].base
= base
;
2761 cfi
->regs
[reg
].offset
= offset
;
2765 static void restore_reg(struct cfi_state
*cfi
, unsigned char reg
)
2767 cfi
->regs
[reg
].base
= initial_func_cfi
.regs
[reg
].base
;
2768 cfi
->regs
[reg
].offset
= initial_func_cfi
.regs
[reg
].offset
;
2772 * A note about DRAP stack alignment:
2774 * GCC has the concept of a DRAP register, which is used to help keep track of
2775 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2776 * register. The typical DRAP pattern is:
2778 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2779 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2780 * 41 ff 72 f8 pushq -0x8(%r10)
2782 * 48 89 e5 mov %rsp,%rbp
2789 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2792 * There are some variations in the epilogues, like:
2800 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2805 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2806 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2807 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2808 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2810 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2813 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2814 * restored beforehand:
2817 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2818 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2820 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2824 static int update_cfi_state(struct instruction
*insn
,
2825 struct instruction
*next_insn
,
2826 struct cfi_state
*cfi
, struct stack_op
*op
)
2828 struct cfi_reg
*cfa
= &cfi
->cfa
;
2829 struct cfi_reg
*regs
= cfi
->regs
;
2831 /* stack operations don't make sense with an undefined CFA */
2832 if (cfa
->base
== CFI_UNDEFINED
) {
2833 if (insn_func(insn
)) {
2834 WARN_INSN(insn
, "undefined stack state");
2840 if (cfi
->type
== UNWIND_HINT_TYPE_REGS
||
2841 cfi
->type
== UNWIND_HINT_TYPE_REGS_PARTIAL
)
2842 return update_cfi_state_regs(insn
, cfi
, op
);
2844 switch (op
->dest
.type
) {
2847 switch (op
->src
.type
) {
2850 if (op
->src
.reg
== CFI_SP
&& op
->dest
.reg
== CFI_BP
&&
2851 cfa
->base
== CFI_SP
&&
2852 check_reg_frame_pos(®s
[CFI_BP
], -cfa
->offset
)) {
2854 /* mov %rsp, %rbp */
2855 cfa
->base
= op
->dest
.reg
;
2856 cfi
->bp_scratch
= false;
2859 else if (op
->src
.reg
== CFI_SP
&&
2860 op
->dest
.reg
== CFI_BP
&& cfi
->drap
) {
2862 /* drap: mov %rsp, %rbp */
2863 regs
[CFI_BP
].base
= CFI_BP
;
2864 regs
[CFI_BP
].offset
= -cfi
->stack_size
;
2865 cfi
->bp_scratch
= false;
2868 else if (op
->src
.reg
== CFI_SP
&& cfa
->base
== CFI_SP
) {
2873 * This is needed for the rare case where GCC
2880 cfi
->vals
[op
->dest
.reg
].base
= CFI_CFA
;
2881 cfi
->vals
[op
->dest
.reg
].offset
= -cfi
->stack_size
;
2884 else if (op
->src
.reg
== CFI_BP
&& op
->dest
.reg
== CFI_SP
&&
2885 (cfa
->base
== CFI_BP
|| cfa
->base
== cfi
->drap_reg
)) {
2890 * Restore the original stack pointer (Clang).
2892 cfi
->stack_size
= -cfi
->regs
[CFI_BP
].offset
;
2895 else if (op
->dest
.reg
== cfa
->base
) {
2897 /* mov %reg, %rsp */
2898 if (cfa
->base
== CFI_SP
&&
2899 cfi
->vals
[op
->src
.reg
].base
== CFI_CFA
) {
2902 * This is needed for the rare case
2903 * where GCC does something dumb like:
2905 * lea 0x8(%rsp), %rcx
2909 cfa
->offset
= -cfi
->vals
[op
->src
.reg
].offset
;
2910 cfi
->stack_size
= cfa
->offset
;
2912 } else if (cfa
->base
== CFI_SP
&&
2913 cfi
->vals
[op
->src
.reg
].base
== CFI_SP_INDIRECT
&&
2914 cfi
->vals
[op
->src
.reg
].offset
== cfa
->offset
) {
2919 * 1: mov %rsp, (%[tos])
2920 * 2: mov %[tos], %rsp
2926 * 1 - places a pointer to the previous
2927 * stack at the Top-of-Stack of the
2930 * 2 - switches to the new stack.
2932 * 3 - pops the Top-of-Stack to restore
2933 * the original stack.
2935 * Note: we set base to SP_INDIRECT
2936 * here and preserve offset. Therefore
2937 * when the unwinder reaches ToS it
2938 * will dereference SP and then add the
2939 * offset to find the next frame, IOW:
2942 cfa
->base
= CFI_SP_INDIRECT
;
2945 cfa
->base
= CFI_UNDEFINED
;
2950 else if (op
->dest
.reg
== CFI_SP
&&
2951 cfi
->vals
[op
->src
.reg
].base
== CFI_SP_INDIRECT
&&
2952 cfi
->vals
[op
->src
.reg
].offset
== cfa
->offset
) {
2955 * The same stack swizzle case 2) as above. But
2956 * because we can't change cfa->base, case 3)
2957 * will become a regular POP. Pretend we're a
2958 * PUSH so things don't go unbalanced.
2960 cfi
->stack_size
+= 8;
2967 if (op
->dest
.reg
== CFI_SP
&& op
->src
.reg
== CFI_SP
) {
2970 cfi
->stack_size
-= op
->src
.offset
;
2971 if (cfa
->base
== CFI_SP
)
2972 cfa
->offset
-= op
->src
.offset
;
2976 if (op
->dest
.reg
== CFI_SP
&& op
->src
.reg
== CFI_BP
) {
2978 /* lea disp(%rbp), %rsp */
2979 cfi
->stack_size
= -(op
->src
.offset
+ regs
[CFI_BP
].offset
);
2983 if (op
->src
.reg
== CFI_SP
&& cfa
->base
== CFI_SP
) {
2985 /* drap: lea disp(%rsp), %drap */
2986 cfi
->drap_reg
= op
->dest
.reg
;
2989 * lea disp(%rsp), %reg
2991 * This is needed for the rare case where GCC
2992 * does something dumb like:
2994 * lea 0x8(%rsp), %rcx
2998 cfi
->vals
[op
->dest
.reg
].base
= CFI_CFA
;
2999 cfi
->vals
[op
->dest
.reg
].offset
= \
3000 -cfi
->stack_size
+ op
->src
.offset
;
3005 if (cfi
->drap
&& op
->dest
.reg
== CFI_SP
&&
3006 op
->src
.reg
== cfi
->drap_reg
) {
3008 /* drap: lea disp(%drap), %rsp */
3010 cfa
->offset
= cfi
->stack_size
= -op
->src
.offset
;
3011 cfi
->drap_reg
= CFI_UNDEFINED
;
3016 if (op
->dest
.reg
== cfi
->cfa
.base
&& !(next_insn
&& next_insn
->hint
)) {
3017 WARN_INSN(insn
, "unsupported stack register modification");
3024 if (op
->dest
.reg
!= CFI_SP
||
3025 (cfi
->drap_reg
!= CFI_UNDEFINED
&& cfa
->base
!= CFI_SP
) ||
3026 (cfi
->drap_reg
== CFI_UNDEFINED
&& cfa
->base
!= CFI_BP
)) {
3027 WARN_INSN(insn
, "unsupported stack pointer realignment");
3031 if (cfi
->drap_reg
!= CFI_UNDEFINED
) {
3032 /* drap: and imm, %rsp */
3033 cfa
->base
= cfi
->drap_reg
;
3034 cfa
->offset
= cfi
->stack_size
= 0;
3039 * Older versions of GCC (4.8ish) realign the stack
3040 * without DRAP, with a frame pointer.
3047 if (op
->dest
.reg
== CFI_SP
&& cfa
->base
== CFI_SP_INDIRECT
) {
3049 /* pop %rsp; # restore from a stack swizzle */
3054 if (!cfi
->drap
&& op
->dest
.reg
== cfa
->base
) {
3060 if (cfi
->drap
&& cfa
->base
== CFI_BP_INDIRECT
&&
3061 op
->dest
.reg
== cfi
->drap_reg
&&
3062 cfi
->drap_offset
== -cfi
->stack_size
) {
3064 /* drap: pop %drap */
3065 cfa
->base
= cfi
->drap_reg
;
3067 cfi
->drap_offset
= -1;
3069 } else if (cfi
->stack_size
== -regs
[op
->dest
.reg
].offset
) {
3072 restore_reg(cfi
, op
->dest
.reg
);
3075 cfi
->stack_size
-= 8;
3076 if (cfa
->base
== CFI_SP
)
3081 case OP_SRC_REG_INDIRECT
:
3082 if (!cfi
->drap
&& op
->dest
.reg
== cfa
->base
&&
3083 op
->dest
.reg
== CFI_BP
) {
3085 /* mov disp(%rsp), %rbp */
3087 cfa
->offset
= cfi
->stack_size
;
3090 if (cfi
->drap
&& op
->src
.reg
== CFI_BP
&&
3091 op
->src
.offset
== cfi
->drap_offset
) {
3093 /* drap: mov disp(%rbp), %drap */
3094 cfa
->base
= cfi
->drap_reg
;
3096 cfi
->drap_offset
= -1;
3099 if (cfi
->drap
&& op
->src
.reg
== CFI_BP
&&
3100 op
->src
.offset
== regs
[op
->dest
.reg
].offset
) {
3102 /* drap: mov disp(%rbp), %reg */
3103 restore_reg(cfi
, op
->dest
.reg
);
3105 } else if (op
->src
.reg
== cfa
->base
&&
3106 op
->src
.offset
== regs
[op
->dest
.reg
].offset
+ cfa
->offset
) {
3108 /* mov disp(%rbp), %reg */
3109 /* mov disp(%rsp), %reg */
3110 restore_reg(cfi
, op
->dest
.reg
);
3112 } else if (op
->src
.reg
== CFI_SP
&&
3113 op
->src
.offset
== regs
[op
->dest
.reg
].offset
+ cfi
->stack_size
) {
3115 /* mov disp(%rsp), %reg */
3116 restore_reg(cfi
, op
->dest
.reg
);
3122 WARN_INSN(insn
, "unknown stack-related instruction");
3130 cfi
->stack_size
+= 8;
3131 if (cfa
->base
== CFI_SP
)
3134 if (op
->src
.type
!= OP_SRC_REG
)
3138 if (op
->src
.reg
== cfa
->base
&& op
->src
.reg
== cfi
->drap_reg
) {
3140 /* drap: push %drap */
3141 cfa
->base
= CFI_BP_INDIRECT
;
3142 cfa
->offset
= -cfi
->stack_size
;
3144 /* save drap so we know when to restore it */
3145 cfi
->drap_offset
= -cfi
->stack_size
;
3147 } else if (op
->src
.reg
== CFI_BP
&& cfa
->base
== cfi
->drap_reg
) {
3149 /* drap: push %rbp */
3150 cfi
->stack_size
= 0;
3154 /* drap: push %reg */
3155 save_reg(cfi
, op
->src
.reg
, CFI_BP
, -cfi
->stack_size
);
3161 save_reg(cfi
, op
->src
.reg
, CFI_CFA
, -cfi
->stack_size
);
3164 /* detect when asm code uses rbp as a scratch register */
3165 if (opts
.stackval
&& insn_func(insn
) && op
->src
.reg
== CFI_BP
&&
3166 cfa
->base
!= CFI_BP
)
3167 cfi
->bp_scratch
= true;
3170 case OP_DEST_REG_INDIRECT
:
3173 if (op
->src
.reg
== cfa
->base
&& op
->src
.reg
== cfi
->drap_reg
) {
3175 /* drap: mov %drap, disp(%rbp) */
3176 cfa
->base
= CFI_BP_INDIRECT
;
3177 cfa
->offset
= op
->dest
.offset
;
3179 /* save drap offset so we know when to restore it */
3180 cfi
->drap_offset
= op
->dest
.offset
;
3183 /* drap: mov reg, disp(%rbp) */
3184 save_reg(cfi
, op
->src
.reg
, CFI_BP
, op
->dest
.offset
);
3187 } else if (op
->dest
.reg
== cfa
->base
) {
3189 /* mov reg, disp(%rbp) */
3190 /* mov reg, disp(%rsp) */
3191 save_reg(cfi
, op
->src
.reg
, CFI_CFA
,
3192 op
->dest
.offset
- cfi
->cfa
.offset
);
3194 } else if (op
->dest
.reg
== CFI_SP
) {
3196 /* mov reg, disp(%rsp) */
3197 save_reg(cfi
, op
->src
.reg
, CFI_CFA
,
3198 op
->dest
.offset
- cfi
->stack_size
);
3200 } else if (op
->src
.reg
== CFI_SP
&& op
->dest
.offset
== 0) {
3202 /* mov %rsp, (%reg); # setup a stack swizzle. */
3203 cfi
->vals
[op
->dest
.reg
].base
= CFI_SP_INDIRECT
;
3204 cfi
->vals
[op
->dest
.reg
].offset
= cfa
->offset
;
3210 if (op
->src
.type
!= OP_SRC_POP
&& op
->src
.type
!= OP_SRC_POPF
) {
3211 WARN_INSN(insn
, "unknown stack-related memory operation");
3216 cfi
->stack_size
-= 8;
3217 if (cfa
->base
== CFI_SP
)
3223 WARN_INSN(insn
, "unknown stack-related instruction");
3231 * The stack layouts of alternatives instructions can sometimes diverge when
3232 * they have stack modifications. That's fine as long as the potential stack
3233 * layouts don't conflict at any given potential instruction boundary.
3235 * Flatten the CFIs of the different alternative code streams (both original
3236 * and replacement) into a single shared CFI array which can be used to detect
3237 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3239 static int propagate_alt_cfi(struct objtool_file
*file
, struct instruction
*insn
)
3241 struct cfi_state
**alt_cfi
;
3244 if (!insn
->alt_group
)
3248 WARN("CFI missing");
3252 alt_cfi
= insn
->alt_group
->cfi
;
3253 group_off
= insn
->offset
- insn
->alt_group
->first_insn
->offset
;
3255 if (!alt_cfi
[group_off
]) {
3256 alt_cfi
[group_off
] = insn
->cfi
;
3258 if (cficmp(alt_cfi
[group_off
], insn
->cfi
)) {
3259 struct alt_group
*orig_group
= insn
->alt_group
->orig_group
?: insn
->alt_group
;
3260 struct instruction
*orig
= orig_group
->first_insn
;
3261 char *where
= offstr(insn
->sec
, insn
->offset
);
3262 WARN_INSN(orig
, "stack layout conflict in alternatives: %s", where
);
3271 static int handle_insn_ops(struct instruction
*insn
,
3272 struct instruction
*next_insn
,
3273 struct insn_state
*state
)
3275 struct stack_op
*op
;
3277 for (op
= insn
->stack_ops
; op
; op
= op
->next
) {
3279 if (update_cfi_state(insn
, next_insn
, &state
->cfi
, op
))
3282 if (!insn
->alt_group
)
3285 if (op
->dest
.type
== OP_DEST_PUSHF
) {
3286 if (!state
->uaccess_stack
) {
3287 state
->uaccess_stack
= 1;
3288 } else if (state
->uaccess_stack
>> 31) {
3289 WARN_INSN(insn
, "PUSHF stack exhausted");
3292 state
->uaccess_stack
<<= 1;
3293 state
->uaccess_stack
|= state
->uaccess
;
3296 if (op
->src
.type
== OP_SRC_POPF
) {
3297 if (state
->uaccess_stack
) {
3298 state
->uaccess
= state
->uaccess_stack
& 1;
3299 state
->uaccess_stack
>>= 1;
3300 if (state
->uaccess_stack
== 1)
3301 state
->uaccess_stack
= 0;
3309 static bool insn_cfi_match(struct instruction
*insn
, struct cfi_state
*cfi2
)
3311 struct cfi_state
*cfi1
= insn
->cfi
;
3315 WARN("CFI missing");
3319 if (memcmp(&cfi1
->cfa
, &cfi2
->cfa
, sizeof(cfi1
->cfa
))) {
3321 WARN_INSN(insn
, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3322 cfi1
->cfa
.base
, cfi1
->cfa
.offset
,
3323 cfi2
->cfa
.base
, cfi2
->cfa
.offset
);
3325 } else if (memcmp(&cfi1
->regs
, &cfi2
->regs
, sizeof(cfi1
->regs
))) {
3326 for (i
= 0; i
< CFI_NUM_REGS
; i
++) {
3327 if (!memcmp(&cfi1
->regs
[i
], &cfi2
->regs
[i
],
3328 sizeof(struct cfi_reg
)))
3331 WARN_INSN(insn
, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3332 i
, cfi1
->regs
[i
].base
, cfi1
->regs
[i
].offset
,
3333 i
, cfi2
->regs
[i
].base
, cfi2
->regs
[i
].offset
);
3337 } else if (cfi1
->type
!= cfi2
->type
) {
3339 WARN_INSN(insn
, "stack state mismatch: type1=%d type2=%d",
3340 cfi1
->type
, cfi2
->type
);
3342 } else if (cfi1
->drap
!= cfi2
->drap
||
3343 (cfi1
->drap
&& cfi1
->drap_reg
!= cfi2
->drap_reg
) ||
3344 (cfi1
->drap
&& cfi1
->drap_offset
!= cfi2
->drap_offset
)) {
3346 WARN_INSN(insn
, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3347 cfi1
->drap
, cfi1
->drap_reg
, cfi1
->drap_offset
,
3348 cfi2
->drap
, cfi2
->drap_reg
, cfi2
->drap_offset
);
3356 static inline bool func_uaccess_safe(struct symbol
*func
)
3359 return func
->uaccess_safe
;
3364 static inline const char *call_dest_name(struct instruction
*insn
)
3366 static char pvname
[19];
3370 if (insn_call_dest(insn
))
3371 return insn_call_dest(insn
)->name
;
3373 rel
= insn_reloc(NULL
, insn
);
3374 if (rel
&& !strcmp(rel
->sym
->name
, "pv_ops")) {
3375 idx
= (rel
->addend
/ sizeof(void *));
3376 snprintf(pvname
, sizeof(pvname
), "pv_ops[%d]", idx
);
3383 static bool pv_call_dest(struct objtool_file
*file
, struct instruction
*insn
)
3385 struct symbol
*target
;
3389 rel
= insn_reloc(file
, insn
);
3390 if (!rel
|| strcmp(rel
->sym
->name
, "pv_ops"))
3393 idx
= (arch_dest_reloc_offset(rel
->addend
) / sizeof(void *));
3395 if (file
->pv_ops
[idx
].clean
)
3398 file
->pv_ops
[idx
].clean
= true;
3400 list_for_each_entry(target
, &file
->pv_ops
[idx
].targets
, pv_target
) {
3401 if (!target
->sec
->noinstr
) {
3402 WARN("pv_ops[%d]: %s", idx
, target
->name
);
3403 file
->pv_ops
[idx
].clean
= false;
3407 return file
->pv_ops
[idx
].clean
;
3410 static inline bool noinstr_call_dest(struct objtool_file
*file
,
3411 struct instruction
*insn
,
3412 struct symbol
*func
)
3415 * We can't deal with indirect function calls at present;
3416 * assume they're instrumented.
3420 return pv_call_dest(file
, insn
);
3426 * If the symbol is from a noinstr section; we good.
3428 if (func
->sec
->noinstr
)
3432 * If the symbol is a static_call trampoline, we can't tell.
3434 if (func
->static_call_tramp
)
3438 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3439 * something 'BAD' happened. At the risk of taking the machine down,
3440 * let them proceed to get the message out.
3442 if (!strncmp(func
->name
, "__ubsan_handle_", 15))
3448 static int validate_call(struct objtool_file
*file
,
3449 struct instruction
*insn
,
3450 struct insn_state
*state
)
3452 if (state
->noinstr
&& state
->instr
<= 0 &&
3453 !noinstr_call_dest(file
, insn
, insn_call_dest(insn
))) {
3454 WARN_INSN(insn
, "call to %s() leaves .noinstr.text section", call_dest_name(insn
));
3458 if (state
->uaccess
&& !func_uaccess_safe(insn_call_dest(insn
))) {
3459 WARN_INSN(insn
, "call to %s() with UACCESS enabled", call_dest_name(insn
));
3464 WARN_INSN(insn
, "call to %s() with DF set", call_dest_name(insn
));
3471 static int validate_sibling_call(struct objtool_file
*file
,
3472 struct instruction
*insn
,
3473 struct insn_state
*state
)
3475 if (insn_func(insn
) && has_modified_stack_frame(insn
, state
)) {
3476 WARN_INSN(insn
, "sibling call from callable instruction with modified stack frame");
3480 return validate_call(file
, insn
, state
);
3483 static int validate_return(struct symbol
*func
, struct instruction
*insn
, struct insn_state
*state
)
3485 if (state
->noinstr
&& state
->instr
> 0) {
3486 WARN_INSN(insn
, "return with instrumentation enabled");
3490 if (state
->uaccess
&& !func_uaccess_safe(func
)) {
3491 WARN_INSN(insn
, "return with UACCESS enabled");
3495 if (!state
->uaccess
&& func_uaccess_safe(func
)) {
3496 WARN_INSN(insn
, "return with UACCESS disabled from a UACCESS-safe function");
3501 WARN_INSN(insn
, "return with DF set");
3505 if (func
&& has_modified_stack_frame(insn
, state
)) {
3506 WARN_INSN(insn
, "return with modified stack frame");
3510 if (state
->cfi
.bp_scratch
) {
3511 WARN_INSN(insn
, "BP used as a scratch register");
3518 static struct instruction
*next_insn_to_validate(struct objtool_file
*file
,
3519 struct instruction
*insn
)
3521 struct alt_group
*alt_group
= insn
->alt_group
;
3524 * Simulate the fact that alternatives are patched in-place. When the
3525 * end of a replacement alt_group is reached, redirect objtool flow to
3526 * the end of the original alt_group.
3528 * insn->alts->insn -> alt_group->first_insn
3530 * alt_group->last_insn
3531 * [alt_group->nop] -> next(orig_group->last_insn)
3534 if (alt_group
->nop
) {
3535 /* ->nop implies ->orig_group */
3536 if (insn
== alt_group
->last_insn
)
3537 return alt_group
->nop
;
3538 if (insn
== alt_group
->nop
)
3541 if (insn
== alt_group
->last_insn
&& alt_group
->orig_group
)
3545 return next_insn_same_sec(file
, insn
);
3548 return next_insn_same_sec(file
, alt_group
->orig_group
->last_insn
);
3552 * Follow the branch starting at the given instruction, and recursively follow
3553 * any other branches (jumps). Meanwhile, track the frame pointer state at
3554 * each instruction and validate all the rules described in
3555 * tools/objtool/Documentation/objtool.txt.
3557 static int validate_branch(struct objtool_file
*file
, struct symbol
*func
,
3558 struct instruction
*insn
, struct insn_state state
)
3560 struct alternative
*alt
;
3561 struct instruction
*next_insn
, *prev_insn
= NULL
;
3562 struct section
*sec
;
3569 next_insn
= next_insn_to_validate(file
, insn
);
3571 if (func
&& insn_func(insn
) && func
!= insn_func(insn
)->pfunc
) {
3572 /* Ignore KCFI type preambles, which always fall through */
3573 if (!strncmp(func
->name
, "__cfi_", 6) ||
3574 !strncmp(func
->name
, "__pfx_", 6))
3577 WARN("%s() falls through to next function %s()",
3578 func
->name
, insn_func(insn
)->name
);
3582 if (func
&& insn
->ignore
) {
3583 WARN_INSN(insn
, "BUG: why am I validating an ignored function?");
3587 visited
= VISITED_BRANCH
<< state
.uaccess
;
3588 if (insn
->visited
& VISITED_BRANCH_MASK
) {
3589 if (!insn
->hint
&& !insn_cfi_match(insn
, &state
.cfi
))
3592 if (insn
->visited
& visited
)
3599 state
.instr
+= insn
->instr
;
3602 if (insn
->restore
) {
3603 struct instruction
*save_insn
, *i
;
3608 sym_for_each_insn_continue_reverse(file
, func
, i
) {
3616 WARN_INSN(insn
, "no corresponding CFI save for CFI restore");
3620 if (!save_insn
->visited
) {
3621 WARN_INSN(insn
, "objtool isn't smart enough to handle this CFI save/restore combo");
3625 insn
->cfi
= save_insn
->cfi
;
3629 state
.cfi
= *insn
->cfi
;
3631 /* XXX track if we actually changed state.cfi */
3633 if (prev_insn
&& !cficmp(prev_insn
->cfi
, &state
.cfi
)) {
3634 insn
->cfi
= prev_insn
->cfi
;
3637 insn
->cfi
= cfi_hash_find_or_add(&state
.cfi
);
3641 insn
->visited
|= visited
;
3643 if (propagate_alt_cfi(file
, insn
))
3646 if (!insn
->ignore_alts
&& insn
->alts
) {
3647 bool skip_orig
= false;
3649 for (alt
= insn
->alts
; alt
; alt
= alt
->next
) {
3653 ret
= validate_branch(file
, func
, alt
->insn
, state
);
3656 BT_FUNC("(alt)", insn
);
3665 if (handle_insn_ops(insn
, next_insn
, &state
))
3668 switch (insn
->type
) {
3671 return validate_return(func
, insn
, &state
);
3674 case INSN_CALL_DYNAMIC
:
3675 ret
= validate_call(file
, insn
, &state
);
3679 if (opts
.stackval
&& func
&& !is_fentry_call(insn
) &&
3680 !has_valid_stack_frame(&state
)) {
3681 WARN_INSN(insn
, "call without frame pointer save/setup");
3690 case INSN_JUMP_CONDITIONAL
:
3691 case INSN_JUMP_UNCONDITIONAL
:
3692 if (is_sibling_call(insn
)) {
3693 ret
= validate_sibling_call(file
, insn
, &state
);
3697 } else if (insn
->jump_dest
) {
3698 ret
= validate_branch(file
, func
,
3699 insn
->jump_dest
, state
);
3702 BT_FUNC("(branch)", insn
);
3707 if (insn
->type
== INSN_JUMP_UNCONDITIONAL
)
3712 case INSN_JUMP_DYNAMIC
:
3713 case INSN_JUMP_DYNAMIC_CONDITIONAL
:
3714 if (is_sibling_call(insn
)) {
3715 ret
= validate_sibling_call(file
, insn
, &state
);
3720 if (insn
->type
== INSN_JUMP_DYNAMIC
)
3725 case INSN_CONTEXT_SWITCH
:
3726 if (func
&& (!next_insn
|| !next_insn
->hint
)) {
3727 WARN_INSN(insn
, "unsupported instruction in callable function");
3733 if (state
.uaccess
) {
3734 WARN_INSN(insn
, "recursive UACCESS enable");
3738 state
.uaccess
= true;
3742 if (!state
.uaccess
&& func
) {
3743 WARN_INSN(insn
, "redundant UACCESS disable");
3747 if (func_uaccess_safe(func
) && !state
.uaccess_stack
) {
3748 WARN_INSN(insn
, "UACCESS-safe disables UACCESS");
3752 state
.uaccess
= false;
3757 WARN_INSN(insn
, "recursive STD");
3765 if (!state
.df
&& func
) {
3766 WARN_INSN(insn
, "redundant CLD");
3781 if (state
.cfi
.cfa
.base
== CFI_UNDEFINED
)
3783 WARN("%s: unexpected end of section", sec
->name
);
3794 static int validate_unwind_hint(struct objtool_file
*file
,
3795 struct instruction
*insn
,
3796 struct insn_state
*state
)
3798 if (insn
->hint
&& !insn
->visited
&& !insn
->ignore
) {
3799 int ret
= validate_branch(file
, insn_func(insn
), insn
, *state
);
3800 if (ret
&& opts
.backtrace
)
3801 BT_FUNC("<=== (hint)", insn
);
3808 static int validate_unwind_hints(struct objtool_file
*file
, struct section
*sec
)
3810 struct instruction
*insn
;
3811 struct insn_state state
;
3817 init_insn_state(file
, &state
, sec
);
3820 sec_for_each_insn(file
, sec
, insn
)
3821 warnings
+= validate_unwind_hint(file
, insn
, &state
);
3823 for_each_insn(file
, insn
)
3824 warnings
+= validate_unwind_hint(file
, insn
, &state
);
3831 * Validate rethunk entry constraint: must untrain RET before the first RET.
3833 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3834 * before an actual RET instruction.
3836 static int validate_unret(struct objtool_file
*file
, struct instruction
*insn
)
3838 struct instruction
*next
, *dest
;
3839 int ret
, warnings
= 0;
3842 next
= next_insn_to_validate(file
, insn
);
3844 if (insn
->visited
& VISITED_UNRET
)
3847 insn
->visited
|= VISITED_UNRET
;
3849 if (!insn
->ignore_alts
&& insn
->alts
) {
3850 struct alternative
*alt
;
3851 bool skip_orig
= false;
3853 for (alt
= insn
->alts
; alt
; alt
= alt
->next
) {
3857 ret
= validate_unret(file
, alt
->insn
);
3860 BT_FUNC("(alt)", insn
);
3869 switch (insn
->type
) {
3871 case INSN_CALL_DYNAMIC
:
3872 case INSN_JUMP_DYNAMIC
:
3873 case INSN_JUMP_DYNAMIC_CONDITIONAL
:
3874 WARN_INSN(insn
, "early indirect call");
3877 case INSN_JUMP_UNCONDITIONAL
:
3878 case INSN_JUMP_CONDITIONAL
:
3879 if (!is_sibling_call(insn
)) {
3880 if (!insn
->jump_dest
) {
3881 WARN_INSN(insn
, "unresolved jump target after linking?!?");
3884 ret
= validate_unret(file
, insn
->jump_dest
);
3886 if (opts
.backtrace
) {
3887 BT_FUNC("(branch%s)", insn
,
3888 insn
->type
== INSN_JUMP_CONDITIONAL
? "-cond" : "");
3893 if (insn
->type
== INSN_JUMP_UNCONDITIONAL
)
3901 dest
= find_insn(file
, insn_call_dest(insn
)->sec
,
3902 insn_call_dest(insn
)->offset
);
3904 WARN("Unresolved function after linking!?: %s",
3905 insn_call_dest(insn
)->name
);
3909 ret
= validate_unret(file
, dest
);
3912 BT_FUNC("(call)", insn
);
3916 * If a call returns without error, it must have seen UNTRAIN_RET.
3917 * Therefore any non-error return is a success.
3922 WARN_INSN(insn
, "RET before UNTRAIN");
3926 if (insn
->retpoline_safe
)
3935 WARN_INSN(insn
, "teh end!");
3945 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3946 * VALIDATE_UNRET_END before RET.
3948 static int validate_unrets(struct objtool_file
*file
)
3950 struct instruction
*insn
;
3951 int ret
, warnings
= 0;
3953 for_each_insn(file
, insn
) {
3957 ret
= validate_unret(file
, insn
);
3959 WARN_INSN(insn
, "Failed UNRET validation");
3968 static int validate_retpoline(struct objtool_file
*file
)
3970 struct instruction
*insn
;
3973 for_each_insn(file
, insn
) {
3974 if (insn
->type
!= INSN_JUMP_DYNAMIC
&&
3975 insn
->type
!= INSN_CALL_DYNAMIC
&&
3976 insn
->type
!= INSN_RETURN
)
3979 if (insn
->retpoline_safe
)
3982 if (insn
->sec
->init
)
3985 if (insn
->type
== INSN_RETURN
) {
3987 WARN_INSN(insn
, "'naked' return found in RETHUNK build");
3991 WARN_INSN(insn
, "indirect %s found in RETPOLINE build",
3992 insn
->type
== INSN_JUMP_DYNAMIC
? "jump" : "call");
4001 static bool is_kasan_insn(struct instruction
*insn
)
4003 return (insn
->type
== INSN_CALL
&&
4004 !strcmp(insn_call_dest(insn
)->name
, "__asan_handle_no_return"));
4007 static bool is_ubsan_insn(struct instruction
*insn
)
4009 return (insn
->type
== INSN_CALL
&&
4010 !strcmp(insn_call_dest(insn
)->name
,
4011 "__ubsan_handle_builtin_unreachable"));
4014 static bool ignore_unreachable_insn(struct objtool_file
*file
, struct instruction
*insn
)
4017 struct instruction
*prev_insn
;
4019 if (insn
->ignore
|| insn
->type
== INSN_NOP
|| insn
->type
== INSN_TRAP
)
4023 * Ignore alternative replacement instructions. This can happen
4024 * when a whitelisted function uses one of the ALTERNATIVE macros.
4026 if (!strcmp(insn
->sec
->name
, ".altinstr_replacement") ||
4027 !strcmp(insn
->sec
->name
, ".altinstr_aux"))
4031 * Whole archive runs might encounter dead code from weak symbols.
4032 * This is where the linker will have dropped the weak symbol in
4033 * favour of a regular symbol, but leaves the code in place.
4035 * In this case we'll find a piece of code (whole function) that is not
4036 * covered by a !section symbol. Ignore them.
4038 if (opts
.link
&& !insn_func(insn
)) {
4039 int size
= find_symbol_hole_containing(insn
->sec
, insn
->offset
);
4040 unsigned long end
= insn
->offset
+ size
;
4042 if (!size
) /* not a hole */
4045 if (size
< 0) /* hole until the end */
4048 sec_for_each_insn_continue(file
, insn
) {
4050 * If we reach a visited instruction at or before the
4051 * end of the hole, ignore the unreachable.
4056 if (insn
->offset
>= end
)
4060 * If this hole jumps to a .cold function, mark it ignore too.
4062 if (insn
->jump_dest
&& insn_func(insn
->jump_dest
) &&
4063 strstr(insn_func(insn
->jump_dest
)->name
, ".cold")) {
4064 struct instruction
*dest
= insn
->jump_dest
;
4065 func_for_each_insn(file
, insn_func(dest
), dest
)
4066 dest
->ignore
= true;
4073 if (!insn_func(insn
))
4076 if (insn_func(insn
)->static_call_tramp
)
4080 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4081 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4082 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4083 * (or occasionally a JMP to UD2).
4085 * It may also insert a UD2 after calling a __noreturn function.
4087 prev_insn
= prev_insn_same_sec(file
, insn
);
4088 if (prev_insn
->dead_end
&&
4089 (insn
->type
== INSN_BUG
||
4090 (insn
->type
== INSN_JUMP_UNCONDITIONAL
&&
4091 insn
->jump_dest
&& insn
->jump_dest
->type
== INSN_BUG
)))
4095 * Check if this (or a subsequent) instruction is related to
4096 * CONFIG_UBSAN or CONFIG_KASAN.
4098 * End the search at 5 instructions to avoid going into the weeds.
4100 for (i
= 0; i
< 5; i
++) {
4102 if (is_kasan_insn(insn
) || is_ubsan_insn(insn
))
4105 if (insn
->type
== INSN_JUMP_UNCONDITIONAL
) {
4106 if (insn
->jump_dest
&&
4107 insn_func(insn
->jump_dest
) == insn_func(insn
)) {
4108 insn
= insn
->jump_dest
;
4115 if (insn
->offset
+ insn
->len
>= insn_func(insn
)->offset
+ insn_func(insn
)->len
)
4118 insn
= next_insn_same_sec(file
, insn
);
4124 static int add_prefix_symbol(struct objtool_file
*file
, struct symbol
*func
)
4126 struct instruction
*insn
, *prev
;
4127 struct cfi_state
*cfi
;
4129 insn
= find_insn(file
, func
->sec
, func
->offset
);
4133 for (prev
= prev_insn_same_sec(file
, insn
);
4135 prev
= prev_insn_same_sec(file
, prev
)) {
4138 if (prev
->type
!= INSN_NOP
)
4141 offset
= func
->offset
- prev
->offset
;
4143 if (offset
> opts
.prefix
)
4146 if (offset
< opts
.prefix
)
4149 elf_create_prefix_symbol(file
->elf
, func
, opts
.prefix
);
4158 * This can happen if stack validation isn't enabled or the
4159 * function is annotated with STACK_FRAME_NON_STANDARD.
4164 /* Propagate insn->cfi to the prefix code */
4165 cfi
= cfi_hash_find_or_add(insn
->cfi
);
4166 for (; prev
!= insn
; prev
= next_insn_same_sec(file
, prev
))
4172 static int add_prefix_symbols(struct objtool_file
*file
)
4174 struct section
*sec
;
4175 struct symbol
*func
;
4178 for_each_sec(file
, sec
) {
4179 if (!(sec
->sh
.sh_flags
& SHF_EXECINSTR
))
4182 sec_for_each_sym(sec
, func
) {
4183 if (func
->type
!= STT_FUNC
)
4186 add_prefix_symbol(file
, func
);
4193 static int validate_symbol(struct objtool_file
*file
, struct section
*sec
,
4194 struct symbol
*sym
, struct insn_state
*state
)
4196 struct instruction
*insn
;
4200 WARN("%s() is missing an ELF size annotation", sym
->name
);
4204 if (sym
->pfunc
!= sym
|| sym
->alias
!= sym
)
4207 insn
= find_insn(file
, sec
, sym
->offset
);
4208 if (!insn
|| insn
->ignore
|| insn
->visited
)
4211 state
->uaccess
= sym
->uaccess_safe
;
4213 ret
= validate_branch(file
, insn_func(insn
), insn
, *state
);
4214 if (ret
&& opts
.backtrace
)
4215 BT_FUNC("<=== (sym)", insn
);
4219 static int validate_section(struct objtool_file
*file
, struct section
*sec
)
4221 struct insn_state state
;
4222 struct symbol
*func
;
4225 sec_for_each_sym(sec
, func
) {
4226 if (func
->type
!= STT_FUNC
)
4229 init_insn_state(file
, &state
, sec
);
4230 set_func_state(&state
.cfi
);
4232 warnings
+= validate_symbol(file
, sec
, func
, &state
);
4238 static int validate_noinstr_sections(struct objtool_file
*file
)
4240 struct section
*sec
;
4243 sec
= find_section_by_name(file
->elf
, ".noinstr.text");
4245 warnings
+= validate_section(file
, sec
);
4246 warnings
+= validate_unwind_hints(file
, sec
);
4249 sec
= find_section_by_name(file
->elf
, ".entry.text");
4251 warnings
+= validate_section(file
, sec
);
4252 warnings
+= validate_unwind_hints(file
, sec
);
4255 sec
= find_section_by_name(file
->elf
, ".cpuidle.text");
4257 warnings
+= validate_section(file
, sec
);
4258 warnings
+= validate_unwind_hints(file
, sec
);
4264 static int validate_functions(struct objtool_file
*file
)
4266 struct section
*sec
;
4269 for_each_sec(file
, sec
) {
4270 if (!(sec
->sh
.sh_flags
& SHF_EXECINSTR
))
4273 warnings
+= validate_section(file
, sec
);
4279 static void mark_endbr_used(struct instruction
*insn
)
4281 if (!list_empty(&insn
->call_node
))
4282 list_del_init(&insn
->call_node
);
4285 static bool noendbr_range(struct objtool_file
*file
, struct instruction
*insn
)
4287 struct symbol
*sym
= find_symbol_containing(insn
->sec
, insn
->offset
-1);
4288 struct instruction
*first
;
4293 first
= find_insn(file
, sym
->sec
, sym
->offset
);
4297 if (first
->type
!= INSN_ENDBR
&& !first
->noendbr
)
4300 return insn
->offset
== sym
->offset
+ sym
->len
;
4303 static int validate_ibt_insn(struct objtool_file
*file
, struct instruction
*insn
)
4305 struct instruction
*dest
;
4306 struct reloc
*reloc
;
4311 * Looking for function pointer load relocations. Ignore
4312 * direct/indirect branches:
4314 switch (insn
->type
) {
4316 case INSN_CALL_DYNAMIC
:
4317 case INSN_JUMP_CONDITIONAL
:
4318 case INSN_JUMP_UNCONDITIONAL
:
4319 case INSN_JUMP_DYNAMIC
:
4320 case INSN_JUMP_DYNAMIC_CONDITIONAL
:
4328 for (reloc
= insn_reloc(file
, insn
);
4330 reloc
= find_reloc_by_dest_range(file
->elf
, insn
->sec
,
4332 (insn
->offset
+ insn
->len
) - (reloc
->offset
+ 1))) {
4335 * static_call_update() references the trampoline, which
4336 * doesn't have (or need) ENDBR. Skip warning in that case.
4338 if (reloc
->sym
->static_call_tramp
)
4341 off
= reloc
->sym
->offset
;
4342 if (reloc
->type
== R_X86_64_PC32
|| reloc
->type
== R_X86_64_PLT32
)
4343 off
+= arch_dest_reloc_offset(reloc
->addend
);
4345 off
+= reloc
->addend
;
4347 dest
= find_insn(file
, reloc
->sym
->sec
, off
);
4351 if (dest
->type
== INSN_ENDBR
) {
4352 mark_endbr_used(dest
);
4356 if (insn_func(dest
) && insn_func(dest
) == insn_func(insn
)) {
4358 * Anything from->to self is either _THIS_IP_ or
4361 * There is no sane way to annotate _THIS_IP_ since the
4362 * compiler treats the relocation as a constant and is
4363 * happy to fold in offsets, skewing any annotation we
4364 * do, leading to vast amounts of false-positives.
4366 * There's also compiler generated _THIS_IP_ through
4367 * KCOV and such which we have no hope of annotating.
4369 * As such, blanket accept self-references without
4376 * Accept anything ANNOTATE_NOENDBR.
4382 * Accept if this is the instruction after a symbol
4383 * that is (no)endbr -- typical code-range usage.
4385 if (noendbr_range(file
, dest
))
4388 WARN_INSN(insn
, "relocation to !ENDBR: %s", offstr(dest
->sec
, dest
->offset
));
4396 static int validate_ibt_data_reloc(struct objtool_file
*file
,
4397 struct reloc
*reloc
)
4399 struct instruction
*dest
;
4401 dest
= find_insn(file
, reloc
->sym
->sec
,
4402 reloc
->sym
->offset
+ reloc
->addend
);
4406 if (dest
->type
== INSN_ENDBR
) {
4407 mark_endbr_used(dest
);
4414 WARN_FUNC("data relocation to !ENDBR: %s",
4415 reloc
->sec
->base
, reloc
->offset
,
4416 offstr(dest
->sec
, dest
->offset
));
4422 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4423 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4424 * NOPs) later, in create_ibt_endbr_seal_sections().
4426 static int validate_ibt(struct objtool_file
*file
)
4428 struct section
*sec
;
4429 struct reloc
*reloc
;
4430 struct instruction
*insn
;
4433 for_each_insn(file
, insn
)
4434 warnings
+= validate_ibt_insn(file
, insn
);
4436 for_each_sec(file
, sec
) {
4438 /* Already done by validate_ibt_insn() */
4439 if (sec
->sh
.sh_flags
& SHF_EXECINSTR
)
4446 * These sections can reference text addresses, but not with
4447 * the intent to indirect branch to them.
4449 if ((!strncmp(sec
->name
, ".discard", 8) &&
4450 strcmp(sec
->name
, ".discard.ibt_endbr_noseal")) ||
4451 !strncmp(sec
->name
, ".debug", 6) ||
4452 !strcmp(sec
->name
, ".altinstructions") ||
4453 !strcmp(sec
->name
, ".ibt_endbr_seal") ||
4454 !strcmp(sec
->name
, ".orc_unwind_ip") ||
4455 !strcmp(sec
->name
, ".parainstructions") ||
4456 !strcmp(sec
->name
, ".retpoline_sites") ||
4457 !strcmp(sec
->name
, ".smp_locks") ||
4458 !strcmp(sec
->name
, ".static_call_sites") ||
4459 !strcmp(sec
->name
, "_error_injection_whitelist") ||
4460 !strcmp(sec
->name
, "_kprobe_blacklist") ||
4461 !strcmp(sec
->name
, "__bug_table") ||
4462 !strcmp(sec
->name
, "__ex_table") ||
4463 !strcmp(sec
->name
, "__jump_table") ||
4464 !strcmp(sec
->name
, "__mcount_loc") ||
4465 !strcmp(sec
->name
, ".kcfi_traps") ||
4466 strstr(sec
->name
, "__patchable_function_entries"))
4469 list_for_each_entry(reloc
, &sec
->reloc
->reloc_list
, list
)
4470 warnings
+= validate_ibt_data_reloc(file
, reloc
);
4476 static int validate_sls(struct objtool_file
*file
)
4478 struct instruction
*insn
, *next_insn
;
4481 for_each_insn(file
, insn
) {
4482 next_insn
= next_insn_same_sec(file
, insn
);
4484 if (insn
->retpoline_safe
)
4487 switch (insn
->type
) {
4489 if (!next_insn
|| next_insn
->type
!= INSN_TRAP
) {
4490 WARN_INSN(insn
, "missing int3 after ret");
4495 case INSN_JUMP_DYNAMIC
:
4496 if (!next_insn
|| next_insn
->type
!= INSN_TRAP
) {
4497 WARN_INSN(insn
, "missing int3 after indirect jump");
4509 static int validate_reachable_instructions(struct objtool_file
*file
)
4511 struct instruction
*insn
;
4513 if (file
->ignore_unreachables
)
4516 for_each_insn(file
, insn
) {
4517 if (insn
->visited
|| ignore_unreachable_insn(file
, insn
))
4520 WARN_INSN(insn
, "unreachable instruction");
4527 int check(struct objtool_file
*file
)
4529 int ret
, warnings
= 0;
4531 arch_initial_func_cfi_state(&initial_func_cfi
);
4532 init_cfi_state(&init_cfi
);
4533 init_cfi_state(&func_cfi
);
4534 set_func_state(&func_cfi
);
4536 if (!cfi_hash_alloc(1UL << (file
->elf
->symbol_bits
- 3)))
4539 cfi_hash_add(&init_cfi
);
4540 cfi_hash_add(&func_cfi
);
4542 ret
= decode_sections(file
);
4551 if (opts
.retpoline
) {
4552 ret
= validate_retpoline(file
);
4558 if (opts
.stackval
|| opts
.orc
|| opts
.uaccess
) {
4559 ret
= validate_functions(file
);
4564 ret
= validate_unwind_hints(file
, NULL
);
4570 ret
= validate_reachable_instructions(file
);
4576 } else if (opts
.noinstr
) {
4577 ret
= validate_noinstr_sections(file
);
4585 * Must be after validate_branch() and friends, it plays
4586 * further games with insn->visited.
4588 ret
= validate_unrets(file
);
4595 ret
= validate_ibt(file
);
4602 ret
= validate_sls(file
);
4608 if (opts
.static_call
) {
4609 ret
= create_static_call_sections(file
);
4615 if (opts
.retpoline
) {
4616 ret
= create_retpoline_sites_sections(file
);
4623 ret
= create_cfi_sections(file
);
4630 ret
= create_return_sites_sections(file
);
4635 if (opts
.hack_skylake
) {
4636 ret
= create_direct_call_sections(file
);
4644 ret
= create_mcount_loc_sections(file
);
4651 ret
= add_prefix_symbols(file
);
4658 ret
= create_ibt_endbr_seal_sections(file
);
4664 if (opts
.orc
&& nr_insns
) {
4665 ret
= orc_create(file
);
4673 printf("nr_insns_visited: %ld\n", nr_insns_visited
);
4674 printf("nr_cfi: %ld\n", nr_cfi
);
4675 printf("nr_cfi_reused: %ld\n", nr_cfi_reused
);
4676 printf("nr_cfi_cache: %ld\n", nr_cfi_cache
);
4681 * For now, don't fail the kernel build on fatal warnings. These
4682 * errors are still fairly common due to the growing matrix of
4683 * supported toolchains and their recent pace of change.