]> git.ipfire.org Git - thirdparty/linux.git/blame - tools/objtool/check.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86/entry: Move thunk restore code into thunk functions
[thirdparty/linux.git] / tools / objtool / check.c
CommitLineData
1ccea77e 1// SPDX-License-Identifier: GPL-2.0-or-later
dcc914f4
JP		 2/*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
dcc914f4
JP		 4 */
5
6#include <string.h>
7#include <stdlib.h>
22682a07 8#include <inttypes.h>
8b946cc3 9#include <sys/mman.h>
dcc914f4 10
7786032e
VG		 11#include <arch/elf.h>
12#include <objtool/builtin.h>
13#include <objtool/cfi.h>
14#include <objtool/arch.h>
15#include <objtool/check.h>
16#include <objtool/special.h>
17#include <objtool/warn.h>
18#include <objtool/endianness.h>
dcc914f4 19
f7515d9f 20#include <linux/objtool_types.h>
dcc914f4
JP		 21#include <linux/hashtable.h>
22#include <linux/kernel.h>
1e7e4788 23#include <linux/static_call_types.h>
dcc914f4 24
dcc914f4 25struct alternative {
d5406654 26 struct alternative *next;
dcc914f4 27 struct instruction *insn;
764eef4b 28 bool skip_orig;
dcc914f4
JP		 29};
30
8b946cc3
PZ		 31static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32
33static struct cfi_init_state initial_func_cfi;
34static struct cfi_state init_cfi;
35static struct cfi_state func_cfi;
dcc914f4 36
627fce14
JP		 37struct instruction *find_insn(struct objtool_file *file,
38 struct section *sec, unsigned long offset)
dcc914f4
JP		 39{
40 struct instruction *insn;
41
87ecb582 42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
dcc914f4
JP		 43 if (insn->sec == sec && insn->offset == offset)
44 return insn;
87ecb582 45 }
dcc914f4
JP		 46
47 return NULL;
48}
49
1c34496e
PZ		 50struct instruction *next_insn_same_sec(struct objtool_file *file,
51 struct instruction *insn)
dcc914f4 52{
1c34496e
PZ		 53 if (insn->idx == INSN_CHUNK_MAX)
54 return find_insn(file, insn->sec, insn->offset + insn->len);
dcc914f4 55
1c34496e
PZ		 56 insn++;
57 if (!insn->len)
dcc914f4
JP		 58 return NULL;
59
1c34496e 60 return insn;
dcc914f4
JP		 61}
62
13810435
JP		 63static struct instruction *next_insn_same_func(struct objtool_file *file,
64 struct instruction *insn)
65{
1c34496e 66 struct instruction *next = next_insn_same_sec(file, insn);
dbcdbdfd 67 struct symbol *func = insn_func(insn);
13810435
JP		 68
69 if (!func)
70 return NULL;
71
1c34496e 72 if (next && insn_func(next) == func)
13810435
JP		 73 return next;
74
75 /* Check if we're already in the subfunction: */
76 if (func == func->cfunc)
77 return NULL;
78
79 /* Move to the subfunction: */
80 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
81}
82
1c34496e
PZ		 83static struct instruction *prev_insn_same_sec(struct objtool_file *file,
84 struct instruction *insn)
85{
86 if (insn->idx == 0) {
87 if (insn->prev_len)
88 return find_insn(file, insn->sec, insn->offset - insn->prev_len);
89 return NULL;
90 }
91
92 return insn - 1;
93}
94
1119d265 95static struct instruction *prev_insn_same_sym(struct objtool_file *file,
1c34496e 96 struct instruction *insn)
1119d265 97{
1c34496e 98 struct instruction *prev = prev_insn_same_sec(file, insn);
1119d265 99
1c34496e 100 if (prev && insn_func(prev) == insn_func(insn))
1119d265
JP		 101 return prev;
102
103 return NULL;
104}
105
1c34496e
PZ		 106#define for_each_insn(file, insn) \
107 for (struct section *__sec, *__fake = (struct section *)1; \
108 __fake; __fake = NULL) \
109 for_each_sec(file, __sec) \
110 sec_for_each_insn(file, __sec, insn)
111
f0f70adb 112#define func_for_each_insn(file, func, insn) \
13810435
JP		 113 for (insn = find_insn(file, func->sec, func->offset); \
114 insn; \
115 insn = next_insn_same_func(file, insn))
116
dbf4aeb0
PZ		 117#define sym_for_each_insn(file, sym, insn) \
118 for (insn = find_insn(file, sym->sec, sym->offset); \
1c34496e
PZ		 119 insn && insn->offset < sym->offset + sym->len; \
120 insn = next_insn_same_sec(file, insn))
dcc914f4 121
dbf4aeb0 122#define sym_for_each_insn_continue_reverse(file, sym, insn) \
1c34496e
PZ		 123 for (insn = prev_insn_same_sec(file, insn); \
124 insn && insn->offset >= sym->offset; \
125 insn = prev_insn_same_sec(file, insn))
dcc914f4
JP		 126
127#define sec_for_each_insn_from(file, insn) \
128 for (; insn; insn = next_insn_same_sec(file, insn))
129
baa41469
JP		 130#define sec_for_each_insn_continue(file, insn) \
131 for (insn = next_insn_same_sec(file, insn); insn; \
132 insn = next_insn_same_sec(file, insn))
dcc914f4 133
c6f5dc28
PZ		 134static inline struct symbol *insn_call_dest(struct instruction *insn)
135{
136 if (insn->type == INSN_JUMP_DYNAMIC ||
137 insn->type == INSN_CALL_DYNAMIC)
138 return NULL;
139
140 return insn->_call_dest;
141}
142
143static inline struct reloc *insn_jump_table(struct instruction *insn)
144{
145 if (insn->type == INSN_JUMP_DYNAMIC ||
146 insn->type == INSN_CALL_DYNAMIC)
147 return insn->_jump_table;
148
149 return NULL;
150}
151
99033461
JP		 152static bool is_jump_table_jump(struct instruction *insn)
153{
154 struct alt_group *alt_group = insn->alt_group;
155
c6f5dc28 156 if (insn_jump_table(insn))
99033461
JP		 157 return true;
158
159 /* Retpoline alternative for a jump table? */
160 return alt_group && alt_group->orig_group &&
c6f5dc28 161 insn_jump_table(alt_group->orig_group->first_insn);
99033461
JP		 162}
163
0c1ddd33
JP		 164static bool is_sibling_call(struct instruction *insn)
165{
ecf11ba4 166 /*
5a9c361a 167 * Assume only STT_FUNC calls have jump-tables.
ecf11ba4 168 */
5a9c361a
PZ		 169 if (insn_func(insn)) {
170 /* An indirect jump is either a sibling call or a jump to a table. */
171 if (insn->type == INSN_JUMP_DYNAMIC)
172 return !is_jump_table_jump(insn);
173 }
0c1ddd33 174
c6f5dc28
PZ		 175 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
176 return (is_static_jump(insn) && insn_call_dest(insn));
0c1ddd33
JP		 177}
178
dcc914f4
JP		 179/*
180 * This checks to see if the given function is a "noreturn" function.
181 *
182 * For global functions which are outside the scope of this object file, we
183 * have to keep a manual list of them.
184 *
185 * For local functions, we have to detect them manually by simply looking for
186 * the lack of a return instruction.
dcc914f4 187 */
8e25c9f8
JP		 188static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
189 int recursion)
dcc914f4
JP		 190{
191 int i;
192 struct instruction *insn;
193 bool empty = true;
194
6245ce4a 195#define NORETURN(func) __stringify(func),
dcc914f4 196 static const char * const global_noreturns[] = {
6245ce4a 197#include "noreturns.h"
dcc914f4 198 };
6245ce4a 199#undef NORETURN
dcc914f4 200
c9bab22b
JP		 201 if (!func)
202 return false;
203
1c47c875 204 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
dcc914f4
JP		 205 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
206 if (!strcmp(func->name, global_noreturns[i]))
8e25c9f8 207 return true;
dcc914f4 208
1c47c875
JP		 209 if (func->bind == STB_WEAK)
210 return false;
211
13810435 212 if (!func->len)
8e25c9f8 213 return false;
dcc914f4 214
13810435 215 insn = find_insn(file, func->sec, func->offset);
5f6e430f 216 if (!insn || !insn_func(insn))
8e25c9f8 217 return false;
13810435 218
f0f70adb 219 func_for_each_insn(file, func, insn) {
dcc914f4
JP		 220 empty = false;
221
222 if (insn->type == INSN_RETURN)
8e25c9f8 223 return false;
dcc914f4
JP		 224 }
225
226 if (empty)
8e25c9f8 227 return false;
dcc914f4
JP		 228
229 /*
230 * A function can have a sibling call instead of a return. In that
231 * case, the function's dead-end status depends on whether the target
232 * of the sibling call returns.
233 */
f0f70adb 234 func_for_each_insn(file, func, insn) {
0c1ddd33 235 if (is_sibling_call(insn)) {
dcc914f4 236 struct instruction *dest = insn->jump_dest;
dcc914f4
JP		 237
238 if (!dest)
239 /* sibling call to another file */
8e25c9f8 240 return false;
dcc914f4 241
0c1ddd33
JP		 242 /* local sibling call */
243 if (recursion == 5) {
244 /*
245 * Infinite recursion: two functions have
246 * sibling calls to each other. This is a very
247 * rare case. It means they aren't dead ends.
248 */
249 return false;
dcc914f4 250 }
dcc914f4 251
dbcdbdfd 252 return __dead_end_function(file, insn_func(dest), recursion+1);
0c1ddd33 253 }
dcc914f4
JP		 254 }
255
8e25c9f8 256 return true;
dcc914f4
JP		 257}
258
8e25c9f8 259static bool dead_end_function(struct objtool_file *file, struct symbol *func)
dcc914f4
JP		 260{
261 return __dead_end_function(file, func, 0);
262}
263
e7c0219b 264static void init_cfi_state(struct cfi_state *cfi)
baa41469
JP		 265{
266 int i;
267
dd88a0a0 268 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b
PZ		 269 cfi->regs[i].base = CFI_UNDEFINED;
270 cfi->vals[i].base = CFI_UNDEFINED;
dd88a0a0 271 }
e7c0219b
PZ		 272 cfi->cfa.base = CFI_UNDEFINED;
273 cfi->drap_reg = CFI_UNDEFINED;
274 cfi->drap_offset = -1;
275}
276
753da417
JP		 277static void init_insn_state(struct objtool_file *file, struct insn_state *state,
278 struct section *sec)
e7c0219b
PZ		 279{
280 memset(state, 0, sizeof(*state));
281 init_cfi_state(&state->cfi);
932f8e98
PZ		 282
283 /*
284 * We need the full vmlinux for noinstr validation, otherwise we can
c6f5dc28
PZ		 285 * not correctly determine insn_call_dest(insn)->sec (external symbols
286 * do not have a section).
932f8e98 287 */
753da417 288 if (opts.link && opts.noinstr && sec)
932f8e98 289 state->noinstr = sec->noinstr;
baa41469
JP		 290}
291
8b946cc3
PZ		 292static struct cfi_state *cfi_alloc(void)
293{
294 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
295 if (!cfi) {
296 WARN("calloc failed");
297 exit(1);
298 }
299 nr_cfi++;
300 return cfi;
301}
302
303static int cfi_bits;
304static struct hlist_head *cfi_hash;
305
306static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
307{
308 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
309 (void *)cfi2 + sizeof(cfi2->hash),
310 sizeof(struct cfi_state) - sizeof(struct hlist_node));
311}
312
313static inline u32 cfi_key(struct cfi_state *cfi)
314{
315 return jhash((void *)cfi + sizeof(cfi->hash),
316 sizeof(*cfi) - sizeof(cfi->hash), 0);
317}
318
319static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
320{
321 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
322 struct cfi_state *obj;
323
324 hlist_for_each_entry(obj, head, hash) {
325 if (!cficmp(cfi, obj)) {
326 nr_cfi_cache++;
327 return obj;
328 }
329 }
330
331 obj = cfi_alloc();
332 *obj = *cfi;
333 hlist_add_head(&obj->hash, head);
334
335 return obj;
336}
337
338static void cfi_hash_add(struct cfi_state *cfi)
339{
340 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
341
342 hlist_add_head(&cfi->hash, head);
343}
344
345static void *cfi_hash_alloc(unsigned long size)
346{
347 cfi_bits = max(10, ilog2(size));
348 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
349 PROT_READ|PROT_WRITE,
350 MAP_PRIVATE|MAP_ANON, -1, 0);
351 if (cfi_hash == (void *)-1L) {
352 WARN("mmap fail cfi_hash");
353 cfi_hash = NULL;
2daf7fab 354 } else if (opts.stats) {
8b946cc3
PZ		 355 printf("cfi_bits: %d\n", cfi_bits);
356 }
357
358 return cfi_hash;
359}
360
361static unsigned long nr_insns;
362static unsigned long nr_insns_visited;
363
dcc914f4
JP		 364/*
365 * Call the arch-specific instruction decoder for all the instructions and add
366 * them to the global instruction list.
367 */
368static int decode_instructions(struct objtool_file *file)
369{
370 struct section *sec;
371 struct symbol *func;
372 unsigned long offset;
373 struct instruction *insn;
374 int ret;
375
baa41469 376 for_each_sec(file, sec) {
1c34496e
PZ		 377 struct instruction *insns = NULL;
378 u8 prev_len = 0;
379 u8 idx = 0;
dcc914f4
JP		 380
381 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
382 continue;
383
627fce14
JP		 384 if (strcmp(sec->name, ".altinstr_replacement") &&
385 strcmp(sec->name, ".altinstr_aux") &&
386 strncmp(sec->name, ".discard.", 9))
387 sec->text = true;
388
0cc9ac8d 389 if (!strcmp(sec->name, ".noinstr.text") ||
951ddecf 390 !strcmp(sec->name, ".entry.text") ||
2b5a0e42 391 !strcmp(sec->name, ".cpuidle.text") ||
951ddecf 392 !strncmp(sec->name, ".text.__x86.", 12))
c4a33939
PZ		 393 sec->noinstr = true;
394
6644ee84
PZ		 395 /*
396 * .init.text code is ran before userspace and thus doesn't
397 * strictly need retpolines, except for modules which are
398 * loaded late, they very much do need retpoline in their
399 * .init.text
400 */
401 if (!strcmp(sec->name, ".init.text") && !opts.module)
402 sec->init = true;
403
fe255fe6 404 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
1c34496e
PZ		 405 if (!insns || idx == INSN_CHUNK_MAX) {
406 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
407 if (!insns) {
408 WARN("malloc failed");
409 return -1;
410 }
411 idx = 0;
412 } else {
413 idx++;
baa41469 414 }
1c34496e
PZ		 415 insn = &insns[idx];
416 insn->idx = idx;
baa41469 417
1c34496e 418 INIT_LIST_HEAD(&insn->call_node);
dcc914f4
JP		 419 insn->sec = sec;
420 insn->offset = offset;
1c34496e 421 insn->prev_len = prev_len;
dcc914f4 422
db2b0c5d 423 ret = arch_decode_instruction(file, sec, offset,
fe255fe6 424 sec->sh.sh_size - offset,
20a55463 425 insn);
dcc914f4 426 if (ret)
1c34496e
PZ		 427 return ret;
428
429 prev_len = insn->len;
dcc914f4 430
0e5b613b
PZ		 431 /*
432 * By default, "ud2" is a dead end unless otherwise
433 * annotated, because GCC 7 inserts it for certain
434 * divide-by-zero cases.
435 */
436 if (insn->type == INSN_BUG)
437 insn->dead_end = true;
438
87ecb582 439 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
1e11f3fd 440 nr_insns++;
dcc914f4
JP		 441 }
442
1c34496e
PZ		 443// printf("%s: last chunk used: %d\n", sec->name, (int)idx);
444
9290e772 445 sec_for_each_sym(sec, func) {
dbcdbdfd
PZ		 446 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
447 continue;
448
cad90e53
NP		 449 if (func->offset == sec->sh.sh_size) {
450 /* Heuristic: likely an "end" symbol */
451 if (func->type == STT_NOTYPE)
452 continue;
453 WARN("%s(): STT_FUNC at end of section",
454 func->name);
455 return -1;
456 }
457
dbcdbdfd 458 if (func->return_thunk || func->alias != func)
dcc914f4
JP		 459 continue;
460
461 if (!find_insn(file, sec, func->offset)) {
462 WARN("%s(): can't find starting instruction",
463 func->name);
464 return -1;
465 }
466
08f87a93 467 sym_for_each_insn(file, func, insn) {
dbcdbdfd
PZ		 468 insn->sym = func;
469 if (func->type == STT_FUNC &&
470 insn->type == INSN_ENDBR &&
471 list_empty(&insn->call_node)) {
472 if (insn->offset == func->offset) {
89bc853e 473 list_add_tail(&insn->call_node, &file->endbr_list);
08f87a93
PZ		 474 file->nr_endbr++;
475 } else {
476 file->nr_endbr_int++;
477 }
478 }
479 }
dcc914f4
JP		 480 }
481 }
482
2daf7fab 483 if (opts.stats)
1e11f3fd
PZ		 484 printf("nr_insns: %lu\n", nr_insns);
485
dcc914f4
JP		 486 return 0;
487}
488
db2b0c5d
PZ		 489/*
490 * Read the pv_ops[] .data table to find the static initialized values.
491 */
492static int add_pv_ops(struct objtool_file *file, const char *symname)
493{
494 struct symbol *sym, *func;
495 unsigned long off, end;
496 struct reloc *rel;
497 int idx;
498
499 sym = find_symbol_by_name(file->elf, symname);
500 if (!sym)
501 return 0;
502
503 off = sym->offset;
504 end = off + sym->len;
505 for (;;) {
506 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
507 if (!rel)
508 break;
509
510 func = rel->sym;
511 if (func->type == STT_SECTION)
512 func = find_symbol_by_offset(rel->sym->sec, rel->addend);
513
514 idx = (rel->offset - sym->offset) / sizeof(unsigned long);
515
516 objtool_pv_add(file, idx, func);
517
518 off = rel->offset + 1;
519 if (off > end)
520 break;
521 }
522
523 return 0;
524}
525
526/*
527 * Allocate and initialize file->pv_ops[].
528 */
529static int init_pv_ops(struct objtool_file *file)
530{
531 static const char *pv_ops_tables[] = {
532 "pv_ops",
533 "xen_cpu_ops",
534 "xen_irq_ops",
535 "xen_mmu_ops",
536 NULL,
537 };
538 const char *pv_ops;
539 struct symbol *sym;
540 int idx, nr;
541
2daf7fab 542 if (!opts.noinstr)
db2b0c5d
PZ		 543 return 0;
544
545 file->pv_ops = NULL;
546
547 sym = find_symbol_by_name(file->elf, "pv_ops");
548 if (!sym)
549 return 0;
550
551 nr = sym->len / sizeof(unsigned long);
552 file->pv_ops = calloc(sizeof(struct pv_state), nr);
553 if (!file->pv_ops)
554 return -1;
555
556 for (idx = 0; idx < nr; idx++)
557 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
558
559 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
560 add_pv_ops(file, pv_ops);
561
562 return 0;
563}
564
6b5dd716
ST		 565static struct instruction *find_last_insn(struct objtool_file *file,
566 struct section *sec)
567{
568 struct instruction *insn = NULL;
569 unsigned int offset;
fe255fe6 570 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
6b5dd716 571
fe255fe6 572 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
6b5dd716
ST		 573 insn = find_insn(file, sec, offset);
574
575 return insn;
576}
577
dcc914f4 578/*
649ea4d5 579 * Mark "ud2" instructions and manually annotated dead ends.
dcc914f4
JP		 580 */
581static int add_dead_ends(struct objtool_file *file)
582{
583 struct section *sec;
f1974222 584 struct reloc *reloc;
dcc914f4 585 struct instruction *insn;
dcc914f4 586
649ea4d5
JP		 587 /*
588 * Check for manually annotated dead ends.
589 */
dcc914f4
JP		 590 sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
591 if (!sec)
649ea4d5 592 goto reachable;
dcc914f4 593
f1974222
MH		 594 list_for_each_entry(reloc, &sec->reloc_list, list) {
595 if (reloc->sym->type != STT_SECTION) {
dcc914f4
JP		 596 WARN("unexpected relocation symbol type in %s", sec->name);
597 return -1;
598 }
f1974222 599 insn = find_insn(file, reloc->sym->sec, reloc->addend);
dcc914f4 600 if (insn)
1c34496e 601 insn = prev_insn_same_sec(file, insn);
fe255fe6 602 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
f1974222 603 insn = find_last_insn(file, reloc->sym->sec);
6b5dd716 604 if (!insn) {
22682a07 605 WARN("can't find unreachable insn at %s+0x%" PRIx64,
f1974222 606 reloc->sym->sec->name, reloc->addend);
dcc914f4
JP		 607 return -1;
608 }
609 } else {
22682a07 610 WARN("can't find unreachable insn at %s+0x%" PRIx64,
f1974222 611 reloc->sym->sec->name, reloc->addend);
dcc914f4
JP		 612 return -1;
613 }
614
615 insn->dead_end = true;
616 }
617
649ea4d5
JP		 618reachable:
619 /*
620 * These manually annotated reachable checks are needed for GCC 4.4,
621 * where the Linux unreachable() macro isn't supported. In that case
622 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
623 * not a dead end.
624 */
625 sec = find_section_by_name(file->elf, ".rela.discard.reachable");
626 if (!sec)
627 return 0;
628
f1974222
MH		 629 list_for_each_entry(reloc, &sec->reloc_list, list) {
630 if (reloc->sym->type != STT_SECTION) {
649ea4d5
JP		 631 WARN("unexpected relocation symbol type in %s", sec->name);
632 return -1;
633 }
f1974222 634 insn = find_insn(file, reloc->sym->sec, reloc->addend);
649ea4d5 635 if (insn)
1c34496e 636 insn = prev_insn_same_sec(file, insn);
fe255fe6 637 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
f1974222 638 insn = find_last_insn(file, reloc->sym->sec);
6b5dd716 639 if (!insn) {
22682a07 640 WARN("can't find reachable insn at %s+0x%" PRIx64,
f1974222 641 reloc->sym->sec->name, reloc->addend);
649ea4d5
JP		 642 return -1;
643 }
644 } else {
22682a07 645 WARN("can't find reachable insn at %s+0x%" PRIx64,
f1974222 646 reloc->sym->sec->name, reloc->addend);
649ea4d5
JP		 647 return -1;
648 }
649
650 insn->dead_end = false;
651 }
652
dcc914f4
JP		 653 return 0;
654}
655
1e7e4788
JP		 656static int create_static_call_sections(struct objtool_file *file)
657{
ef47cc01 658 struct section *sec;
1e7e4788
JP		 659 struct static_call_site *site;
660 struct instruction *insn;
661 struct symbol *key_sym;
662 char *key_name, *tmp;
663 int idx;
664
665 sec = find_section_by_name(file->elf, ".static_call_sites");
666 if (sec) {
667 INIT_LIST_HEAD(&file->static_call_list);
668 WARN("file already has .static_call_sites section, skipping");
669 return 0;
670 }
671
672 if (list_empty(&file->static_call_list))
673 return 0;
674
675 idx = 0;
43d5430a 676 list_for_each_entry(insn, &file->static_call_list, call_node)
1e7e4788
JP		 677 idx++;
678
679 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
680 sizeof(struct static_call_site), idx);
681 if (!sec)
682 return -1;
683
1e7e4788 684 idx = 0;
43d5430a 685 list_for_each_entry(insn, &file->static_call_list, call_node) {
1e7e4788
JP		 686
687 site = (struct static_call_site *)sec->data->d_buf + idx;
688 memset(site, 0, sizeof(struct static_call_site));
689
690 /* populate reloc for 'addr' */
ef47cc01
PZ		 691 if (elf_add_reloc_to_insn(file->elf, sec,
692 idx * sizeof(struct static_call_site),
693 R_X86_64_PC32,
694 insn->sec, insn->offset))
44f6a7c0 695 return -1;
1e7e4788
JP		 696
697 /* find key symbol */
c6f5dc28 698 key_name = strdup(insn_call_dest(insn)->name);
1e7e4788
JP		 699 if (!key_name) {
700 perror("strdup");
701 return -1;
702 }
703 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
704 STATIC_CALL_TRAMP_PREFIX_LEN)) {
705 WARN("static_call: trampoline name malformed: %s", key_name);
3da73f10 706 free(key_name);
1e7e4788
JP		 707 return -1;
708 }
709 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
710 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
711
712 key_sym = find_symbol_by_name(file->elf, tmp);
713 if (!key_sym) {
2daf7fab 714 if (!opts.module) {
73f44fe1 715 WARN("static_call: can't find static_call_key symbol: %s", tmp);
3da73f10 716 free(key_name);
73f44fe1
JP		 717 return -1;
718 }
719
720 /*
721 * For modules(), the key might not be exported, which
722 * means the module can make static calls but isn't
723 * allowed to change them.
724 *
725 * In that case we temporarily set the key to be the
726 * trampoline address. This is fixed up in
727 * static_call_add_module().
728 */
c6f5dc28 729 key_sym = insn_call_dest(insn);
1e7e4788
JP		 730 }
731 free(key_name);
732
733 /* populate reloc for 'key' */
ef47cc01
PZ		 734 if (elf_add_reloc(file->elf, sec,
735 idx * sizeof(struct static_call_site) + 4,
736 R_X86_64_PC32, key_sym,
737 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
1e7e4788 738 return -1;
1e7e4788
JP		 739
740 idx++;
741 }
742
1e7e4788
JP		 743 return 0;
744}
745
134ab5bd
PZ		 746static int create_retpoline_sites_sections(struct objtool_file *file)
747{
748 struct instruction *insn;
749 struct section *sec;
750 int idx;
751
752 sec = find_section_by_name(file->elf, ".retpoline_sites");
753 if (sec) {
754 WARN("file already has .retpoline_sites, skipping");
755 return 0;
756 }
757
758 idx = 0;
759 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
760 idx++;
761
762 if (!idx)
763 return 0;
764
765 sec = elf_create_section(file->elf, ".retpoline_sites", 0,
766 sizeof(int), idx);
767 if (!sec) {
768 WARN("elf_create_section: .retpoline_sites");
769 return -1;
770 }
771
772 idx = 0;
773 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
774
775 int *site = (int *)sec->data->d_buf + idx;
776 *site = 0;
777
778 if (elf_add_reloc_to_insn(file->elf, sec,
779 idx * sizeof(int),
780 R_X86_64_PC32,
781 insn->sec, insn->offset)) {
782 WARN("elf_add_reloc_to_insn: .retpoline_sites");
783 return -1;
784 }
785
786 idx++;
787 }
788
789 return 0;
790}
791
d9e9d230
PZ		 792static int create_return_sites_sections(struct objtool_file *file)
793{
794 struct instruction *insn;
795 struct section *sec;
796 int idx;
797
798 sec = find_section_by_name(file->elf, ".return_sites");
799 if (sec) {
800 WARN("file already has .return_sites, skipping");
801 return 0;
802 }
803
804 idx = 0;
805 list_for_each_entry(insn, &file->return_thunk_list, call_node)
806 idx++;
807
808 if (!idx)
809 return 0;
810
811 sec = elf_create_section(file->elf, ".return_sites", 0,
812 sizeof(int), idx);
813 if (!sec) {
814 WARN("elf_create_section: .return_sites");
815 return -1;
816 }
817
818 idx = 0;
819 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
820
821 int *site = (int *)sec->data->d_buf + idx;
822 *site = 0;
823
824 if (elf_add_reloc_to_insn(file->elf, sec,
825 idx * sizeof(int),
826 R_X86_64_PC32,
827 insn->sec, insn->offset)) {
828 WARN("elf_add_reloc_to_insn: .return_sites");
829 return -1;
830 }
831
832 idx++;
833 }
834
835 return 0;
836}
837
89bc853e
PZ		 838static int create_ibt_endbr_seal_sections(struct objtool_file *file)
839{
840 struct instruction *insn;
841 struct section *sec;
842 int idx;
843
844 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
845 if (sec) {
846 WARN("file already has .ibt_endbr_seal, skipping");
847 return 0;
848 }
849
850 idx = 0;
851 list_for_each_entry(insn, &file->endbr_list, call_node)
852 idx++;
853
2daf7fab 854 if (opts.stats) {
89bc853e
PZ		 855 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
856 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
857 printf("ibt: superfluous ENDBR: %d\n", idx);
858 }
859
860 if (!idx)
861 return 0;
862
863 sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
864 sizeof(int), idx);
865 if (!sec) {
866 WARN("elf_create_section: .ibt_endbr_seal");
867 return -1;
868 }
869
870 idx = 0;
871 list_for_each_entry(insn, &file->endbr_list, call_node) {
872
873 int *site = (int *)sec->data->d_buf + idx;
03d7a105 874 struct symbol *sym = insn->sym;
89bc853e
PZ		 875 *site = 0;
876
03d7a105
MK		 877 if (opts.module && sym && sym->type == STT_FUNC &&
878 insn->offset == sym->offset &&
879 (!strcmp(sym->name, "init_module") ||
880 !strcmp(sym->name, "cleanup_module")))
881 WARN("%s(): not an indirect call target", sym->name);
882
89bc853e
PZ		 883 if (elf_add_reloc_to_insn(file->elf, sec,
884 idx * sizeof(int),
885 R_X86_64_PC32,
886 insn->sec, insn->offset)) {
887 WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
888 return -1;
889 }
890
891 idx++;
892 }
893
894 return 0;
895}
896
9a479f76
PZ		 897static int create_cfi_sections(struct objtool_file *file)
898{
9290e772 899 struct section *sec;
9a479f76
PZ		 900 struct symbol *sym;
901 unsigned int *loc;
902 int idx;
903
904 sec = find_section_by_name(file->elf, ".cfi_sites");
905 if (sec) {
906 INIT_LIST_HEAD(&file->call_list);
907 WARN("file already has .cfi_sites section, skipping");
908 return 0;
909 }
910
911 idx = 0;
9290e772
JP		 912 for_each_sym(file, sym) {
913 if (sym->type != STT_FUNC)
9a479f76
PZ		 914 continue;
915
9290e772
JP		 916 if (strncmp(sym->name, "__cfi_", 6))
917 continue;
9a479f76 918
9290e772 919 idx++;
9a479f76
PZ		 920 }
921
922 sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx);
923 if (!sec)
924 return -1;
925
926 idx = 0;
9290e772
JP		 927 for_each_sym(file, sym) {
928 if (sym->type != STT_FUNC)
9a479f76
PZ		 929 continue;
930
9290e772
JP		 931 if (strncmp(sym->name, "__cfi_", 6))
932 continue;
9a479f76 933
9290e772
JP		 934 loc = (unsigned int *)sec->data->d_buf + idx;
935 memset(loc, 0, sizeof(unsigned int));
9a479f76 936
9290e772
JP		 937 if (elf_add_reloc_to_insn(file->elf, sec,
938 idx * sizeof(unsigned int),
939 R_X86_64_PC32,
940 sym->sec, sym->offset))
941 return -1;
9a479f76 942
9290e772 943 idx++;
9a479f76
PZ		 944 }
945
946 return 0;
947}
948
99d00215
PZ		 949static int create_mcount_loc_sections(struct objtool_file *file)
950{
86ea7f36 951 int addrsize = elf_class_addrsize(file->elf);
99d00215 952 struct instruction *insn;
86ea7f36 953 struct section *sec;
99d00215
PZ		 954 int idx;
955
956 sec = find_section_by_name(file->elf, "__mcount_loc");
957 if (sec) {
958 INIT_LIST_HEAD(&file->mcount_loc_list);
959 WARN("file already has __mcount_loc section, skipping");
960 return 0;
961 }
962
963 if (list_empty(&file->mcount_loc_list))
964 return 0;
965
966 idx = 0;
c509331b 967 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
99d00215
PZ		 968 idx++;
969
86ea7f36 970 sec = elf_create_section(file->elf, "__mcount_loc", 0, addrsize, idx);
99d00215
PZ		 971 if (!sec)
972 return -1;
973
86ea7f36
CL		 974 sec->sh.sh_addralign = addrsize;
975
99d00215 976 idx = 0;
c509331b 977 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
86ea7f36 978 void *loc;
99d00215 979
86ea7f36
CL		 980 loc = sec->data->d_buf + idx;
981 memset(loc, 0, addrsize);
99d00215 982
86ea7f36 983 if (elf_add_reloc_to_insn(file->elf, sec, idx,
c1449735 984 addrsize == sizeof(u64) ? R_ABS64 : R_ABS32,
ef47cc01 985 insn->sec, insn->offset))
99d00215 986 return -1;
99d00215 987
86ea7f36 988 idx += addrsize;
99d00215
PZ		 989 }
990
99d00215
PZ		 991 return 0;
992}
993
00abd384
PZ		 994static int create_direct_call_sections(struct objtool_file *file)
995{
996 struct instruction *insn;
997 struct section *sec;
998 unsigned int *loc;
999 int idx;
1000
1001 sec = find_section_by_name(file->elf, ".call_sites");
1002 if (sec) {
1003 INIT_LIST_HEAD(&file->call_list);
1004 WARN("file already has .call_sites section, skipping");
1005 return 0;
1006 }
1007
1008 if (list_empty(&file->call_list))
1009 return 0;
1010
1011 idx = 0;
1012 list_for_each_entry(insn, &file->call_list, call_node)
1013 idx++;
1014
1015 sec = elf_create_section(file->elf, ".call_sites", 0, sizeof(unsigned int), idx);
1016 if (!sec)
1017 return -1;
1018
1019 idx = 0;
1020 list_for_each_entry(insn, &file->call_list, call_node) {
1021
1022 loc = (unsigned int *)sec->data->d_buf + idx;
1023 memset(loc, 0, sizeof(unsigned int));
1024
1025 if (elf_add_reloc_to_insn(file->elf, sec,
1026 idx * sizeof(unsigned int),
1027 R_X86_64_PC32,
1028 insn->sec, insn->offset))
1029 return -1;
1030
1031 idx++;
1032 }
1033
1034 return 0;
1035}
1036
dcc914f4
JP		 1037/*
1038 * Warnings shouldn't be reported for ignored functions.
1039 */
1040static void add_ignores(struct objtool_file *file)
1041{
1042 struct instruction *insn;
1043 struct section *sec;
1044 struct symbol *func;
f1974222 1045 struct reloc *reloc;
dcc914f4 1046
aaf5c623
PZ		 1047 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1048 if (!sec)
1049 return;
dcc914f4 1050
f1974222
MH		 1051 list_for_each_entry(reloc, &sec->reloc_list, list) {
1052 switch (reloc->sym->type) {
aaf5c623 1053 case STT_FUNC:
f1974222 1054 func = reloc->sym;
aaf5c623
PZ		 1055 break;
1056
1057 case STT_SECTION:
f1974222 1058 func = find_func_by_offset(reloc->sym->sec, reloc->addend);
7acfe531 1059 if (!func)
dcc914f4 1060 continue;
aaf5c623 1061 break;
dcc914f4 1062
aaf5c623 1063 default:
f1974222 1064 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
aaf5c623 1065 continue;
dcc914f4 1066 }
aaf5c623 1067
f0f70adb 1068 func_for_each_insn(file, func, insn)
aaf5c623 1069 insn->ignore = true;
dcc914f4
JP		 1070 }
1071}
1072
ea24213d
PZ		 1073/*
1074 * This is a whitelist of functions that is allowed to be called with AC set.
1075 * The list is meant to be minimal and only contains compiler instrumentation
1076 * ABI and a few functions used to implement *_{to,from}_user() functions.
1077 *
1078 * These functions must not directly change AC, but may PUSHF/POPF.
1079 */
1080static const char *uaccess_safe_builtin[] = {
1081 /* KASAN */
1082 "kasan_report",
f00748bf 1083 "kasan_check_range",
ea24213d
PZ		 1084 /* KASAN out-of-line */
1085 "__asan_loadN_noabort",
1086 "__asan_load1_noabort",
1087 "__asan_load2_noabort",
1088 "__asan_load4_noabort",
1089 "__asan_load8_noabort",
1090 "__asan_load16_noabort",
1091 "__asan_storeN_noabort",
1092 "__asan_store1_noabort",
1093 "__asan_store2_noabort",
1094 "__asan_store4_noabort",
1095 "__asan_store8_noabort",
1096 "__asan_store16_noabort",
b0b8e56b
JH		 1097 "__kasan_check_read",
1098 "__kasan_check_write",
ea24213d
PZ		 1099 /* KASAN in-line */
1100 "__asan_report_load_n_noabort",
1101 "__asan_report_load1_noabort",
1102 "__asan_report_load2_noabort",
1103 "__asan_report_load4_noabort",
1104 "__asan_report_load8_noabort",
1105 "__asan_report_load16_noabort",
1106 "__asan_report_store_n_noabort",
1107 "__asan_report_store1_noabort",
1108 "__asan_report_store2_noabort",
1109 "__asan_report_store4_noabort",
1110 "__asan_report_store8_noabort",
1111 "__asan_report_store16_noabort",
5f5c9712 1112 /* KCSAN */
9967683c 1113 "__kcsan_check_access",
0525bd82
ME		 1114 "__kcsan_mb",
1115 "__kcsan_wmb",
1116 "__kcsan_rmb",
1117 "__kcsan_release",
5f5c9712
ME		 1118 "kcsan_found_watchpoint",
1119 "kcsan_setup_watchpoint",
9967683c 1120 "kcsan_check_scoped_accesses",
50a19ad4
ME		 1121 "kcsan_disable_current",
1122 "kcsan_enable_current_nowarn",
5f5c9712
ME		 1123 /* KCSAN/TSAN */
1124 "__tsan_func_entry",
1125 "__tsan_func_exit",
1126 "__tsan_read_range",
1127 "__tsan_write_range",
1128 "__tsan_read1",
1129 "__tsan_read2",
1130 "__tsan_read4",
1131 "__tsan_read8",
1132 "__tsan_read16",
1133 "__tsan_write1",
1134 "__tsan_write2",
1135 "__tsan_write4",
1136 "__tsan_write8",
1137 "__tsan_write16",
a81b3759
ME		 1138 "__tsan_read_write1",
1139 "__tsan_read_write2",
1140 "__tsan_read_write4",
1141 "__tsan_read_write8",
1142 "__tsan_read_write16",
63646fcb
ME		 1143 "__tsan_volatile_read1",
1144 "__tsan_volatile_read2",
1145 "__tsan_volatile_read4",
1146 "__tsan_volatile_read8",
1147 "__tsan_volatile_read16",
1148 "__tsan_volatile_write1",
1149 "__tsan_volatile_write2",
1150 "__tsan_volatile_write4",
1151 "__tsan_volatile_write8",
1152 "__tsan_volatile_write16",
883957b1
ME		 1153 "__tsan_atomic8_load",
1154 "__tsan_atomic16_load",
1155 "__tsan_atomic32_load",
1156 "__tsan_atomic64_load",
1157 "__tsan_atomic8_store",
1158 "__tsan_atomic16_store",
1159 "__tsan_atomic32_store",
1160 "__tsan_atomic64_store",
1161 "__tsan_atomic8_exchange",
1162 "__tsan_atomic16_exchange",
1163 "__tsan_atomic32_exchange",
1164 "__tsan_atomic64_exchange",
1165 "__tsan_atomic8_fetch_add",
1166 "__tsan_atomic16_fetch_add",
1167 "__tsan_atomic32_fetch_add",
1168 "__tsan_atomic64_fetch_add",
1169 "__tsan_atomic8_fetch_sub",
1170 "__tsan_atomic16_fetch_sub",
1171 "__tsan_atomic32_fetch_sub",
1172 "__tsan_atomic64_fetch_sub",
1173 "__tsan_atomic8_fetch_and",
1174 "__tsan_atomic16_fetch_and",
1175 "__tsan_atomic32_fetch_and",
1176 "__tsan_atomic64_fetch_and",
1177 "__tsan_atomic8_fetch_or",
1178 "__tsan_atomic16_fetch_or",
1179 "__tsan_atomic32_fetch_or",
1180 "__tsan_atomic64_fetch_or",
1181 "__tsan_atomic8_fetch_xor",
1182 "__tsan_atomic16_fetch_xor",
1183 "__tsan_atomic32_fetch_xor",
1184 "__tsan_atomic64_fetch_xor",
1185 "__tsan_atomic8_fetch_nand",
1186 "__tsan_atomic16_fetch_nand",
1187 "__tsan_atomic32_fetch_nand",
1188 "__tsan_atomic64_fetch_nand",
1189 "__tsan_atomic8_compare_exchange_strong",
1190 "__tsan_atomic16_compare_exchange_strong",
1191 "__tsan_atomic32_compare_exchange_strong",
1192 "__tsan_atomic64_compare_exchange_strong",
1193 "__tsan_atomic8_compare_exchange_weak",
1194 "__tsan_atomic16_compare_exchange_weak",
1195 "__tsan_atomic32_compare_exchange_weak",
1196 "__tsan_atomic64_compare_exchange_weak",
1197 "__tsan_atomic8_compare_exchange_val",
1198 "__tsan_atomic16_compare_exchange_val",
1199 "__tsan_atomic32_compare_exchange_val",
1200 "__tsan_atomic64_compare_exchange_val",
1201 "__tsan_atomic_thread_fence",
1202 "__tsan_atomic_signal_fence",
d5d46924
AB		 1203 "__tsan_unaligned_read16",
1204 "__tsan_unaligned_write16",
ea24213d
PZ		 1205 /* KCOV */
1206 "write_comp_data",
ae033f08 1207 "check_kcov_mode",
ea24213d
PZ		 1208 "__sanitizer_cov_trace_pc",
1209 "__sanitizer_cov_trace_const_cmp1",
1210 "__sanitizer_cov_trace_const_cmp2",
1211 "__sanitizer_cov_trace_const_cmp4",
1212 "__sanitizer_cov_trace_const_cmp8",
1213 "__sanitizer_cov_trace_cmp1",
1214 "__sanitizer_cov_trace_cmp2",
1215 "__sanitizer_cov_trace_cmp4",
1216 "__sanitizer_cov_trace_cmp8",
36b1c700 1217 "__sanitizer_cov_trace_switch",
40b22c9d
AP		 1218 /* KMSAN */
1219 "kmsan_copy_to_user",
1220 "kmsan_report",
1221 "kmsan_unpoison_entry_regs",
1222 "kmsan_unpoison_memory",
1223 "__msan_chain_origin",
1224 "__msan_get_context_state",
1225 "__msan_instrument_asm_store",
1226 "__msan_metadata_ptr_for_load_1",
1227 "__msan_metadata_ptr_for_load_2",
1228 "__msan_metadata_ptr_for_load_4",
1229 "__msan_metadata_ptr_for_load_8",
1230 "__msan_metadata_ptr_for_load_n",
1231 "__msan_metadata_ptr_for_store_1",
1232 "__msan_metadata_ptr_for_store_2",
1233 "__msan_metadata_ptr_for_store_4",
1234 "__msan_metadata_ptr_for_store_8",
1235 "__msan_metadata_ptr_for_store_n",
1236 "__msan_poison_alloca",
1237 "__msan_warning",
ea24213d
PZ		 1238 /* UBSAN */
1239 "ubsan_type_mismatch_common",
1240 "__ubsan_handle_type_mismatch",
1241 "__ubsan_handle_type_mismatch_v1",
9a50dcaf 1242 "__ubsan_handle_shift_out_of_bounds",
f18b0d7e 1243 "__ubsan_handle_load_invalid_value",
7f530fba
JP		 1244 /* STACKLEAK */
1245 "stackleak_track_stack",
ea24213d
PZ		 1246 /* misc */
1247 "csum_partial_copy_generic",
ec6347bb
DW		 1248 "copy_mc_fragile",
1249 "copy_mc_fragile_handle_tail",
5da8e4a6 1250 "copy_mc_enhanced_fast_string",
ea24213d 1251 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
8c9b6a88 1252 "rep_stos_alternative",
427fda2c 1253 "rep_movs_alternative",
3639a535 1254 "__copy_user_nocache",
ea24213d
PZ		 1255 NULL
1256};
1257
1258static void add_uaccess_safe(struct objtool_file *file)
1259{
1260 struct symbol *func;
1261 const char **name;
1262
2daf7fab 1263 if (!opts.uaccess)
ea24213d
PZ		 1264 return;
1265
1266 for (name = uaccess_safe_builtin; *name; name++) {
1267 func = find_symbol_by_name(file->elf, *name);
1268 if (!func)
1269 continue;
1270
e10cd8fe 1271 func->uaccess_safe = true;
dcc914f4
JP		 1272 }
1273}
1274
258c7605
JP		 1275/*
1276 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1277 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1278 * But it at least allows objtool to understand the control flow *around* the
1279 * retpoline.
1280 */
ff05ab23 1281static int add_ignore_alternatives(struct objtool_file *file)
258c7605
JP		 1282{
1283 struct section *sec;
f1974222 1284 struct reloc *reloc;
258c7605
JP		 1285 struct instruction *insn;
1286
ff05ab23 1287 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
258c7605
JP		 1288 if (!sec)
1289 return 0;
1290
f1974222
MH		 1291 list_for_each_entry(reloc, &sec->reloc_list, list) {
1292 if (reloc->sym->type != STT_SECTION) {
258c7605
JP		 1293 WARN("unexpected relocation symbol type in %s", sec->name);
1294 return -1;
1295 }
1296
f1974222 1297 insn = find_insn(file, reloc->sym->sec, reloc->addend);
258c7605 1298 if (!insn) {
ff05ab23 1299 WARN("bad .discard.ignore_alts entry");
258c7605
JP		 1300 return -1;
1301 }
1302
1303 insn->ignore_alts = true;
1304 }
1305
1306 return 0;
1307}
1308
530b4ddd
PZ		 1309__weak bool arch_is_retpoline(struct symbol *sym)
1310{
1311 return false;
1312}
1313
d9e9d230
PZ		 1314__weak bool arch_is_rethunk(struct symbol *sym)
1315{
1316 return false;
1317}
1318
7bd2a600
PZ		 1319static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1320{
0932dbe1
PZ		 1321 struct reloc *reloc;
1322
1323 if (insn->no_reloc)
7bd2a600
PZ		 1324 return NULL;
1325
0932dbe1
PZ		 1326 if (!file)
1327 return NULL;
db2b0c5d 1328
0932dbe1
PZ		 1329 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1330 insn->offset, insn->len);
1331 if (!reloc) {
1332 insn->no_reloc = 1;
1333 return NULL;
7bd2a600
PZ		 1334 }
1335
0932dbe1 1336 return reloc;
7bd2a600
PZ		 1337}
1338
f56dae88
PZ		 1339static void remove_insn_ops(struct instruction *insn)
1340{
3ee88df1 1341 struct stack_op *op, *next;
f56dae88 1342
3ee88df1
PZ		 1343 for (op = insn->stack_ops; op; op = next) {
1344 next = op->next;
f56dae88
PZ		 1345 free(op);
1346 }
3ee88df1 1347 insn->stack_ops = NULL;
f56dae88
PZ		 1348}
1349
dd003ede
PZ		 1350static void annotate_call_site(struct objtool_file *file,
1351 struct instruction *insn, bool sibling)
f56dae88
PZ		 1352{
1353 struct reloc *reloc = insn_reloc(file, insn);
c6f5dc28 1354 struct symbol *sym = insn_call_dest(insn);
f56dae88 1355
dd003ede
PZ		 1356 if (!sym)
1357 sym = reloc->sym;
1358
1359 /*
1360 * Alternative replacement code is just template code which is
1361 * sometimes copied to the original instruction. For now, don't
1362 * annotate it. (In the future we might consider annotating the
1363 * original instruction if/when it ever makes sense to do so.)
1364 */
1365 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
f56dae88
PZ		 1366 return;
1367
dd003ede
PZ		 1368 if (sym->static_call_tramp) {
1369 list_add_tail(&insn->call_node, &file->static_call_list);
1370 return;
f56dae88
PZ		 1371 }
1372
134ab5bd
PZ		 1373 if (sym->retpoline_thunk) {
1374 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1375 return;
1376 }
1377
f56dae88 1378 /*
05098119
ME		 1379 * Many compilers cannot disable KCOV or sanitizer calls with a function
1380 * attribute so they need a little help, NOP out any such calls from
1381 * noinstr text.
f56dae88 1382 */
22102f45 1383 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
f56dae88
PZ		 1384 if (reloc) {
1385 reloc->type = R_NONE;
1386 elf_write_reloc(file->elf, reloc);
1387 }
1388
1389 elf_write_insn(file->elf, insn->sec,
1390 insn->offset, insn->len,
1391 sibling ? arch_ret_insn(insn->len)
1392 : arch_nop_insn(insn->len));
1393
1394 insn->type = sibling ? INSN_RETURN : INSN_NOP;
7a53f408
PZ		 1395
1396 if (sibling) {
1397 /*
1398 * We've replaced the tail-call JMP insn by two new
1399 * insn: RET; INT3, except we only have a single struct
1400 * insn here. Mark it retpoline_safe to avoid the SLS
1401 * warning, instead of adding another insn.
1402 */
1403 insn->retpoline_safe = true;
1404 }
1405
dd003ede 1406 return;
f56dae88
PZ		 1407 }
1408
2daf7fab 1409 if (opts.mcount && sym->fentry) {
f56dae88 1410 if (sibling)
246b2c85 1411 WARN_INSN(insn, "tail call to __fentry__ !?!?");
280981d6
SV		 1412 if (opts.mnop) {
1413 if (reloc) {
1414 reloc->type = R_NONE;
1415 elf_write_reloc(file->elf, reloc);
1416 }
f56dae88 1417
280981d6
SV		 1418 elf_write_insn(file->elf, insn->sec,
1419 insn->offset, insn->len,
1420 arch_nop_insn(insn->len));
f56dae88 1421
280981d6
SV		 1422 insn->type = INSN_NOP;
1423 }
f56dae88 1424
c509331b 1425 list_add_tail(&insn->call_node, &file->mcount_loc_list);
dd003ede 1426 return;
f56dae88 1427 }
0e5b613b 1428
00abd384
PZ		 1429 if (insn->type == INSN_CALL && !insn->sec->init)
1430 list_add_tail(&insn->call_node, &file->call_list);
1431
0e5b613b
PZ		 1432 if (!sibling && dead_end_function(file, sym))
1433 insn->dead_end = true;
dd003ede
PZ		 1434}
1435
1436static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1437 struct symbol *dest, bool sibling)
1438{
c6f5dc28 1439 insn->_call_dest = dest;
dd003ede
PZ		 1440 if (!dest)
1441 return;
f56dae88
PZ		 1442
1443 /*
1444 * Whatever stack impact regular CALLs have, should be undone
1445 * by the RETURN of the called function.
1446 *
1447 * Annotated intra-function calls retain the stack_ops but
1448 * are converted to JUMP, see read_intra_function_calls().
1449 */
1450 remove_insn_ops(insn);
dd003ede
PZ		 1451
1452 annotate_call_site(file, insn, sibling);
f56dae88
PZ		 1453}
1454
134ab5bd
PZ		 1455static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1456{
1457 /*
1458 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1459 * so convert them accordingly.
1460 */
1461 switch (insn->type) {
1462 case INSN_CALL:
1463 insn->type = INSN_CALL_DYNAMIC;
1464 break;
1465 case INSN_JUMP_UNCONDITIONAL:
1466 insn->type = INSN_JUMP_DYNAMIC;
1467 break;
1468 case INSN_JUMP_CONDITIONAL:
1469 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1470 break;
1471 default:
1472 return;
1473 }
1474
1475 insn->retpoline_safe = true;
1476
1477 /*
1478 * Whatever stack impact regular CALLs have, should be undone
1479 * by the RETURN of the called function.
1480 *
1481 * Annotated intra-function calls retain the stack_ops but
1482 * are converted to JUMP, see read_intra_function_calls().
1483 */
1484 remove_insn_ops(insn);
1485
1486 annotate_call_site(file, insn, false);
1487}
08f87a93 1488
a149180f 1489static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
d9e9d230
PZ		 1490{
1491 /*
1492 * Return thunk tail calls are really just returns in disguise,
1493 * so convert them accordingly.
1494 */
1495 insn->type = INSN_RETURN;
1496 insn->retpoline_safe = true;
1497
a149180f
PZ		 1498 if (add)
1499 list_add_tail(&insn->call_node, &file->return_thunk_list);
d9e9d230
PZ		 1500}
1501
5a9c361a
PZ		 1502static bool is_first_func_insn(struct objtool_file *file,
1503 struct instruction *insn, struct symbol *sym)
08f87a93 1504{
5a9c361a 1505 if (insn->offset == sym->offset)
d139bca4
PZ		 1506 return true;
1507
5a9c361a 1508 /* Allow direct CALL/JMP past ENDBR */
2daf7fab 1509 if (opts.ibt) {
d139bca4
PZ		 1510 struct instruction *prev = prev_insn_same_sym(file, insn);
1511
1512 if (prev && prev->type == INSN_ENDBR &&
5a9c361a 1513 insn->offset == sym->offset + prev->len)
d139bca4
PZ		 1514 return true;
1515 }
1516
1517 return false;
08f87a93
PZ		 1518}
1519
5a9c361a
PZ		 1520/*
1521 * A sibling call is a tail-call to another symbol -- to differentiate from a
1522 * recursive tail-call which is to the same symbol.
1523 */
1524static bool jump_is_sibling_call(struct objtool_file *file,
1525 struct instruction *from, struct instruction *to)
1526{
1527 struct symbol *fs = from->sym;
1528 struct symbol *ts = to->sym;
1529
1530 /* Not a sibling call if from/to a symbol hole */
1531 if (!fs || !ts)
1532 return false;
1533
1534 /* Not a sibling call if not targeting the start of a symbol. */
1535 if (!is_first_func_insn(file, to, ts))
1536 return false;
1537
1538 /* Disallow sibling calls into STT_NOTYPE */
1539 if (ts->type == STT_NOTYPE)
1540 return false;
1541
1542 /* Must not be self to be a sibling */
1543 return fs->pfunc != ts->pfunc;
1544}
1545
dcc914f4
JP		 1546/*
1547 * Find the destination instructions for all jumps.
1548 */
1549static int add_jump_destinations(struct objtool_file *file)
1550{
26ff6041 1551 struct instruction *insn, *jump_dest;
f1974222 1552 struct reloc *reloc;
dcc914f4
JP		 1553 struct section *dest_sec;
1554 unsigned long dest_off;
1555
1556 for_each_insn(file, insn) {
34c861e8
JP		 1557 if (insn->jump_dest) {
1558 /*
1559 * handle_group_alt() may have previously set
1560 * 'jump_dest' for some alternatives.
1561 */
1562 continue;
1563 }
a2296140 1564 if (!is_static_jump(insn))
dcc914f4
JP		 1565 continue;
1566
7bd2a600 1567 reloc = insn_reloc(file, insn);
f1974222 1568 if (!reloc) {
dcc914f4 1569 dest_sec = insn->sec;
bfb08f22 1570 dest_off = arch_jump_destination(insn);
f1974222
MH		 1571 } else if (reloc->sym->type == STT_SECTION) {
1572 dest_sec = reloc->sym->sec;
1573 dest_off = arch_dest_reloc_offset(reloc->addend);
1739c66e 1574 } else if (reloc->sym->retpoline_thunk) {
134ab5bd 1575 add_retpoline_call(file, insn);
39b73533 1576 continue;
d9e9d230 1577 } else if (reloc->sym->return_thunk) {
a149180f 1578 add_return_call(file, insn, true);
d9e9d230 1579 continue;
dbcdbdfd 1580 } else if (insn_func(insn)) {
26ff6041
JP		 1581 /*
1582 * External sibling call or internal sibling call with
1583 * STT_FUNC reloc.
1584 */
f56dae88 1585 add_call_dest(file, insn, reloc->sym, true);
dcc914f4 1586 continue;
ecf11ba4
JP		 1587 } else if (reloc->sym->sec->idx) {
1588 dest_sec = reloc->sym->sec;
1589 dest_off = reloc->sym->sym.st_value +
1590 arch_dest_reloc_offset(reloc->addend);
1591 } else {
1592 /* non-func asm code jumping to another file */
1593 continue;
dcc914f4
JP		 1594 }
1595
26ff6041
JP		 1596 jump_dest = find_insn(file, dest_sec, dest_off);
1597 if (!jump_dest) {
a149180f
PZ		 1598 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1599
1600 /*
1601 * This is a special case for zen_untrain_ret().
1602 * It jumps to __x86_return_thunk(), but objtool
1603 * can't find the thunk's starting RET
1604 * instruction, because the RET is also in the
1605 * middle of another instruction. Objtool only
1606 * knows about the outer instruction.
1607 */
1608 if (sym && sym->return_thunk) {
1609 add_return_call(file, insn, false);
1610 continue;
1611 }
1612
246b2c85
JP		 1613 WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1614 dest_sec->name, dest_off);
dcc914f4
JP		 1615 return -1;
1616 }
cd77849a
JP		 1617
1618 /*
54262aa2 1619 * Cross-function jump.
cd77849a 1620 */
dbcdbdfd
PZ		 1621 if (insn_func(insn) && insn_func(jump_dest) &&
1622 insn_func(insn) != insn_func(jump_dest)) {
54262aa2
PZ		 1623
1624 /*
1625 * For GCC 8+, create parent/child links for any cold
1626 * subfunctions. This is _mostly_ redundant with a
1627 * similar initialization in read_symbols().
1628 *
1629 * If a function has aliases, we want the *first* such
1630 * function in the symbol table to be the subfunction's
1631 * parent. In that case we overwrite the
1632 * initialization done in read_symbols().
1633 *
1634 * However this code can't completely replace the
1635 * read_symbols() code because this doesn't detect the
1636 * case where the parent function's only reference to a
e7c2bc37 1637 * subfunction is through a jump table.
54262aa2 1638 */
dbcdbdfd
PZ		 1639 if (!strstr(insn_func(insn)->name, ".cold") &&
1640 strstr(insn_func(jump_dest)->name, ".cold")) {
1641 insn_func(insn)->cfunc = insn_func(jump_dest);
1642 insn_func(jump_dest)->pfunc = insn_func(insn);
54262aa2 1643 }
cd77849a 1644 }
26ff6041 1645
5a9c361a
PZ		 1646 if (jump_is_sibling_call(file, insn, jump_dest)) {
1647 /*
1648 * Internal sibling call without reloc or with
1649 * STT_SECTION reloc.
1650 */
1651 add_call_dest(file, insn, insn_func(jump_dest), true);
1652 continue;
1653 }
1654
26ff6041 1655 insn->jump_dest = jump_dest;
dcc914f4
JP		 1656 }
1657
1658 return 0;
1659}
1660
2b232a22
JT		 1661static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1662{
1663 struct symbol *call_dest;
1664
1665 call_dest = find_func_by_offset(sec, offset);
1666 if (!call_dest)
1667 call_dest = find_symbol_by_offset(sec, offset);
1668
1669 return call_dest;
1670}
1671
dcc914f4
JP		 1672/*
1673 * Find the destination instructions for all calls.
1674 */
1675static int add_call_destinations(struct objtool_file *file)
1676{
1677 struct instruction *insn;
1678 unsigned long dest_off;
f56dae88 1679 struct symbol *dest;
f1974222 1680 struct reloc *reloc;
dcc914f4
JP		 1681
1682 for_each_insn(file, insn) {
1683 if (insn->type != INSN_CALL)
1684 continue;
1685
7bd2a600 1686 reloc = insn_reloc(file, insn);
f1974222 1687 if (!reloc) {
bfb08f22 1688 dest_off = arch_jump_destination(insn);
f56dae88
PZ		 1689 dest = find_call_destination(insn->sec, dest_off);
1690
1691 add_call_dest(file, insn, dest, false);
a845c7cf 1692
7acfe531
JP		 1693 if (insn->ignore)
1694 continue;
1695
c6f5dc28 1696 if (!insn_call_dest(insn)) {
246b2c85 1697 WARN_INSN(insn, "unannotated intra-function call");
dcc914f4
JP		 1698 return -1;
1699 }
a845c7cf 1700
c6f5dc28 1701 if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
246b2c85 1702 WARN_INSN(insn, "unsupported call to non-function");
7acfe531
JP		 1703 return -1;
1704 }
1705
f1974222
MH		 1706 } else if (reloc->sym->type == STT_SECTION) {
1707 dest_off = arch_dest_reloc_offset(reloc->addend);
f56dae88
PZ		 1708 dest = find_call_destination(reloc->sym->sec, dest_off);
1709 if (!dest) {
246b2c85
JP		 1710 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1711 reloc->sym->sec->name, dest_off);
dcc914f4
JP		 1712 return -1;
1713 }
bcb1b6ff 1714
f56dae88
PZ		 1715 add_call_dest(file, insn, dest, false);
1716
1739c66e 1717 } else if (reloc->sym->retpoline_thunk) {
134ab5bd 1718 add_retpoline_call(file, insn);
bcb1b6ff 1719
dcc914f4 1720 } else
f56dae88 1721 add_call_dest(file, insn, reloc->sym, false);
dcc914f4
JP		 1722 }
1723
1724 return 0;
1725}
1726
1727/*
c9c324dc
JP		 1728 * The .alternatives section requires some extra special care over and above
1729 * other special sections because alternatives are patched in place.
dcc914f4
JP		 1730 */
1731static int handle_group_alt(struct objtool_file *file,
1732 struct special_alt *special_alt,
1733 struct instruction *orig_insn,
1734 struct instruction **new_insn)
1735{
a706bb08 1736 struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
b23cc71c 1737 struct alt_group *orig_alt_group, *new_alt_group;
dcc914f4
JP		 1738 unsigned long dest_off;
1739
a706bb08 1740 orig_alt_group = orig_insn->alt_group;
b23cc71c 1741 if (!orig_alt_group) {
a706bb08 1742 struct instruction *last_orig_insn = NULL;
c9c324dc 1743
a706bb08
PZ		 1744 orig_alt_group = malloc(sizeof(*orig_alt_group));
1745 if (!orig_alt_group) {
1746 WARN("malloc failed");
1747 return -1;
1748 }
1749 orig_alt_group->cfi = calloc(special_alt->orig_len,
1750 sizeof(struct cfi_state *));
1751 if (!orig_alt_group->cfi) {
1752 WARN("calloc failed");
1753 return -1;
1754 }
dcc914f4 1755
a706bb08
PZ		 1756 insn = orig_insn;
1757 sec_for_each_insn_from(file, insn) {
1758 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1759 break;
dcc914f4 1760
a706bb08
PZ		 1761 insn->alt_group = orig_alt_group;
1762 last_orig_insn = insn;
1763 }
1764 orig_alt_group->orig_group = NULL;
1765 orig_alt_group->first_insn = orig_insn;
1766 orig_alt_group->last_insn = last_orig_insn;
1c34496e 1767 orig_alt_group->nop = NULL;
a706bb08
PZ		 1768 } else {
1769 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1770 orig_alt_group->first_insn->offset != special_alt->orig_len) {
246b2c85 1771 WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
a706bb08
PZ		 1772 orig_alt_group->last_insn->offset +
1773 orig_alt_group->last_insn->len -
1774 orig_alt_group->first_insn->offset,
1775 special_alt->orig_len);
1776 return -1;
1777 }
1778 }
17bc3391 1779
c9c324dc
JP		 1780 new_alt_group = malloc(sizeof(*new_alt_group));
1781 if (!new_alt_group) {
1782 WARN("malloc failed");
1783 return -1;
dcc914f4 1784 }
dcc914f4 1785
c9c324dc
JP		 1786 if (special_alt->new_len < special_alt->orig_len) {
1787 /*
1788 * Insert a fake nop at the end to make the replacement
1789 * alt_group the same size as the original. This is needed to
1790 * allow propagate_alt_cfi() to do its magic. When the last
1791 * instruction affects the stack, the instruction after it (the
1792 * nop) will propagate the new state to the shared CFI array.
1793 */
1794 nop = malloc(sizeof(*nop));
1795 if (!nop) {
1796 WARN("malloc failed");
17bc3391
JP		 1797 return -1;
1798 }
c9c324dc 1799 memset(nop, 0, sizeof(*nop));
c9c324dc
JP		 1800
1801 nop->sec = special_alt->new_sec;
1802 nop->offset = special_alt->new_off + special_alt->new_len;
1803 nop->len = special_alt->orig_len - special_alt->new_len;
1804 nop->type = INSN_NOP;
dbcdbdfd 1805 nop->sym = orig_insn->sym;
c9c324dc
JP		 1806 nop->alt_group = new_alt_group;
1807 nop->ignore = orig_insn->ignore_alts;
dcc914f4 1808 }
17bc3391 1809
c9c324dc
JP		 1810 if (!special_alt->new_len) {
1811 *new_insn = nop;
1812 goto end;
dcc914f4
JP		 1813 }
1814
dcc914f4
JP		 1815 insn = *new_insn;
1816 sec_for_each_insn_from(file, insn) {
45245f51
JT		 1817 struct reloc *alt_reloc;
1818
dcc914f4
JP		 1819 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1820 break;
1821
1822 last_new_insn = insn;
1823
a845c7cf 1824 insn->ignore = orig_insn->ignore_alts;
dbcdbdfd 1825 insn->sym = orig_insn->sym;
b23cc71c 1826 insn->alt_group = new_alt_group;
a845c7cf 1827
dc419723
JP		 1828 /*
1829 * Since alternative replacement code is copy/pasted by the
1830 * kernel after applying relocations, generally such code can't
1831 * have relative-address relocation references to outside the
1832 * .altinstr_replacement section, unless the arch's
1833 * alternatives code can adjust the relative offsets
1834 * accordingly.
dc419723 1835 */
7bd2a600 1836 alt_reloc = insn_reloc(file, insn);
61c6065e 1837 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
45245f51 1838 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
dc419723 1839
246b2c85 1840 WARN_INSN(insn, "unsupported relocation in alternatives section");
dc419723
JP		 1841 return -1;
1842 }
1843
a2296140 1844 if (!is_static_jump(insn))
dcc914f4
JP		 1845 continue;
1846
1847 if (!insn->immediate)
1848 continue;
1849
bfb08f22 1850 dest_off = arch_jump_destination(insn);
34c861e8 1851 if (dest_off == special_alt->new_off + special_alt->new_len) {
a706bb08 1852 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
34c861e8 1853 if (!insn->jump_dest) {
246b2c85 1854 WARN_INSN(insn, "can't find alternative jump destination");
34c861e8
JP		 1855 return -1;
1856 }
dcc914f4
JP		 1857 }
1858 }
1859
1860 if (!last_new_insn) {
1861 WARN_FUNC("can't find last new alternative instruction",
1862 special_alt->new_sec, special_alt->new_off);
1863 return -1;
1864 }
1865
c9c324dc 1866end:
b23cc71c
JP		 1867 new_alt_group->orig_group = orig_alt_group;
1868 new_alt_group->first_insn = *new_insn;
1c34496e
PZ		 1869 new_alt_group->last_insn = last_new_insn;
1870 new_alt_group->nop = nop;
c9c324dc 1871 new_alt_group->cfi = orig_alt_group->cfi;
dcc914f4
JP		 1872 return 0;
1873}
1874
1875/*
1876 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1877 * If the original instruction is a jump, make the alt entry an effective nop
1878 * by just skipping the original instruction.
1879 */
1880static int handle_jump_alt(struct objtool_file *file,
1881 struct special_alt *special_alt,
1882 struct instruction *orig_insn,
1883 struct instruction **new_insn)
1884{
48001d26
PZ		 1885 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1886 orig_insn->type != INSN_NOP) {
e2d9494b 1887
246b2c85 1888 WARN_INSN(orig_insn, "unsupported instruction at jump label");
dcc914f4
JP		 1889 return -1;
1890 }
1891
4ab7674f 1892 if (opts.hack_jump_label && special_alt->key_addend & 2) {
6d37b83c
PZ		 1893 struct reloc *reloc = insn_reloc(file, orig_insn);
1894
1895 if (reloc) {
1896 reloc->type = R_NONE;
1897 elf_write_reloc(file->elf, reloc);
1898 }
1899 elf_write_insn(file->elf, orig_insn->sec,
1900 orig_insn->offset, orig_insn->len,
1901 arch_nop_insn(orig_insn->len));
1902 orig_insn->type = INSN_NOP;
48001d26
PZ		 1903 }
1904
1905 if (orig_insn->type == INSN_NOP) {
1906 if (orig_insn->len == 2)
1907 file->jl_nop_short++;
1908 else
1909 file->jl_nop_long++;
1910
1911 return 0;
6d37b83c
PZ		 1912 }
1913
e2d9494b
PZ		 1914 if (orig_insn->len == 2)
1915 file->jl_short++;
1916 else
1917 file->jl_long++;
1918
1c34496e 1919 *new_insn = next_insn_same_sec(file, orig_insn);
dcc914f4
JP		 1920 return 0;
1921}
1922
1923/*
1924 * Read all the special sections which have alternate instructions which can be
1925 * patched in or redirected to at runtime. Each instruction having alternate
1926 * instruction(s) has them added to its insn->alts list, which will be
1927 * traversed in validate_branch().
1928 */
1929static int add_special_section_alts(struct objtool_file *file)
1930{
1931 struct list_head special_alts;
1932 struct instruction *orig_insn, *new_insn;
1933 struct special_alt *special_alt, *tmp;
1934 struct alternative *alt;
1935 int ret;
1936
1937 ret = special_get_alts(file->elf, &special_alts);
1938 if (ret)
1939 return ret;
1940
1941 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
dcc914f4
JP		 1942
1943 orig_insn = find_insn(file, special_alt->orig_sec,
1944 special_alt->orig_off);
1945 if (!orig_insn) {
1946 WARN_FUNC("special: can't find orig instruction",
1947 special_alt->orig_sec, special_alt->orig_off);
1948 ret = -1;
1949 goto out;
1950 }
1951
1952 new_insn = NULL;
1953 if (!special_alt->group || special_alt->new_len) {
1954 new_insn = find_insn(file, special_alt->new_sec,
1955 special_alt->new_off);
1956 if (!new_insn) {
1957 WARN_FUNC("special: can't find new instruction",
1958 special_alt->new_sec,
1959 special_alt->new_off);
1960 ret = -1;
1961 goto out;
1962 }
1963 }
1964
1965 if (special_alt->group) {
7170cf47 1966 if (!special_alt->orig_len) {
246b2c85 1967 WARN_INSN(orig_insn, "empty alternative entry");
7170cf47
JT		 1968 continue;
1969 }
1970
dcc914f4
JP		 1971 ret = handle_group_alt(file, special_alt, orig_insn,
1972 &new_insn);
1973 if (ret)
1974 goto out;
1975 } else if (special_alt->jump_or_nop) {
1976 ret = handle_jump_alt(file, special_alt, orig_insn,
1977 &new_insn);
1978 if (ret)
1979 goto out;
1980 }
1981
258c7605
JP		 1982 alt = malloc(sizeof(*alt));
1983 if (!alt) {
1984 WARN("malloc failed");
1985 ret = -1;
1986 goto out;
1987 }
1988
dcc914f4 1989 alt->insn = new_insn;
764eef4b 1990 alt->skip_orig = special_alt->skip_orig;
ea24213d 1991 orig_insn->ignore_alts |= special_alt->skip_alt;
d5406654
PZ		 1992 alt->next = orig_insn->alts;
1993 orig_insn->alts = alt;
dcc914f4
JP		 1994
1995 list_del(&special_alt->list);
1996 free(special_alt);
1997 }
1998
2daf7fab 1999 if (opts.stats) {
e2d9494b
PZ		 2000 printf("jl\\\tNOP\tJMP\n");
2001 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2002 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2003 }
2004
dcc914f4
JP		 2005out:
2006 return ret;
2007}
2008
e7c2bc37 2009static int add_jump_table(struct objtool_file *file, struct instruction *insn,
f1974222 2010 struct reloc *table)
dcc914f4 2011{
f1974222 2012 struct reloc *reloc = table;
e7c2bc37 2013 struct instruction *dest_insn;
dcc914f4 2014 struct alternative *alt;
dbcdbdfd 2015 struct symbol *pfunc = insn_func(insn)->pfunc;
fd35c88b 2016 unsigned int prev_offset = 0;
dcc914f4 2017
e7c2bc37 2018 /*
f1974222 2019 * Each @reloc is a switch table relocation which points to the target
e7c2bc37
JP		 2020 * instruction.
2021 */
f1974222 2022 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
bd98c813
JH		 2023
2024 /* Check for the end of the table: */
f1974222 2025 if (reloc != table && reloc->jump_table_start)
dcc914f4
JP		 2026 break;
2027
e7c2bc37 2028 /* Make sure the table entries are consecutive: */
f1974222 2029 if (prev_offset && reloc->offset != prev_offset + 8)
fd35c88b
JP		 2030 break;
2031
2032 /* Detect function pointers from contiguous objects: */
f1974222
MH		 2033 if (reloc->sym->sec == pfunc->sec &&
2034 reloc->addend == pfunc->offset)
fd35c88b
JP		 2035 break;
2036
f1974222 2037 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
e7c2bc37 2038 if (!dest_insn)
dcc914f4
JP		 2039 break;
2040
e7c2bc37 2041 /* Make sure the destination is in the same function: */
dbcdbdfd 2042 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
13810435 2043 break;
dcc914f4
JP		 2044
2045 alt = malloc(sizeof(*alt));
2046 if (!alt) {
2047 WARN("malloc failed");
2048 return -1;
2049 }
2050
e7c2bc37 2051 alt->insn = dest_insn;
d5406654
PZ		 2052 alt->next = insn->alts;
2053 insn->alts = alt;
f1974222 2054 prev_offset = reloc->offset;
fd35c88b
JP		 2055 }
2056
2057 if (!prev_offset) {
246b2c85 2058 WARN_INSN(insn, "can't find switch jump table");
fd35c88b 2059 return -1;
dcc914f4
JP		 2060 }
2061
2062 return 0;
2063}
2064
2065/*
d871f7b5
RG		 2066 * find_jump_table() - Given a dynamic jump, find the switch jump table
2067 * associated with it.
dcc914f4 2068 */
f1974222 2069static struct reloc *find_jump_table(struct objtool_file *file,
dcc914f4
JP		 2070 struct symbol *func,
2071 struct instruction *insn)
2072{
d871f7b5 2073 struct reloc *table_reloc;
113d4bc9 2074 struct instruction *dest_insn, *orig_insn = insn;
dcc914f4 2075
99ce7962
PZ		 2076 /*
2077 * Backward search using the @first_jump_src links, these help avoid
2078 * much of the 'in between' code. Which avoids us getting confused by
2079 * it.
2080 */
7dec80cc 2081 for (;
dbcdbdfd 2082 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
1119d265 2083 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
99ce7962 2084
7dec80cc 2085 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
dcc914f4
JP		 2086 break;
2087
2088 /* allow small jumps within the range */
2089 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2090 insn->jump_dest &&
2091 (insn->jump_dest->offset <= insn->offset ||
2092 insn->jump_dest->offset > orig_insn->offset))
2093 break;
2094
d871f7b5 2095 table_reloc = arch_find_switch_table(file, insn);
f1974222 2096 if (!table_reloc)
e7c2bc37 2097 continue;
f1974222 2098 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
dbcdbdfd 2099 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
113d4bc9 2100 continue;
7dec80cc 2101
f1974222 2102 return table_reloc;
dcc914f4
JP		 2103 }
2104
2105 return NULL;
2106}
2107
bd98c813
JH		 2108/*
2109 * First pass: Mark the head of each jump table so that in the next pass,
2110 * we know when a given jump table ends and the next one starts.
2111 */
2112static void mark_func_jump_tables(struct objtool_file *file,
2113 struct symbol *func)
dcc914f4 2114{
bd98c813 2115 struct instruction *insn, *last = NULL;
f1974222 2116 struct reloc *reloc;
dcc914f4 2117
f0f70adb 2118 func_for_each_insn(file, func, insn) {
99ce7962
PZ		 2119 if (!last)
2120 last = insn;
2121
2122 /*
2123 * Store back-pointers for unconditional forward jumps such
e7c2bc37 2124 * that find_jump_table() can back-track using those and
99ce7962
PZ		 2125 * avoid some potentially confusing code.
2126 */
2127 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2128 insn->offset > last->offset &&
2129 insn->jump_dest->offset > insn->offset &&
2130 !insn->jump_dest->first_jump_src) {
2131
2132 insn->jump_dest->first_jump_src = insn;
2133 last = insn->jump_dest;
2134 }
2135
dcc914f4
JP		 2136 if (insn->type != INSN_JUMP_DYNAMIC)
2137 continue;
2138
f1974222
MH		 2139 reloc = find_jump_table(file, func, insn);
2140 if (reloc) {
2141 reloc->jump_table_start = true;
c6f5dc28 2142 insn->_jump_table = reloc;
dcc914f4 2143 }
dcc914f4 2144 }
bd98c813
JH		 2145}
2146
2147static int add_func_jump_tables(struct objtool_file *file,
2148 struct symbol *func)
2149{
2150 struct instruction *insn;
2151 int ret;
2152
f0f70adb 2153 func_for_each_insn(file, func, insn) {
c6f5dc28 2154 if (!insn_jump_table(insn))
bd98c813 2155 continue;
dcc914f4 2156
c6f5dc28 2157 ret = add_jump_table(file, insn, insn_jump_table(insn));
dcc914f4
JP		 2158 if (ret)
2159 return ret;
2160 }
2161
2162 return 0;
2163}
2164
2165/*
2166 * For some switch statements, gcc generates a jump table in the .rodata
2167 * section which contains a list of addresses within the function to jump to.
2168 * This finds these jump tables and adds them to the insn->alts lists.
2169 */
e7c2bc37 2170static int add_jump_table_alts(struct objtool_file *file)
dcc914f4 2171{
dcc914f4
JP		 2172 struct symbol *func;
2173 int ret;
2174
4a60aa05 2175 if (!file->rodata)
dcc914f4
JP		 2176 return 0;
2177
9290e772
JP		 2178 for_each_sym(file, func) {
2179 if (func->type != STT_FUNC)
2180 continue;
dcc914f4 2181
9290e772
JP		 2182 mark_func_jump_tables(file, func);
2183 ret = add_func_jump_tables(file, func);
2184 if (ret)
2185 return ret;
dcc914f4
JP		 2186 }
2187
2188 return 0;
2189}
2190
b735bd3e
JP		 2191static void set_func_state(struct cfi_state *state)
2192{
2193 state->cfa = initial_func_cfi.cfa;
2194 memcpy(&state->regs, &initial_func_cfi.regs,
2195 CFI_NUM_REGS * sizeof(struct cfi_reg));
2196 state->stack_size = initial_func_cfi.cfa.offset;
fb799447 2197 state->type = UNWIND_HINT_TYPE_CALL;
b735bd3e
JP		 2198}
2199
39358a03
JP		 2200static int read_unwind_hints(struct objtool_file *file)
2201{
8b946cc3 2202 struct cfi_state cfi = init_cfi;
f1974222 2203 struct section *sec, *relocsec;
39358a03
JP		 2204 struct unwind_hint *hint;
2205 struct instruction *insn;
8b946cc3 2206 struct reloc *reloc;
39358a03
JP		 2207 int i;
2208
2209 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2210 if (!sec)
2211 return 0;
2212
f1974222
MH		 2213 relocsec = sec->reloc;
2214 if (!relocsec) {
39358a03
JP		 2215 WARN("missing .rela.discard.unwind_hints section");
2216 return -1;
2217 }
2218
fe255fe6 2219 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
39358a03
JP		 2220 WARN("struct unwind_hint size mismatch");
2221 return -1;
2222 }
2223
2224 file->hints = true;
2225
fe255fe6 2226 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
39358a03
JP		 2227 hint = (struct unwind_hint *)sec->data->d_buf + i;
2228
f1974222
MH		 2229 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2230 if (!reloc) {
2231 WARN("can't find reloc for unwind_hints[%d]", i);
39358a03
JP		 2232 return -1;
2233 }
2234
f1974222 2235 insn = find_insn(file, reloc->sym->sec, reloc->addend);
39358a03
JP		 2236 if (!insn) {
2237 WARN("can't find insn for unwind_hints[%d]", i);
2238 return -1;
2239 }
2240
b735bd3e 2241 insn->hint = true;
39358a03 2242
8faea26e
JP		 2243 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2244 insn->hint = false;
2245 insn->save = true;
2246 continue;
2247 }
2248
2249 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2250 insn->restore = true;
2251 continue;
2252 }
2253
a09a6e23 2254 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
08f87a93
PZ		 2255 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2256
a09a6e23
PZ		 2257 if (sym && sym->bind == STB_GLOBAL) {
2258 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
246b2c85 2259 WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
a09a6e23 2260 }
08f87a93
PZ		 2261 }
2262 }
2263
b735bd3e 2264 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
8b946cc3 2265 insn->cfi = &func_cfi;
39358a03
JP		 2266 continue;
2267 }
2268
8b946cc3
PZ		 2269 if (insn->cfi)
2270 cfi = *(insn->cfi);
2271
2272 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
246b2c85 2273 WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
39358a03
JP		 2274 return -1;
2275 }
2276
0646c28b 2277 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
8b946cc3 2278 cfi.type = hint->type;
00c8f01c 2279 cfi.signal = hint->signal;
8b946cc3
PZ		 2280
2281 insn->cfi = cfi_hash_find_or_add(&cfi);
39358a03
JP		 2282 }
2283
2284 return 0;
2285}
2286
96db4a98
PZ		 2287static int read_noendbr_hints(struct objtool_file *file)
2288{
2289 struct section *sec;
2290 struct instruction *insn;
2291 struct reloc *reloc;
2292
2293 sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2294 if (!sec)
2295 return 0;
2296
2297 list_for_each_entry(reloc, &sec->reloc_list, list) {
2298 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
2299 if (!insn) {
2300 WARN("bad .discard.noendbr entry");
2301 return -1;
2302 }
2303
2304 insn->noendbr = 1;
2305 }
2306
2307 return 0;
2308}
2309
b5bc2231
PZ		 2310static int read_retpoline_hints(struct objtool_file *file)
2311{
63474dc4 2312 struct section *sec;
b5bc2231 2313 struct instruction *insn;
f1974222 2314 struct reloc *reloc;
b5bc2231 2315
63474dc4 2316 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
b5bc2231
PZ		 2317 if (!sec)
2318 return 0;
2319
f1974222
MH		 2320 list_for_each_entry(reloc, &sec->reloc_list, list) {
2321 if (reloc->sym->type != STT_SECTION) {
63474dc4 2322 WARN("unexpected relocation symbol type in %s", sec->name);
b5bc2231
PZ		 2323 return -1;
2324 }
2325
f1974222 2326 insn = find_insn(file, reloc->sym->sec, reloc->addend);
b5bc2231 2327 if (!insn) {
63474dc4 2328 WARN("bad .discard.retpoline_safe entry");
b5bc2231
PZ		 2329 return -1;
2330 }
2331
2332 if (insn->type != INSN_JUMP_DYNAMIC &&
9bb2ec60 2333 insn->type != INSN_CALL_DYNAMIC &&
a09a6e23
PZ		 2334 insn->type != INSN_RETURN &&
2335 insn->type != INSN_NOP) {
246b2c85 2336 WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
b5bc2231
PZ		 2337 return -1;
2338 }
2339
2340 insn->retpoline_safe = true;
2341 }
2342
2343 return 0;
2344}
2345
c4a33939
PZ		 2346static int read_instr_hints(struct objtool_file *file)
2347{
2348 struct section *sec;
2349 struct instruction *insn;
f1974222 2350 struct reloc *reloc;
c4a33939
PZ		 2351
2352 sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2353 if (!sec)
2354 return 0;
2355
f1974222
MH		 2356 list_for_each_entry(reloc, &sec->reloc_list, list) {
2357 if (reloc->sym->type != STT_SECTION) {
c4a33939
PZ		 2358 WARN("unexpected relocation symbol type in %s", sec->name);
2359 return -1;
2360 }
2361
f1974222 2362 insn = find_insn(file, reloc->sym->sec, reloc->addend);
c4a33939
PZ		 2363 if (!insn) {
2364 WARN("bad .discard.instr_end entry");
2365 return -1;
2366 }
2367
2368 insn->instr--;
2369 }
2370
2371 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2372 if (!sec)
2373 return 0;
2374
f1974222
MH		 2375 list_for_each_entry(reloc, &sec->reloc_list, list) {
2376 if (reloc->sym->type != STT_SECTION) {
c4a33939
PZ		 2377 WARN("unexpected relocation symbol type in %s", sec->name);
2378 return -1;
2379 }
2380
f1974222 2381 insn = find_insn(file, reloc->sym->sec, reloc->addend);
c4a33939
PZ		 2382 if (!insn) {
2383 WARN("bad .discard.instr_begin entry");
2384 return -1;
2385 }
2386
2387 insn->instr++;
2388 }
2389
2390 return 0;
2391}
2392
4708ea14
JP		 2393static int read_validate_unret_hints(struct objtool_file *file)
2394{
2395 struct section *sec;
2396 struct instruction *insn;
2397 struct reloc *reloc;
2398
2399 sec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2400 if (!sec)
2401 return 0;
2402
2403 list_for_each_entry(reloc, &sec->reloc_list, list) {
2404 if (reloc->sym->type != STT_SECTION) {
2405 WARN("unexpected relocation symbol type in %s", sec->name);
2406 return -1;
2407 }
2408
2409 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2410 if (!insn) {
2411 WARN("bad .discard.instr_end entry");
2412 return -1;
2413 }
2414 insn->unret = 1;
2415 }
2416
2417 return 0;
2418}
2419
2420
8aa8eb2a
AC		 2421static int read_intra_function_calls(struct objtool_file *file)
2422{
2423 struct instruction *insn;
2424 struct section *sec;
f1974222 2425 struct reloc *reloc;
8aa8eb2a
AC		 2426
2427 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2428 if (!sec)
2429 return 0;
2430
f1974222 2431 list_for_each_entry(reloc, &sec->reloc_list, list) {
8aa8eb2a
AC		 2432 unsigned long dest_off;
2433
f1974222 2434 if (reloc->sym->type != STT_SECTION) {
8aa8eb2a
AC		 2435 WARN("unexpected relocation symbol type in %s",
2436 sec->name);
2437 return -1;
2438 }
2439
f1974222 2440 insn = find_insn(file, reloc->sym->sec, reloc->addend);
8aa8eb2a
AC		 2441 if (!insn) {
2442 WARN("bad .discard.intra_function_call entry");
2443 return -1;
2444 }
2445
2446 if (insn->type != INSN_CALL) {
246b2c85 2447 WARN_INSN(insn, "intra_function_call not a direct call");
8aa8eb2a
AC		 2448 return -1;
2449 }
2450
2451 /*
2452 * Treat intra-function CALLs as JMPs, but with a stack_op.
2453 * See add_call_destinations(), which strips stack_ops from
2454 * normal CALLs.
2455 */
2456 insn->type = INSN_JUMP_UNCONDITIONAL;
2457
7b3e3186 2458 dest_off = arch_jump_destination(insn);
8aa8eb2a
AC		 2459 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2460 if (!insn->jump_dest) {
246b2c85 2461 WARN_INSN(insn, "can't find call dest at %s+0x%lx",
8aa8eb2a
AC		 2462 insn->sec->name, dest_off);
2463 return -1;
2464 }
2465 }
2466
2467 return 0;
2468}
2469
05098119
ME		 2470/*
2471 * Return true if name matches an instrumentation function, where calls to that
2472 * function from noinstr code can safely be removed, but compilers won't do so.
2473 */
2474static bool is_profiling_func(const char *name)
2475{
2476 /*
2477 * Many compilers cannot disable KCOV with a function attribute.
2478 */
2479 if (!strncmp(name, "__sanitizer_cov_", 16))
2480 return true;
2481
2482 /*
2483 * Some compilers currently do not remove __tsan_func_entry/exit nor
2484 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2485 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2486 * minimum Clang version is 14.0, this can be removed.
2487 */
2488 if (!strncmp(name, "__tsan_func_", 12) ||
2489 !strcmp(name, "__tsan_atomic_signal_fence"))
2490 return true;
2491
2492 return false;
2493}
2494
1739c66e 2495static int classify_symbols(struct objtool_file *file)
1e7e4788 2496{
1e7e4788
JP		 2497 struct symbol *func;
2498
9290e772
JP		 2499 for_each_sym(file, func) {
2500 if (func->bind != STB_GLOBAL)
2501 continue;
1739c66e 2502
9290e772
JP		 2503 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2504 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2505 func->static_call_tramp = true;
1739c66e 2506
9290e772
JP		 2507 if (arch_is_retpoline(func))
2508 func->retpoline_thunk = true;
1739c66e 2509
9290e772
JP		 2510 if (arch_is_rethunk(func))
2511 func->return_thunk = true;
d9e9d230 2512
9290e772
JP		 2513 if (arch_ftrace_match(func->name))
2514 func->fentry = true;
1739c66e 2515
9290e772
JP		 2516 if (is_profiling_func(func->name))
2517 func->profiling_func = true;
1e7e4788
JP		 2518 }
2519
2520 return 0;
2521}
2522
4a60aa05
AX		 2523static void mark_rodata(struct objtool_file *file)
2524{
2525 struct section *sec;
2526 bool found = false;
2527
2528 /*
87b512de
JP		 2529 * Search for the following rodata sections, each of which can
2530 * potentially contain jump tables:
2531 *
2532 * - .rodata: can contain GCC switch tables
2533 * - .rodata.<func>: same, if -fdata-sections is being used
2534 * - .rodata..c_jump_table: contains C annotated jump tables
2535 *
2536 * .rodata.str1.* sections are ignored; they don't contain jump tables.
4a60aa05
AX		 2537 */
2538 for_each_sec(file, sec) {
1ee44470
MS		 2539 if (!strncmp(sec->name, ".rodata", 7) &&
2540 !strstr(sec->name, ".str1.")) {
4a60aa05
AX		 2541 sec->rodata = true;
2542 found = true;
2543 }
2544 }
2545
2546 file->rodata = found;
2547}
2548
dcc914f4
JP		 2549static int decode_sections(struct objtool_file *file)
2550{
2551 int ret;
2552
4a60aa05
AX		 2553 mark_rodata(file);
2554
db2b0c5d
PZ		 2555 ret = init_pv_ops(file);
2556 if (ret)
2557 return ret;
2558
dbcdbdfd
PZ		 2559 /*
2560 * Must be before add_{jump_call}_destination.
2561 */
2562 ret = classify_symbols(file);
2563 if (ret)
2564 return ret;
2565
dcc914f4
JP		 2566 ret = decode_instructions(file);
2567 if (ret)
2568 return ret;
2569
dcc914f4 2570 add_ignores(file);
ea24213d 2571 add_uaccess_safe(file);
dcc914f4 2572
ff05ab23 2573 ret = add_ignore_alternatives(file);
258c7605
JP		 2574 if (ret)
2575 return ret;
2576
08f87a93
PZ		 2577 /*
2578 * Must be before read_unwind_hints() since that needs insn->noendbr.
2579 */
96db4a98
PZ		 2580 ret = read_noendbr_hints(file);
2581 if (ret)
2582 return ret;
2583
43d5430a 2584 /*
34c861e8
JP		 2585 * Must be before add_jump_destinations(), which depends on 'func'
2586 * being set for alternatives, to enable proper sibling call detection.
43d5430a 2587 */
de6fbced
SV		 2588 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2589 ret = add_special_section_alts(file);
2590 if (ret)
2591 return ret;
2592 }
dcc914f4 2593
34c861e8 2594 ret = add_jump_destinations(file);
dcc914f4
JP		 2595 if (ret)
2596 return ret;
2597
a958c4fe
PZ		 2598 /*
2599 * Must be before add_call_destination(); it changes INSN_CALL to
2600 * INSN_JUMP.
2601 */
8aa8eb2a
AC		 2602 ret = read_intra_function_calls(file);
2603 if (ret)
2604 return ret;
2605
a845c7cf 2606 ret = add_call_destinations(file);
dcc914f4
JP		 2607 if (ret)
2608 return ret;
2609
0e5b613b
PZ		 2610 /*
2611 * Must be after add_call_destinations() such that it can override
2612 * dead_end_function() marks.
2613 */
2614 ret = add_dead_ends(file);
2615 if (ret)
2616 return ret;
2617
e7c2bc37 2618 ret = add_jump_table_alts(file);
dcc914f4
JP		 2619 if (ret)
2620 return ret;
2621
39358a03
JP		 2622 ret = read_unwind_hints(file);
2623 if (ret)
2624 return ret;
2625
b5bc2231
PZ		 2626 ret = read_retpoline_hints(file);
2627 if (ret)
2628 return ret;
2629
c4a33939
PZ		 2630 ret = read_instr_hints(file);
2631 if (ret)
2632 return ret;
2633
4708ea14
JP		 2634 ret = read_validate_unret_hints(file);
2635 if (ret)
2636 return ret;
2637
dcc914f4
JP		 2638 return 0;
2639}
2640
2641static bool is_fentry_call(struct instruction *insn)
2642{
1739c66e 2643 if (insn->type == INSN_CALL &&
c6f5dc28
PZ		 2644 insn_call_dest(insn) &&
2645 insn_call_dest(insn)->fentry)
dcc914f4
JP		 2646 return true;
2647
2648 return false;
2649}
2650
e25eea89 2651static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
dcc914f4 2652{
e7c0219b 2653 struct cfi_state *cfi = &state->cfi;
baa41469
JP		 2654 int i;
2655
e7c0219b 2656 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
e25eea89
PZ		 2657 return true;
2658
b735bd3e 2659 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
baa41469
JP		 2660 return true;
2661
b735bd3e 2662 if (cfi->stack_size != initial_func_cfi.cfa.offset)
e25eea89
PZ		 2663 return true;
2664
2665 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b
PZ		 2666 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2667 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
baa41469 2668 return true;
e25eea89 2669 }
baa41469
JP		 2670
2671 return false;
2672}
2673
fb084fde
JT		 2674static bool check_reg_frame_pos(const struct cfi_reg *reg,
2675 int expected_offset)
2676{
2677 return reg->base == CFI_CFA &&
2678 reg->offset == expected_offset;
2679}
2680
baa41469
JP		 2681static bool has_valid_stack_frame(struct insn_state *state)
2682{
e7c0219b
PZ		 2683 struct cfi_state *cfi = &state->cfi;
2684
fb084fde
JT		 2685 if (cfi->cfa.base == CFI_BP &&
2686 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2687 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
baa41469
JP		 2688 return true;
2689
e7c0219b 2690 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
baa41469
JP		 2691 return true;
2692
2693 return false;
dcc914f4
JP		 2694}
2695
e7c0219b
PZ		 2696static int update_cfi_state_regs(struct instruction *insn,
2697 struct cfi_state *cfi,
65ea47dc 2698 struct stack_op *op)
627fce14 2699{
e7c0219b 2700 struct cfi_reg *cfa = &cfi->cfa;
627fce14 2701
d8dd25a4 2702 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
627fce14
JP		 2703 return 0;
2704
2705 /* push */
ea24213d 2706 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
627fce14
JP		 2707 cfa->offset += 8;
2708
2709 /* pop */
ea24213d 2710 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
627fce14
JP		 2711 cfa->offset -= 8;
2712
2713 /* add immediate to sp */
2714 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2715 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2716 cfa->offset -= op->src.offset;
2717
2718 return 0;
2719}
2720
e7c0219b 2721static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
dcc914f4 2722{
bf4d1a83 2723 if (arch_callee_saved_reg(reg) &&
e7c0219b
PZ		 2724 cfi->regs[reg].base == CFI_UNDEFINED) {
2725 cfi->regs[reg].base = base;
2726 cfi->regs[reg].offset = offset;
baa41469 2727 }
dcc914f4
JP		 2728}
2729
e7c0219b 2730static void restore_reg(struct cfi_state *cfi, unsigned char reg)
dcc914f4 2731{
e7c0219b
PZ		 2732 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2733 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
baa41469
JP		 2734}
2735
2736/*
2737 * A note about DRAP stack alignment:
2738 *
2739 * GCC has the concept of a DRAP register, which is used to help keep track of
2740 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2741 * register. The typical DRAP pattern is:
2742 *
2743 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2744 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2745 * 41 ff 72 f8 pushq -0x8(%r10)
2746 * 55 push %rbp
2747 * 48 89 e5 mov %rsp,%rbp
2748 * (more pushes)
2749 * 41 52 push %r10
2750 * ...
2751 * 41 5a pop %r10
2752 * (more pops)
2753 * 5d pop %rbp
2754 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2755 * c3 retq
2756 *
2757 * There are some variations in the epilogues, like:
2758 *
2759 * 5b pop %rbx
2760 * 41 5a pop %r10
2761 * 41 5c pop %r12
2762 * 41 5d pop %r13
2763 * 41 5e pop %r14
2764 * c9 leaveq
2765 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2766 * c3 retq
2767 *
2768 * and:
2769 *
2770 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2771 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2772 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2773 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2774 * c9 leaveq
2775 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2776 * c3 retq
2777 *
2778 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2779 * restored beforehand:
2780 *
2781 * 41 55 push %r13
2782 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2783 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2784 * ...
2785 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2786 * 41 5d pop %r13
2787 * c3 retq
2788 */
d54dba41
PZ		 2789static int update_cfi_state(struct instruction *insn,
2790 struct instruction *next_insn,
2791 struct cfi_state *cfi, struct stack_op *op)
baa41469 2792{
e7c0219b
PZ		 2793 struct cfi_reg *cfa = &cfi->cfa;
2794 struct cfi_reg *regs = cfi->regs;
baa41469
JP		 2795
2796 /* stack operations don't make sense with an undefined CFA */
2797 if (cfa->base == CFI_UNDEFINED) {
dbcdbdfd 2798 if (insn_func(insn)) {
246b2c85 2799 WARN_INSN(insn, "undefined stack state");
baa41469
JP		 2800 return -1;
2801 }
2802 return 0;
2803 }
2804
ee819aed
JT		 2805 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2806 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
e7c0219b 2807 return update_cfi_state_regs(insn, cfi, op);
627fce14 2808
baa41469
JP		 2809 switch (op->dest.type) {
2810
2811 case OP_DEST_REG:
2812 switch (op->src.type) {
2813
2814 case OP_SRC_REG:
0d0970ee
JP		 2815 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2816 cfa->base == CFI_SP &&
fb084fde 2817 check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
0d0970ee
JP		 2818
2819 /* mov %rsp, %rbp */
2820 cfa->base = op->dest.reg;
e7c0219b 2821 cfi->bp_scratch = false;
0d0970ee 2822 }
dd88a0a0 2823
0d0970ee 2824 else if (op->src.reg == CFI_SP &&
e7c0219b 2825 op->dest.reg == CFI_BP && cfi->drap) {
dd88a0a0 2826
0d0970ee
JP		 2827 /* drap: mov %rsp, %rbp */
2828 regs[CFI_BP].base = CFI_BP;
e7c0219b
PZ		 2829 regs[CFI_BP].offset = -cfi->stack_size;
2830 cfi->bp_scratch = false;
0d0970ee 2831 }
dd88a0a0 2832
0d0970ee
JP		 2833 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2834
2835 /*
2836 * mov %rsp, %reg
2837 *
2838 * This is needed for the rare case where GCC
2839 * does:
2840 *
2841 * mov %rsp, %rax
2842 * ...
2843 * mov %rax, %rsp
2844 */
e7c0219b
PZ		 2845 cfi->vals[op->dest.reg].base = CFI_CFA;
2846 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
dd88a0a0
JP		 2847 }
2848
3c1f0583 2849 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
ffc7e74f 2850 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
3c1f0583
JP		 2851
2852 /*
2853 * mov %rbp, %rsp
2854 *
2855 * Restore the original stack pointer (Clang).
2856 */
e7c0219b 2857 cfi->stack_size = -cfi->regs[CFI_BP].offset;
3c1f0583
JP		 2858 }
2859
dd88a0a0
JP		 2860 else if (op->dest.reg == cfa->base) {
2861
2862 /* mov %reg, %rsp */
2863 if (cfa->base == CFI_SP &&
e7c0219b 2864 cfi->vals[op->src.reg].base == CFI_CFA) {
dd88a0a0
JP		 2865
2866 /*
2867 * This is needed for the rare case
2868 * where GCC does something dumb like:
2869 *
2870 * lea 0x8(%rsp), %rcx
2871 * ...
2872 * mov %rcx, %rsp
2873 */
e7c0219b
PZ		 2874 cfa->offset = -cfi->vals[op->src.reg].offset;
2875 cfi->stack_size = cfa->offset;
dd88a0a0 2876
aafeb14e
PZ		 2877 } else if (cfa->base == CFI_SP &&
2878 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2879 cfi->vals[op->src.reg].offset == cfa->offset) {
2880
2881 /*
2882 * Stack swizzle:
2883 *
2884 * 1: mov %rsp, (%[tos])
2885 * 2: mov %[tos], %rsp
2886 * ...
2887 * 3: pop %rsp
2888 *
2889 * Where:
2890 *
2891 * 1 - places a pointer to the previous
2892 * stack at the Top-of-Stack of the
2893 * new stack.
2894 *
2895 * 2 - switches to the new stack.
2896 *
2897 * 3 - pops the Top-of-Stack to restore
2898 * the original stack.
2899 *
2900 * Note: we set base to SP_INDIRECT
2901 * here and preserve offset. Therefore
2902 * when the unwinder reaches ToS it
2903 * will dereference SP and then add the
2904 * offset to find the next frame, IOW:
2905 * (%rsp) + offset.
2906 */
2907 cfa->base = CFI_SP_INDIRECT;
2908
dd88a0a0
JP		 2909 } else {
2910 cfa->base = CFI_UNDEFINED;
2911 cfa->offset = 0;
2912 }
baa41469
JP		 2913 }
2914
724c8a23
PZ		 2915 else if (op->dest.reg == CFI_SP &&
2916 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2917 cfi->vals[op->src.reg].offset == cfa->offset) {
2918
2919 /*
2920 * The same stack swizzle case 2) as above. But
2921 * because we can't change cfa->base, case 3)
2922 * will become a regular POP. Pretend we're a
2923 * PUSH so things don't go unbalanced.
2924 */
2925 cfi->stack_size += 8;
2926 }
2927
2928
baa41469
JP		 2929 break;
2930
2931 case OP_SRC_ADD:
2932 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2933
2934 /* add imm, %rsp */
e7c0219b 2935 cfi->stack_size -= op->src.offset;
baa41469
JP		 2936 if (cfa->base == CFI_SP)
2937 cfa->offset -= op->src.offset;
2938 break;
2939 }
2940
2941 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2942
2943 /* lea disp(%rbp), %rsp */
e7c0219b 2944 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
baa41469
JP		 2945 break;
2946 }
2947
dd88a0a0 2948 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
baa41469
JP		 2949
2950 /* drap: lea disp(%rsp), %drap */
e7c0219b 2951 cfi->drap_reg = op->dest.reg;
dd88a0a0
JP		 2952
2953 /*
2954 * lea disp(%rsp), %reg
2955 *
2956 * This is needed for the rare case where GCC
2957 * does something dumb like:
2958 *
2959 * lea 0x8(%rsp), %rcx
2960 * ...
2961 * mov %rcx, %rsp
2962 */
e7c0219b
PZ		 2963 cfi->vals[op->dest.reg].base = CFI_CFA;
2964 cfi->vals[op->dest.reg].offset = \
2965 -cfi->stack_size + op->src.offset;
dd88a0a0 2966
baa41469
JP		 2967 break;
2968 }
2969
e7c0219b
PZ		 2970 if (cfi->drap && op->dest.reg == CFI_SP &&
2971 op->src.reg == cfi->drap_reg) {
baa41469
JP		 2972
2973 /* drap: lea disp(%drap), %rsp */
2974 cfa->base = CFI_SP;
e7c0219b
PZ		 2975 cfa->offset = cfi->stack_size = -op->src.offset;
2976 cfi->drap_reg = CFI_UNDEFINED;
2977 cfi->drap = false;
baa41469
JP		 2978 break;
2979 }
2980
d54dba41 2981 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
246b2c85 2982 WARN_INSN(insn, "unsupported stack register modification");
baa41469
JP		 2983 return -1;
2984 }
2985
2986 break;
2987
2988 case OP_SRC_AND:
2989 if (op->dest.reg != CFI_SP ||
e7c0219b
PZ		 2990 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2991 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
246b2c85 2992 WARN_INSN(insn, "unsupported stack pointer realignment");
baa41469
JP		 2993 return -1;
2994 }
2995
e7c0219b 2996 if (cfi->drap_reg != CFI_UNDEFINED) {
baa41469 2997 /* drap: and imm, %rsp */
e7c0219b
PZ		 2998 cfa->base = cfi->drap_reg;
2999 cfa->offset = cfi->stack_size = 0;
3000 cfi->drap = true;
baa41469
JP		 3001 }
3002
3003 /*
3004 * Older versions of GCC (4.8ish) realign the stack
3005 * without DRAP, with a frame pointer.
3006 */
3007
3008 break;
3009
3010 case OP_SRC_POP:
ea24213d 3011 case OP_SRC_POPF:
aafeb14e
PZ		 3012 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3013
3014 /* pop %rsp; # restore from a stack swizzle */
3015 cfa->base = CFI_SP;
3016 break;
3017 }
3018
e7c0219b 3019 if (!cfi->drap && op->dest.reg == cfa->base) {
baa41469
JP		 3020
3021 /* pop %rbp */
3022 cfa->base = CFI_SP;
3023 }
3024
e7c0219b
PZ		 3025 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3026 op->dest.reg == cfi->drap_reg &&
3027 cfi->drap_offset == -cfi->stack_size) {
baa41469 3028
bf4d1a83 3029 /* drap: pop %drap */
e7c0219b 3030 cfa->base = cfi->drap_reg;
bf4d1a83 3031 cfa->offset = 0;
e7c0219b 3032 cfi->drap_offset = -1;
baa41469 3033
ffc7e74f 3034 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
baa41469 3035
bf4d1a83 3036 /* pop %reg */
e7c0219b 3037 restore_reg(cfi, op->dest.reg);
baa41469
JP		 3038 }
3039
e7c0219b 3040 cfi->stack_size -= 8;
baa41469
JP		 3041 if (cfa->base == CFI_SP)
3042 cfa->offset -= 8;
3043
3044 break;
3045
3046 case OP_SRC_REG_INDIRECT:
201ef5a9
JT		 3047 if (!cfi->drap && op->dest.reg == cfa->base &&
3048 op->dest.reg == CFI_BP) {
3049
3050 /* mov disp(%rsp), %rbp */
3051 cfa->base = CFI_SP;
3052 cfa->offset = cfi->stack_size;
3053 }
3054
e7c0219b
PZ		 3055 if (cfi->drap && op->src.reg == CFI_BP &&
3056 op->src.offset == cfi->drap_offset) {
bf4d1a83
JP		 3057
3058 /* drap: mov disp(%rbp), %drap */
e7c0219b 3059 cfa->base = cfi->drap_reg;
bf4d1a83 3060 cfa->offset = 0;
e7c0219b 3061 cfi->drap_offset = -1;
bf4d1a83
JP		 3062 }
3063
e7c0219b 3064 if (cfi->drap && op->src.reg == CFI_BP &&
baa41469
JP		 3065 op->src.offset == regs[op->dest.reg].offset) {
3066
3067 /* drap: mov disp(%rbp), %reg */
e7c0219b 3068 restore_reg(cfi, op->dest.reg);
baa41469
JP		 3069
3070 } else if (op->src.reg == cfa->base &&
3071 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3072
3073 /* mov disp(%rbp), %reg */
3074 /* mov disp(%rsp), %reg */
e7c0219b 3075 restore_reg(cfi, op->dest.reg);
201ef5a9
JT		 3076
3077 } else if (op->src.reg == CFI_SP &&
3078 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3079
3080 /* mov disp(%rsp), %reg */
3081 restore_reg(cfi, op->dest.reg);
baa41469
JP		 3082 }
3083
3084 break;
3085
3086 default:
246b2c85 3087 WARN_INSN(insn, "unknown stack-related instruction");
baa41469
JP		 3088 return -1;
3089 }
3090
3091 break;
3092
3093 case OP_DEST_PUSH:
ea24213d 3094 case OP_DEST_PUSHF:
e7c0219b 3095 cfi->stack_size += 8;
baa41469
JP		 3096 if (cfa->base == CFI_SP)
3097 cfa->offset += 8;
3098
3099 if (op->src.type != OP_SRC_REG)
3100 break;
3101
e7c0219b
PZ		 3102 if (cfi->drap) {
3103 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
baa41469
JP		 3104
3105 /* drap: push %drap */
3106 cfa->base = CFI_BP_INDIRECT;
e7c0219b 3107 cfa->offset = -cfi->stack_size;
baa41469 3108
bf4d1a83 3109 /* save drap so we know when to restore it */
e7c0219b 3110 cfi->drap_offset = -cfi->stack_size;
baa41469 3111
e7c0219b 3112 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
baa41469
JP		 3113
3114 /* drap: push %rbp */
e7c0219b 3115 cfi->stack_size = 0;
baa41469 3116
f4f80398 3117 } else {
baa41469
JP		 3118
3119 /* drap: push %reg */
e7c0219b 3120 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
baa41469
JP		 3121 }
3122
3123 } else {
3124
3125 /* push %reg */
e7c0219b 3126 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
baa41469
JP		 3127 }
3128
3129 /* detect when asm code uses rbp as a scratch register */
dbcdbdfd 3130 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
baa41469 3131 cfa->base != CFI_BP)
e7c0219b 3132 cfi->bp_scratch = true;
baa41469
JP		 3133 break;
3134
3135 case OP_DEST_REG_INDIRECT:
3136
e7c0219b
PZ		 3137 if (cfi->drap) {
3138 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
baa41469
JP		 3139
3140 /* drap: mov %drap, disp(%rbp) */
3141 cfa->base = CFI_BP_INDIRECT;
3142 cfa->offset = op->dest.offset;
3143
bf4d1a83 3144 /* save drap offset so we know when to restore it */
e7c0219b 3145 cfi->drap_offset = op->dest.offset;
f4f80398 3146 } else {
baa41469
JP		 3147
3148 /* drap: mov reg, disp(%rbp) */
e7c0219b 3149 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
baa41469
JP		 3150 }
3151
3152 } else if (op->dest.reg == cfa->base) {
3153
3154 /* mov reg, disp(%rbp) */
3155 /* mov reg, disp(%rsp) */
e7c0219b
PZ		 3156 save_reg(cfi, op->src.reg, CFI_CFA,
3157 op->dest.offset - cfi->cfa.offset);
201ef5a9
JT		 3158
3159 } else if (op->dest.reg == CFI_SP) {
3160
3161 /* mov reg, disp(%rsp) */
3162 save_reg(cfi, op->src.reg, CFI_CFA,
3163 op->dest.offset - cfi->stack_size);
aafeb14e
PZ		 3164
3165 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3166
3167 /* mov %rsp, (%reg); # setup a stack swizzle. */
3168 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3169 cfi->vals[op->dest.reg].offset = cfa->offset;
baa41469
JP		 3170 }
3171
3172 break;
3173
baa41469 3174 case OP_DEST_MEM:
ea24213d 3175 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
246b2c85 3176 WARN_INSN(insn, "unknown stack-related memory operation");
baa41469
JP		 3177 return -1;
3178 }
3179
3180 /* pop mem */
e7c0219b 3181 cfi->stack_size -= 8;
baa41469
JP		 3182 if (cfa->base == CFI_SP)
3183 cfa->offset -= 8;
3184
3185 break;
3186
3187 default:
246b2c85 3188 WARN_INSN(insn, "unknown stack-related instruction");
baa41469
JP		 3189 return -1;
3190 }
3191
3192 return 0;
3193}
3194
c9c324dc
JP		 3195/*
3196 * The stack layouts of alternatives instructions can sometimes diverge when
3197 * they have stack modifications. That's fine as long as the potential stack
3198 * layouts don't conflict at any given potential instruction boundary.
3199 *
3200 * Flatten the CFIs of the different alternative code streams (both original
3201 * and replacement) into a single shared CFI array which can be used to detect
3202 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3203 */
3204static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
65ea47dc 3205{
c9c324dc
JP		 3206 struct cfi_state **alt_cfi;
3207 int group_off;
65ea47dc 3208
c9c324dc
JP		 3209 if (!insn->alt_group)
3210 return 0;
65ea47dc 3211
8b946cc3
PZ		 3212 if (!insn->cfi) {
3213 WARN("CFI missing");
3214 return -1;
3215 }
3216
c9c324dc
JP		 3217 alt_cfi = insn->alt_group->cfi;
3218 group_off = insn->offset - insn->alt_group->first_insn->offset;
65ea47dc 3219
c9c324dc 3220 if (!alt_cfi[group_off]) {
8b946cc3 3221 alt_cfi[group_off] = insn->cfi;
c9c324dc 3222 } else {
8b946cc3 3223 if (cficmp(alt_cfi[group_off], insn->cfi)) {
a706bb08
PZ		 3224 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3225 struct instruction *orig = orig_group->first_insn;
3226 char *where = offstr(insn->sec, insn->offset);
246b2c85 3227 WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
a706bb08 3228 free(where);
ab3852ab
PZ		 3229 return -1;
3230 }
c9c324dc
JP		 3231 }
3232
3233 return 0;
3234}
3235
d54dba41
PZ		 3236static int handle_insn_ops(struct instruction *insn,
3237 struct instruction *next_insn,
3238 struct insn_state *state)
c9c324dc
JP		 3239{
3240 struct stack_op *op;
3241
3ee88df1 3242 for (op = insn->stack_ops; op; op = op->next) {
c9c324dc 3243
d54dba41 3244 if (update_cfi_state(insn, next_insn, &state->cfi, op))
c9c324dc 3245 return 1;
ab3852ab 3246
ba08abca
PZ		 3247 if (!insn->alt_group)
3248 continue;
3249
65ea47dc
JT		 3250 if (op->dest.type == OP_DEST_PUSHF) {
3251 if (!state->uaccess_stack) {
3252 state->uaccess_stack = 1;
3253 } else if (state->uaccess_stack >> 31) {
246b2c85 3254 WARN_INSN(insn, "PUSHF stack exhausted");
65ea47dc
JT		 3255 return 1;
3256 }
3257 state->uaccess_stack <<= 1;
3258 state->uaccess_stack |= state->uaccess;
3259 }
3260
3261 if (op->src.type == OP_SRC_POPF) {
3262 if (state->uaccess_stack) {
3263 state->uaccess = state->uaccess_stack & 1;
3264 state->uaccess_stack >>= 1;
3265 if (state->uaccess_stack == 1)
3266 state->uaccess_stack = 0;
3267 }
3268 }
3269 }
3270
3271 return 0;
3272}
3273
e7c0219b 3274static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
baa41469 3275{
8b946cc3 3276 struct cfi_state *cfi1 = insn->cfi;
baa41469
JP		 3277 int i;
3278
8b946cc3
PZ		 3279 if (!cfi1) {
3280 WARN("CFI missing");
3281 return false;
3282 }
3283
e7c0219b
PZ		 3284 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3285
246b2c85 3286 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
e7c0219b
PZ		 3287 cfi1->cfa.base, cfi1->cfa.offset,
3288 cfi2->cfa.base, cfi2->cfa.offset);
baa41469 3289
e7c0219b 3290 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
baa41469 3291 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b 3292 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
baa41469
JP		 3293 sizeof(struct cfi_reg)))
3294 continue;
3295
246b2c85 3296 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
e7c0219b
PZ		 3297 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3298 i, cfi2->regs[i].base, cfi2->regs[i].offset);
baa41469
JP		 3299 break;
3300 }
3301
e7c0219b
PZ		 3302 } else if (cfi1->type != cfi2->type) {
3303
246b2c85
JP		 3304 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3305 cfi1->type, cfi2->type);
e7c0219b
PZ		 3306
3307 } else if (cfi1->drap != cfi2->drap ||
3308 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3309 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
627fce14 3310
246b2c85 3311 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
e7c0219b
PZ		 3312 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3313 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
baa41469
JP		 3314
3315 } else
3316 return true;
3317
3318 return false;
dcc914f4
JP		 3319}
3320
ea24213d
PZ		 3321static inline bool func_uaccess_safe(struct symbol *func)
3322{
3323 if (func)
e10cd8fe 3324 return func->uaccess_safe;
ea24213d
PZ		 3325
3326 return false;
3327}
3328
0c1ddd33 3329static inline const char *call_dest_name(struct instruction *insn)
ea24213d 3330{
82880283 3331 static char pvname[19];
db2b0c5d
PZ		 3332 struct reloc *rel;
3333 int idx;
3334
c6f5dc28
PZ		 3335 if (insn_call_dest(insn))
3336 return insn_call_dest(insn)->name;
ea24213d 3337
db2b0c5d
PZ		 3338 rel = insn_reloc(NULL, insn);
3339 if (rel && !strcmp(rel->sym->name, "pv_ops")) {
3340 idx = (rel->addend / sizeof(void *));
3341 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3342 return pvname;
3343 }
3344
ea24213d
PZ		 3345 return "{dynamic}";
3346}
3347
db2b0c5d
PZ		 3348static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3349{
3350 struct symbol *target;
3351 struct reloc *rel;
3352 int idx;
3353
3354 rel = insn_reloc(file, insn);
3355 if (!rel || strcmp(rel->sym->name, "pv_ops"))
3356 return false;
3357
3358 idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
3359
3360 if (file->pv_ops[idx].clean)
3361 return true;
3362
3363 file->pv_ops[idx].clean = true;
3364
3365 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3366 if (!target->sec->noinstr) {
3367 WARN("pv_ops[%d]: %s", idx, target->name);
3368 file->pv_ops[idx].clean = false;
3369 }
3370 }
3371
3372 return file->pv_ops[idx].clean;
3373}
3374
3375static inline bool noinstr_call_dest(struct objtool_file *file,
3376 struct instruction *insn,
3377 struct symbol *func)
6b643a07
PZ		 3378{
3379 /*
3380 * We can't deal with indirect function calls at present;
3381 * assume they're instrumented.
3382 */
db2b0c5d
PZ		 3383 if (!func) {
3384 if (file->pv_ops)
3385 return pv_call_dest(file, insn);
3386
6b643a07 3387 return false;
db2b0c5d 3388 }
6b643a07
PZ		 3389
3390 /*
3391 * If the symbol is from a noinstr section; we good.
3392 */
3393 if (func->sec->noinstr)
3394 return true;
3395
2b5a0e42
PZ		 3396 /*
3397 * If the symbol is a static_call trampoline, we can't tell.
3398 */
3399 if (func->static_call_tramp)
3400 return true;
3401
6b643a07
PZ		 3402 /*
3403 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3404 * something 'BAD' happened. At the risk of taking the machine down,
3405 * let them proceed to get the message out.
3406 */
3407 if (!strncmp(func->name, "__ubsan_handle_", 15))
3408 return true;
3409
3410 return false;
3411}
3412
db2b0c5d
PZ		 3413static int validate_call(struct objtool_file *file,
3414 struct instruction *insn,
3415 struct insn_state *state)
ea24213d 3416{
c4a33939 3417 if (state->noinstr && state->instr <= 0 &&
c6f5dc28 3418 !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
246b2c85 3419 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
c4a33939
PZ		 3420 return 1;
3421 }
3422
c6f5dc28 3423 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
246b2c85 3424 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
ea24213d
PZ		 3425 return 1;
3426 }
3427
2f0f9e9a 3428 if (state->df) {
246b2c85 3429 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
2f0f9e9a
PZ		 3430 return 1;
3431 }
3432
ea24213d
PZ		 3433 return 0;
3434}
3435
db2b0c5d
PZ		 3436static int validate_sibling_call(struct objtool_file *file,
3437 struct instruction *insn,
3438 struct insn_state *state)
54262aa2 3439{
5a9c361a 3440 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
246b2c85 3441 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
54262aa2
PZ		 3442 return 1;
3443 }
3444
db2b0c5d 3445 return validate_call(file, insn, state);
54262aa2
PZ		 3446}
3447
a92e92d1
PZ		 3448static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3449{
c4a33939 3450 if (state->noinstr && state->instr > 0) {
246b2c85 3451 WARN_INSN(insn, "return with instrumentation enabled");
c4a33939
PZ		 3452 return 1;
3453 }
3454
a92e92d1 3455 if (state->uaccess && !func_uaccess_safe(func)) {
246b2c85 3456 WARN_INSN(insn, "return with UACCESS enabled");
a92e92d1
PZ		 3457 return 1;
3458 }
3459
3460 if (!state->uaccess && func_uaccess_safe(func)) {
246b2c85 3461 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
a92e92d1
PZ		 3462 return 1;
3463 }
3464
3465 if (state->df) {
246b2c85 3466 WARN_INSN(insn, "return with DF set");
a92e92d1
PZ		 3467 return 1;
3468 }
3469
e25eea89 3470 if (func && has_modified_stack_frame(insn, state)) {
246b2c85 3471 WARN_INSN(insn, "return with modified stack frame");
a92e92d1
PZ		 3472 return 1;
3473 }
3474
e7c0219b 3475 if (state->cfi.bp_scratch) {
246b2c85 3476 WARN_INSN(insn, "BP used as a scratch register");
a92e92d1
PZ		 3477 return 1;
3478 }
3479
3480 return 0;
3481}
3482
c9c324dc
JP		 3483static struct instruction *next_insn_to_validate(struct objtool_file *file,
3484 struct instruction *insn)
7117f16b 3485{
b23cc71c 3486 struct alt_group *alt_group = insn->alt_group;
7117f16b 3487
c9c324dc
JP		 3488 /*
3489 * Simulate the fact that alternatives are patched in-place. When the
3490 * end of a replacement alt_group is reached, redirect objtool flow to
3491 * the end of the original alt_group.
1c34496e
PZ		 3492 *
3493 * insn->alts->insn -> alt_group->first_insn
3494 * ...
3495 * alt_group->last_insn
3496 * [alt_group->nop] -> next(orig_group->last_insn)
c9c324dc 3497 */
1c34496e
PZ		 3498 if (alt_group) {
3499 if (alt_group->nop) {
3500 /* ->nop implies ->orig_group */
3501 if (insn == alt_group->last_insn)
3502 return alt_group->nop;
3503 if (insn == alt_group->nop)
3504 goto next_orig;
3505 }
3506 if (insn == alt_group->last_insn && alt_group->orig_group)
3507 goto next_orig;
3508 }
c9c324dc
JP		 3509
3510 return next_insn_same_sec(file, insn);
1c34496e
PZ		 3511
3512next_orig:
3513 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
7117f16b
PZ		 3514}
3515
dcc914f4
JP		 3516/*
3517 * Follow the branch starting at the given instruction, and recursively follow
3518 * any other branches (jumps). Meanwhile, track the frame pointer state at
3519 * each instruction and validate all the rules described in
d6a21f2d 3520 * tools/objtool/Documentation/objtool.txt.
dcc914f4 3521 */
c705cecc 3522static int validate_branch(struct objtool_file *file, struct symbol *func,
b7460462 3523 struct instruction *insn, struct insn_state state)
dcc914f4
JP		 3524{
3525 struct alternative *alt;
8b946cc3 3526 struct instruction *next_insn, *prev_insn = NULL;
dcc914f4 3527 struct section *sec;
882a0db9 3528 u8 visited;
dcc914f4
JP		 3529 int ret;
3530
dcc914f4 3531 sec = insn->sec;
dcc914f4 3532
dcc914f4 3533 while (1) {
c9c324dc 3534 next_insn = next_insn_to_validate(file, insn);
39358a03 3535
dbcdbdfd 3536 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3c68a92d 3537 /* Ignore KCFI type preambles, which always fall through */
9f2899fe
PZ		 3538 if (!strncmp(func->name, "__cfi_", 6) ||
3539 !strncmp(func->name, "__pfx_", 6))
3c68a92d
ST		 3540 return 0;
3541
ee97638b 3542 WARN("%s() falls through to next function %s()",
dbcdbdfd 3543 func->name, insn_func(insn)->name);
ee97638b 3544 return 1;
dcc914f4
JP		 3545 }
3546
4855022a 3547 if (func && insn->ignore) {
246b2c85 3548 WARN_INSN(insn, "BUG: why am I validating an ignored function?");
12b25729 3549 return 1;
4855022a
JP		 3550 }
3551
a09a6e23
PZ		 3552 visited = VISITED_BRANCH << state.uaccess;
3553 if (insn->visited & VISITED_BRANCH_MASK) {
e7c0219b 3554 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
dcc914f4 3555 return 1;
dcc914f4 3556
882a0db9 3557 if (insn->visited & visited)
ea24213d 3558 return 0;
8b946cc3
PZ		 3559 } else {
3560 nr_insns_visited++;
dcc914f4
JP		 3561 }
3562
c4a33939
PZ		 3563 if (state.noinstr)
3564 state.instr += insn->instr;
3565
8b946cc3 3566 if (insn->hint) {
8faea26e
JP		 3567 if (insn->restore) {
3568 struct instruction *save_insn, *i;
3569
3570 i = insn;
3571 save_insn = NULL;
3572
3573 sym_for_each_insn_continue_reverse(file, func, i) {
3574 if (i->save) {
3575 save_insn = i;
3576 break;
3577 }
3578 }
3579
3580 if (!save_insn) {
246b2c85 3581 WARN_INSN(insn, "no corresponding CFI save for CFI restore");
8faea26e
JP		 3582 return 1;
3583 }
3584
3585 if (!save_insn->visited) {
246b2c85 3586 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
8faea26e
JP		 3587 return 1;
3588 }
3589
3590 insn->cfi = save_insn->cfi;
3591 nr_cfi_reused++;
3592 }
3593
8b946cc3
PZ		 3594 state.cfi = *insn->cfi;
3595 } else {
3596 /* XXX track if we actually changed state.cfi */
3597
3598 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3599 insn->cfi = prev_insn->cfi;
3600 nr_cfi_reused++;
3601 } else {
3602 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3603 }
3604 }
dcc914f4 3605
882a0db9 3606 insn->visited |= visited;
baa41469 3607
c9c324dc
JP		 3608 if (propagate_alt_cfi(file, insn))
3609 return 1;
3610
d5406654 3611 if (!insn->ignore_alts && insn->alts) {
764eef4b
PZ		 3612 bool skip_orig = false;
3613
d5406654 3614 for (alt = insn->alts; alt; alt = alt->next) {
764eef4b
PZ		 3615 if (alt->skip_orig)
3616 skip_orig = true;
3617
c705cecc 3618 ret = validate_branch(file, func, alt->insn, state);
7697eee3 3619 if (ret) {
ced23d2e 3620 BT_INSN(insn, "(alt)");
7697eee3
PZ		 3621 return ret;
3622 }
a845c7cf 3623 }
764eef4b
PZ		 3624
3625 if (skip_orig)
3626 return 0;
dcc914f4
JP		 3627 }
3628
d54dba41 3629 if (handle_insn_ops(insn, next_insn, &state))
60041bcd
PZ		 3630 return 1;
3631
dcc914f4
JP		 3632 switch (insn->type) {
3633
dcc914f4 3634 case INSN_RETURN:
a92e92d1 3635 return validate_return(func, insn, &state);
dcc914f4
JP		 3636
3637 case INSN_CALL:
ea24213d 3638 case INSN_CALL_DYNAMIC:
db2b0c5d 3639 ret = validate_call(file, insn, &state);
ea24213d
PZ		 3640 if (ret)
3641 return ret;
dcc914f4 3642
72064474 3643 if (opts.stackval && func && !is_fentry_call(insn) &&
c9bab22b 3644 !has_valid_stack_frame(&state)) {
246b2c85 3645 WARN_INSN(insn, "call without frame pointer save/setup");
dcc914f4
JP		 3646 return 1;
3647 }
c9bab22b 3648
0e5b613b 3649 if (insn->dead_end)
c9bab22b
JP		 3650 return 0;
3651
dcc914f4
JP		 3652 break;
3653
3654 case INSN_JUMP_CONDITIONAL:
3655 case INSN_JUMP_UNCONDITIONAL:
ecf11ba4 3656 if (is_sibling_call(insn)) {
db2b0c5d 3657 ret = validate_sibling_call(file, insn, &state);
dcc914f4 3658 if (ret)
54262aa2 3659 return ret;
4855022a 3660
0c1ddd33 3661 } else if (insn->jump_dest) {
c705cecc
JP		 3662 ret = validate_branch(file, func,
3663 insn->jump_dest, state);
7697eee3 3664 if (ret) {
ced23d2e 3665 BT_INSN(insn, "(branch)");
7697eee3
PZ		 3666 return ret;
3667 }
4855022a 3668 }
dcc914f4
JP		 3669
3670 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3671 return 0;
3672
3673 break;
3674
3675 case INSN_JUMP_DYNAMIC:
b68b9907 3676 case INSN_JUMP_DYNAMIC_CONDITIONAL:
ecf11ba4 3677 if (is_sibling_call(insn)) {
db2b0c5d 3678 ret = validate_sibling_call(file, insn, &state);
54262aa2
PZ		 3679 if (ret)
3680 return ret;
dcc914f4
JP		 3681 }
3682
b68b9907
JP		 3683 if (insn->type == INSN_JUMP_DYNAMIC)
3684 return 0;
3685
3686 break;
dcc914f4 3687
39358a03
JP		 3688 case INSN_CONTEXT_SWITCH:
3689 if (func && (!next_insn || !next_insn->hint)) {
246b2c85 3690 WARN_INSN(insn, "unsupported instruction in callable function");
39358a03
JP		 3691 return 1;
3692 }
3693 return 0;
3694
ea24213d
PZ		 3695 case INSN_STAC:
3696 if (state.uaccess) {
246b2c85 3697 WARN_INSN(insn, "recursive UACCESS enable");
ea24213d
PZ		 3698 return 1;
3699 }
3700
3701 state.uaccess = true;
3702 break;
3703
3704 case INSN_CLAC:
c705cecc 3705 if (!state.uaccess && func) {
246b2c85 3706 WARN_INSN(insn, "redundant UACCESS disable");
ea24213d
PZ		 3707 return 1;
3708 }
3709
3710 if (func_uaccess_safe(func) && !state.uaccess_stack) {
246b2c85 3711 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
ea24213d
PZ		 3712 return 1;
3713 }
3714
3715 state.uaccess = false;
baa41469
JP		 3716 break;
3717
2f0f9e9a 3718 case INSN_STD:
6f567c93 3719 if (state.df) {
246b2c85 3720 WARN_INSN(insn, "recursive STD");
6f567c93
JP		 3721 return 1;
3722 }
2f0f9e9a
PZ		 3723
3724 state.df = true;
3725 break;
3726
3727 case INSN_CLD:
6f567c93 3728 if (!state.df && func) {
246b2c85 3729 WARN_INSN(insn, "redundant CLD");
6f567c93
JP		 3730 return 1;
3731 }
2f0f9e9a
PZ		 3732
3733 state.df = false;
baa41469
JP		 3734 break;
3735
dcc914f4
JP		 3736 default:
3737 break;
3738 }
3739
3740 if (insn->dead_end)
3741 return 0;
3742
00d96180 3743 if (!next_insn) {
e7c0219b 3744 if (state.cfi.cfa.base == CFI_UNDEFINED)
00d96180 3745 return 0;
dcc914f4
JP		 3746 WARN("%s: unexpected end of section", sec->name);
3747 return 1;
3748 }
00d96180 3749
8b946cc3 3750 prev_insn = insn;
00d96180 3751 insn = next_insn;
dcc914f4
JP		 3752 }
3753
3754 return 0;
3755}
3756
1c34496e
PZ		 3757static int validate_unwind_hint(struct objtool_file *file,
3758 struct instruction *insn,
3759 struct insn_state *state)
3760{
3761 if (insn->hint && !insn->visited && !insn->ignore) {
3762 int ret = validate_branch(file, insn_func(insn), insn, *state);
ced23d2e
JP		 3763 if (ret)
3764 BT_INSN(insn, "<=== (hint)");
1c34496e
PZ		 3765 return ret;
3766 }
3767
3768 return 0;
3769}
3770
932f8e98 3771static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
39358a03
JP		 3772{
3773 struct instruction *insn;
39358a03 3774 struct insn_state state;
1c34496e 3775 int warnings = 0;
39358a03
JP		 3776
3777 if (!file->hints)
3778 return 0;
3779
753da417 3780 init_insn_state(file, &state, sec);
39358a03 3781
932f8e98 3782 if (sec) {
1c34496e
PZ		 3783 sec_for_each_insn(file, sec, insn)
3784 warnings += validate_unwind_hint(file, insn, &state);
932f8e98 3785 } else {
1c34496e
PZ		 3786 for_each_insn(file, insn)
3787 warnings += validate_unwind_hint(file, insn, &state);
39358a03
JP		 3788 }
3789
3790 return warnings;
3791}
3792
a09a6e23
PZ		 3793/*
3794 * Validate rethunk entry constraint: must untrain RET before the first RET.
3795 *
4708ea14 3796 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
a09a6e23
PZ		 3797 * before an actual RET instruction.
3798 */
4708ea14 3799static int validate_unret(struct objtool_file *file, struct instruction *insn)
a09a6e23
PZ		 3800{
3801 struct instruction *next, *dest;
d49d1666 3802 int ret;
a09a6e23
PZ		 3803
3804 for (;;) {
3805 next = next_insn_to_validate(file, insn);
3806
4708ea14 3807 if (insn->visited & VISITED_UNRET)
a09a6e23
PZ		 3808 return 0;
3809
4708ea14 3810 insn->visited |= VISITED_UNRET;
a09a6e23 3811
d5406654 3812 if (!insn->ignore_alts && insn->alts) {
a09a6e23
PZ		 3813 struct alternative *alt;
3814 bool skip_orig = false;
3815
d5406654 3816 for (alt = insn->alts; alt; alt = alt->next) {
a09a6e23
PZ		 3817 if (alt->skip_orig)
3818 skip_orig = true;
3819
4708ea14 3820 ret = validate_unret(file, alt->insn);
a09a6e23 3821 if (ret) {
ced23d2e 3822 BT_INSN(insn, "(alt)");
a09a6e23
PZ		 3823 return ret;
3824 }
3825 }
3826
3827 if (skip_orig)
3828 return 0;
3829 }
3830
3831 switch (insn->type) {
3832
3833 case INSN_CALL_DYNAMIC:
3834 case INSN_JUMP_DYNAMIC:
3835 case INSN_JUMP_DYNAMIC_CONDITIONAL:
246b2c85 3836 WARN_INSN(insn, "early indirect call");
a09a6e23
PZ		 3837 return 1;
3838
3839 case INSN_JUMP_UNCONDITIONAL:
3840 case INSN_JUMP_CONDITIONAL:
3841 if (!is_sibling_call(insn)) {
3842 if (!insn->jump_dest) {
246b2c85 3843 WARN_INSN(insn, "unresolved jump target after linking?!?");
a09a6e23
PZ		 3844 return -1;
3845 }
4708ea14 3846 ret = validate_unret(file, insn->jump_dest);
a09a6e23 3847 if (ret) {
ced23d2e
JP		 3848 BT_INSN(insn, "(branch%s)",
3849 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
a09a6e23
PZ		 3850 return ret;
3851 }
3852
3853 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3854 return 0;
3855
3856 break;
3857 }
3858
3859 /* fallthrough */
3860 case INSN_CALL:
c6f5dc28
PZ		 3861 dest = find_insn(file, insn_call_dest(insn)->sec,
3862 insn_call_dest(insn)->offset);
a09a6e23
PZ		 3863 if (!dest) {
3864 WARN("Unresolved function after linking!?: %s",
c6f5dc28 3865 insn_call_dest(insn)->name);
a09a6e23
PZ		 3866 return -1;
3867 }
3868
4708ea14 3869 ret = validate_unret(file, dest);
a09a6e23 3870 if (ret) {
ced23d2e 3871 BT_INSN(insn, "(call)");
a09a6e23
PZ		 3872 return ret;
3873 }
3874 /*
3875 * If a call returns without error, it must have seen UNTRAIN_RET.
3876 * Therefore any non-error return is a success.
3877 */
3878 return 0;
3879
3880 case INSN_RETURN:
246b2c85 3881 WARN_INSN(insn, "RET before UNTRAIN");
a09a6e23
PZ		 3882 return 1;
3883
3884 case INSN_NOP:
3885 if (insn->retpoline_safe)
3886 return 0;
3887 break;
3888
3889 default:
3890 break;
3891 }
3892
3893 if (!next) {
246b2c85 3894 WARN_INSN(insn, "teh end!");
a09a6e23
PZ		 3895 return -1;
3896 }
3897 insn = next;
3898 }
3899
d49d1666 3900 return 0;
a09a6e23
PZ		 3901}
3902
3903/*
4708ea14
JP		 3904 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3905 * VALIDATE_UNRET_END before RET.
a09a6e23 3906 */
4708ea14 3907static int validate_unrets(struct objtool_file *file)
a09a6e23
PZ		 3908{
3909 struct instruction *insn;
3910 int ret, warnings = 0;
3911
3912 for_each_insn(file, insn) {
4708ea14 3913 if (!insn->unret)
a09a6e23
PZ		 3914 continue;
3915
4708ea14 3916 ret = validate_unret(file, insn);
a09a6e23 3917 if (ret < 0) {
246b2c85 3918 WARN_INSN(insn, "Failed UNRET validation");
a09a6e23
PZ		 3919 return ret;
3920 }
3921 warnings += ret;
3922 }
3923
3924 return warnings;
3925}
3926
b5bc2231
PZ		 3927static int validate_retpoline(struct objtool_file *file)
3928{
3929 struct instruction *insn;
3930 int warnings = 0;
3931
3932 for_each_insn(file, insn) {
3933 if (insn->type != INSN_JUMP_DYNAMIC &&
9bb2ec60
PZ		 3934 insn->type != INSN_CALL_DYNAMIC &&
3935 insn->type != INSN_RETURN)
b5bc2231
PZ		 3936 continue;
3937
3938 if (insn->retpoline_safe)
3939 continue;
3940
6644ee84 3941 if (insn->sec->init)
ca41b97e
PZ		 3942 continue;
3943
9bb2ec60 3944 if (insn->type == INSN_RETURN) {
f43b9876 3945 if (opts.rethunk) {
246b2c85 3946 WARN_INSN(insn, "'naked' return found in RETHUNK build");
f43b9876
PZ		 3947 } else
3948 continue;
9bb2ec60 3949 } else {
246b2c85 3950 WARN_INSN(insn, "indirect %s found in RETPOLINE build",
9bb2ec60
PZ		 3951 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3952 }
b5bc2231
PZ		 3953
3954 warnings++;
3955 }
3956
3957 return warnings;
3958}
3959
dcc914f4
JP		 3960static bool is_kasan_insn(struct instruction *insn)
3961{
3962 return (insn->type == INSN_CALL &&
c6f5dc28 3963 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
dcc914f4
JP		 3964}
3965
3966static bool is_ubsan_insn(struct instruction *insn)
3967{
3968 return (insn->type == INSN_CALL &&
c6f5dc28 3969 !strcmp(insn_call_dest(insn)->name,
dcc914f4
JP		 3970 "__ubsan_handle_builtin_unreachable"));
3971}
3972
14db1f0a 3973static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
dcc914f4
JP		 3974{
3975 int i;
14db1f0a 3976 struct instruction *prev_insn;
dcc914f4 3977
1ffbe4e9 3978 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
baa41469
JP		 3979 return true;
3980
3981 /*
82a8954a 3982 * Ignore alternative replacement instructions. This can happen
0e2bb2bc 3983 * when a whitelisted function uses one of the ALTERNATIVE macros.
baa41469 3984 */
82a8954a 3985 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
0e2bb2bc 3986 !strcmp(insn->sec->name, ".altinstr_aux"))
dcc914f4
JP		 3987 return true;
3988
4adb2368 3989 /*
753da417 3990 * Whole archive runs might encounter dead code from weak symbols.
4adb2368
PZ		 3991 * This is where the linker will have dropped the weak symbol in
3992 * favour of a regular symbol, but leaves the code in place.
3993 *
3994 * In this case we'll find a piece of code (whole function) that is not
3995 * covered by a !section symbol. Ignore them.
3996 */
dbcdbdfd 3997 if (opts.link && !insn_func(insn)) {
4adb2368
PZ		 3998 int size = find_symbol_hole_containing(insn->sec, insn->offset);
3999 unsigned long end = insn->offset + size;
4000
4001 if (!size) /* not a hole */
4002 return false;
4003
4004 if (size < 0) /* hole until the end */
4005 return true;
4006
4007 sec_for_each_insn_continue(file, insn) {
4008 /*
4009 * If we reach a visited instruction at or before the
4010 * end of the hole, ignore the unreachable.
4011 */
4012 if (insn->visited)
4013 return true;
4014
4015 if (insn->offset >= end)
4016 break;
4017
4018 /*
4019 * If this hole jumps to a .cold function, mark it ignore too.
4020 */
dbcdbdfd
PZ		 4021 if (insn->jump_dest && insn_func(insn->jump_dest) &&
4022 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4adb2368 4023 struct instruction *dest = insn->jump_dest;
dbcdbdfd 4024 func_for_each_insn(file, insn_func(dest), dest)
4adb2368
PZ		 4025 dest->ignore = true;
4026 }
4027 }
4028
4029 return false;
4030 }
4031
dbcdbdfd 4032 if (!insn_func(insn))
bd841d61
JP		 4033 return false;
4034
dbcdbdfd 4035 if (insn_func(insn)->static_call_tramp)
2105a927
PZ		 4036 return true;
4037
bd841d61
JP		 4038 /*
4039 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4040 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4041 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4042 * (or occasionally a JMP to UD2).
14db1f0a
IH		 4043 *
4044 * It may also insert a UD2 after calling a __noreturn function.
bd841d61 4045 */
1c34496e 4046 prev_insn = prev_insn_same_sec(file, insn);
6126ed5d 4047 if (prev_insn->dead_end &&
bd841d61
JP		 4048 (insn->type == INSN_BUG ||
4049 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4050 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4051 return true;
4052
dcc914f4
JP		 4053 /*
4054 * Check if this (or a subsequent) instruction is related to
4055 * CONFIG_UBSAN or CONFIG_KASAN.
4056 *
4057 * End the search at 5 instructions to avoid going into the weeds.
4058 */
4059 for (i = 0; i < 5; i++) {
4060
4061 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4062 return true;
4063
fe24e271
JP		 4064 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4065 if (insn->jump_dest &&
dbcdbdfd 4066 insn_func(insn->jump_dest) == insn_func(insn)) {
fe24e271
JP		 4067 insn = insn->jump_dest;
4068 continue;
4069 }
4070
4071 break;
dcc914f4
JP		 4072 }
4073
dbcdbdfd 4074 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
dcc914f4 4075 break;
fe24e271 4076
1c34496e 4077 insn = next_insn_same_sec(file, insn);
dcc914f4
JP		 4078 }
4079
4080 return false;
4081}
4082
bd456a1b 4083static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
9f2899fe 4084{
bd456a1b 4085 struct instruction *insn, *prev;
5743654f 4086 struct cfi_state *cfi;
9f2899fe 4087
bd456a1b
JP		 4088 insn = find_insn(file, func->sec, func->offset);
4089 if (!insn)
4090 return -1;
9f2899fe 4091
bd456a1b
JP		 4092 for (prev = prev_insn_same_sec(file, insn);
4093 prev;
4094 prev = prev_insn_same_sec(file, prev)) {
4095 u64 offset;
9f2899fe
PZ		 4096
4097 if (prev->type != INSN_NOP)
bd456a1b 4098 return -1;
9f2899fe
PZ		 4099
4100 offset = func->offset - prev->offset;
bd456a1b
JP		 4101
4102 if (offset > opts.prefix)
4103 return -1;
4104
4105 if (offset < opts.prefix)
4106 continue;
4107
4108 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4109 break;
9f2899fe
PZ		 4110 }
4111
bd456a1b
JP		 4112 if (!prev)
4113 return -1;
4114
5743654f
JP		 4115 if (!insn->cfi) {
4116 /*
4117 * This can happen if stack validation isn't enabled or the
4118 * function is annotated with STACK_FRAME_NON_STANDARD.
4119 */
4120 return 0;
4121 }
4122
4123 /* Propagate insn->cfi to the prefix code */
4124 cfi = cfi_hash_find_or_add(insn->cfi);
4125 for (; prev != insn; prev = next_insn_same_sec(file, prev))
4126 prev->cfi = cfi;
4127
9f2899fe
PZ		 4128 return 0;
4129}
4130
bd456a1b
JP		 4131static int add_prefix_symbols(struct objtool_file *file)
4132{
4133 struct section *sec;
4134 struct symbol *func;
bd456a1b
JP		 4135
4136 for_each_sec(file, sec) {
4137 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4138 continue;
4139
4140 sec_for_each_sym(sec, func) {
4141 if (func->type != STT_FUNC)
4142 continue;
4143
4144 add_prefix_symbol(file, func);
4145 }
4146 }
4147
d49d1666 4148 return 0;
bd456a1b
JP		 4149}
4150
4b5e2e7f
PZ		 4151static int validate_symbol(struct objtool_file *file, struct section *sec,
4152 struct symbol *sym, struct insn_state *state)
dcc914f4 4153{
dcc914f4 4154 struct instruction *insn;
4b5e2e7f
PZ		 4155 int ret;
4156
4157 if (!sym->len) {
4158 WARN("%s() is missing an ELF size annotation", sym->name);
4159 return 1;
4160 }
4161
4162 if (sym->pfunc != sym || sym->alias != sym)
4163 return 0;
4164
4165 insn = find_insn(file, sec, sym->offset);
4166 if (!insn || insn->ignore || insn->visited)
4167 return 0;
4168
4169 state->uaccess = sym->uaccess_safe;
4170
dbcdbdfd 4171 ret = validate_branch(file, insn_func(insn), insn, *state);
ced23d2e
JP		 4172 if (ret)
4173 BT_INSN(insn, "<=== (sym)");
4b5e2e7f
PZ		 4174 return ret;
4175}
4176
4177static int validate_section(struct objtool_file *file, struct section *sec)
4178{
baa41469 4179 struct insn_state state;
4b5e2e7f
PZ		 4180 struct symbol *func;
4181 int warnings = 0;
dcc914f4 4182
9290e772 4183 sec_for_each_sym(sec, func) {
350994bf
PZ		 4184 if (func->type != STT_FUNC)
4185 continue;
e10cd8fe 4186
753da417 4187 init_insn_state(file, &state, sec);
b735bd3e 4188 set_func_state(&state.cfi);
0699e551 4189
4b5e2e7f 4190 warnings += validate_symbol(file, sec, func, &state);
dcc914f4
JP		 4191 }
4192
dcc914f4
JP		 4193 return warnings;
4194}
4195
753da417 4196static int validate_noinstr_sections(struct objtool_file *file)
c4a33939
PZ		 4197{
4198 struct section *sec;
932f8e98 4199 int warnings = 0;
c4a33939
PZ		 4200
4201 sec = find_section_by_name(file->elf, ".noinstr.text");
0cc9ac8d
TG		 4202 if (sec) {
4203 warnings += validate_section(file, sec);
4204 warnings += validate_unwind_hints(file, sec);
4205 }
c4a33939 4206
0cc9ac8d
TG		 4207 sec = find_section_by_name(file->elf, ".entry.text");
4208 if (sec) {
4209 warnings += validate_section(file, sec);
4210 warnings += validate_unwind_hints(file, sec);
4211 }
932f8e98 4212
2b5a0e42
PZ		 4213 sec = find_section_by_name(file->elf, ".cpuidle.text");
4214 if (sec) {
4215 warnings += validate_section(file, sec);
4216 warnings += validate_unwind_hints(file, sec);
4217 }
4218
932f8e98 4219 return warnings;
c4a33939
PZ		 4220}
4221
350994bf
PZ		 4222static int validate_functions(struct objtool_file *file)
4223{
4224 struct section *sec;
4225 int warnings = 0;
4226
da837bd6
PZ		 4227 for_each_sec(file, sec) {
4228 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4229 continue;
4230
350994bf 4231 warnings += validate_section(file, sec);
da837bd6 4232 }
350994bf
PZ		 4233
4234 return warnings;
4235}
4236
3c6f9f77 4237static void mark_endbr_used(struct instruction *insn)
08f87a93 4238{
3c6f9f77
JP		 4239 if (!list_empty(&insn->call_node))
4240 list_del_init(&insn->call_node);
4241}
4242
08ef8c40
PZ		 4243static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4244{
4245 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4246 struct instruction *first;
4247
4248 if (!sym)
4249 return false;
4250
4251 first = find_insn(file, sym->sec, sym->offset);
4252 if (!first)
4253 return false;
4254
4255 if (first->type != INSN_ENDBR && !first->noendbr)
4256 return false;
4257
4258 return insn->offset == sym->offset + sym->len;
4259}
4260
3c6f9f77
JP		 4261static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4262{
4263 struct instruction *dest;
08f87a93 4264 struct reloc *reloc;
3c6f9f77
JP		 4265 unsigned long off;
4266 int warnings = 0;
08f87a93 4267
3c6f9f77
JP		 4268 /*
4269 * Looking for function pointer load relocations. Ignore
4270 * direct/indirect branches:
4271 */
4272 switch (insn->type) {
4273 case INSN_CALL:
4274 case INSN_CALL_DYNAMIC:
4275 case INSN_JUMP_CONDITIONAL:
4276 case INSN_JUMP_UNCONDITIONAL:
4277 case INSN_JUMP_DYNAMIC:
4278 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4279 case INSN_RETURN:
4280 case INSN_NOP:
4281 return 0;
4282 default:
4283 break;
4284 }
08f87a93 4285
3c6f9f77
JP		 4286 for (reloc = insn_reloc(file, insn);
4287 reloc;
4288 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4289 reloc->offset + 1,
4290 (insn->offset + insn->len) - (reloc->offset + 1))) {
08f87a93 4291
3c6f9f77
JP		 4292 /*
4293 * static_call_update() references the trampoline, which
4294 * doesn't have (or need) ENDBR. Skip warning in that case.
4295 */
4296 if (reloc->sym->static_call_tramp)
08f87a93
PZ		 4297 continue;
4298
3c6f9f77
JP		 4299 off = reloc->sym->offset;
4300 if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
4301 off += arch_dest_reloc_offset(reloc->addend);
4302 else
4303 off += reloc->addend;
4304
4305 dest = find_insn(file, reloc->sym->sec, off);
4306 if (!dest)
08f87a93
PZ		 4307 continue;
4308
3c6f9f77
JP		 4309 if (dest->type == INSN_ENDBR) {
4310 mark_endbr_used(dest);
08f87a93 4311 continue;
3c6f9f77 4312 }
08f87a93 4313
dbcdbdfd 4314 if (insn_func(dest) && insn_func(dest) == insn_func(insn)) {
3c6f9f77
JP		 4315 /*
4316 * Anything from->to self is either _THIS_IP_ or
4317 * IRET-to-self.
4318 *
4319 * There is no sane way to annotate _THIS_IP_ since the
4320 * compiler treats the relocation as a constant and is
4321 * happy to fold in offsets, skewing any annotation we
4322 * do, leading to vast amounts of false-positives.
4323 *
4324 * There's also compiler generated _THIS_IP_ through
4325 * KCOV and such which we have no hope of annotating.
4326 *
4327 * As such, blanket accept self-references without
4328 * issue.
4329 */
08f87a93 4330 continue;
3c6f9f77 4331 }
08f87a93 4332
08ef8c40
PZ		 4333 /*
4334 * Accept anything ANNOTATE_NOENDBR.
4335 */
3c6f9f77 4336 if (dest->noendbr)
08f87a93
PZ		 4337 continue;
4338
08ef8c40
PZ		 4339 /*
4340 * Accept if this is the instruction after a symbol
4341 * that is (no)endbr -- typical code-range usage.
4342 */
4343 if (noendbr_range(file, dest))
4344 continue;
4345
246b2c85 4346 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
3c6f9f77
JP		 4347
4348 warnings++;
4349 }
4350
4351 return warnings;
4352}
4353
4354static int validate_ibt_data_reloc(struct objtool_file *file,
4355 struct reloc *reloc)
4356{
4357 struct instruction *dest;
4358
4359 dest = find_insn(file, reloc->sym->sec,
4360 reloc->sym->offset + reloc->addend);
4361 if (!dest)
4362 return 0;
4363
4364 if (dest->type == INSN_ENDBR) {
4365 mark_endbr_used(dest);
4366 return 0;
4367 }
4368
4369 if (dest->noendbr)
4370 return 0;
4371
4372 WARN_FUNC("data relocation to !ENDBR: %s",
4373 reloc->sec->base, reloc->offset,
4374 offstr(dest->sec, dest->offset));
4375
4376 return 1;
4377}
4378
4379/*
4380 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4381 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4382 * NOPs) later, in create_ibt_endbr_seal_sections().
4383 */
4384static int validate_ibt(struct objtool_file *file)
4385{
4386 struct section *sec;
4387 struct reloc *reloc;
4388 struct instruction *insn;
4389 int warnings = 0;
4390
4391 for_each_insn(file, insn)
4392 warnings += validate_ibt_insn(file, insn);
4393
4394 for_each_sec(file, sec) {
4395
4396 /* Already done by validate_ibt_insn() */
4397 if (sec->sh.sh_flags & SHF_EXECINSTR)
08f87a93
PZ		 4398 continue;
4399
3c6f9f77
JP		 4400 if (!sec->reloc)
4401 continue;
08f87a93 4402
3c6f9f77
JP		 4403 /*
4404 * These sections can reference text addresses, but not with
4405 * the intent to indirect branch to them.
4406 */
e27e5bea
JP		 4407 if ((!strncmp(sec->name, ".discard", 8) &&
4408 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
3c6f9f77
JP		 4409 !strncmp(sec->name, ".debug", 6) ||
4410 !strcmp(sec->name, ".altinstructions") ||
4411 !strcmp(sec->name, ".ibt_endbr_seal") ||
4412 !strcmp(sec->name, ".orc_unwind_ip") ||
4413 !strcmp(sec->name, ".parainstructions") ||
4414 !strcmp(sec->name, ".retpoline_sites") ||
4415 !strcmp(sec->name, ".smp_locks") ||
4416 !strcmp(sec->name, ".static_call_sites") ||
4417 !strcmp(sec->name, "_error_injection_whitelist") ||
4418 !strcmp(sec->name, "_kprobe_blacklist") ||
4419 !strcmp(sec->name, "__bug_table") ||
4420 !strcmp(sec->name, "__ex_table") ||
4421 !strcmp(sec->name, "__jump_table") ||
3c68a92d 4422 !strcmp(sec->name, "__mcount_loc") ||
0326074f 4423 !strcmp(sec->name, ".kcfi_traps") ||
9440155c 4424 strstr(sec->name, "__patchable_function_entries"))
3c6f9f77 4425 continue;
08f87a93 4426
3c6f9f77
JP		 4427 list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
4428 warnings += validate_ibt_data_reloc(file, reloc);
08f87a93
PZ		 4429 }
4430
3c6f9f77 4431 return warnings;
08f87a93
PZ		 4432}
4433
c2bdd61c
JP		 4434static int validate_sls(struct objtool_file *file)
4435{
4436 struct instruction *insn, *next_insn;
4437 int warnings = 0;
4438
4439 for_each_insn(file, insn) {
4440 next_insn = next_insn_same_sec(file, insn);
4441
4442 if (insn->retpoline_safe)
4443 continue;
4444
4445 switch (insn->type) {
4446 case INSN_RETURN:
4447 if (!next_insn || next_insn->type != INSN_TRAP) {
246b2c85 4448 WARN_INSN(insn, "missing int3 after ret");
c2bdd61c
JP		 4449 warnings++;
4450 }
4451
4452 break;
4453 case INSN_JUMP_DYNAMIC:
4454 if (!next_insn || next_insn->type != INSN_TRAP) {
246b2c85 4455 WARN_INSN(insn, "missing int3 after indirect jump");
c2bdd61c
JP		 4456 warnings++;
4457 }
4458 break;
4459 default:
4460 break;
4461 }
4462 }
4463
4464 return warnings;
4465}
4466
55eeab2a
JP		 4467static bool ignore_noreturn_call(struct instruction *insn)
4468{
4469 struct symbol *call_dest = insn_call_dest(insn);
4470
4471 /*
4472 * FIXME: hack, we need a real noreturn solution
4473 *
4474 * Problem is, exc_double_fault() may or may not return, depending on
4475 * whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4476 * to the kernel config.
4477 *
4478 * Other potential ways to fix it:
4479 *
4480 * - have compiler communicate __noreturn functions somehow
4481 * - remove CONFIG_X86_ESPFIX64
4482 * - read the .config file
4483 * - add a cmdline option
4484 * - create a generic objtool annotation format (vs a bunch of custom
4485 * formats) and annotate it
4486 */
4487 if (!strcmp(call_dest->name, "exc_double_fault")) {
4488 /* prevent further unreachable warnings for the caller */
4489 insn->sym->warned = 1;
4490 return true;
4491 }
4492
4493 return false;
4494}
4495
baa41469 4496static int validate_reachable_instructions(struct objtool_file *file)
dcc914f4 4497{
fedb724c
JP		 4498 struct instruction *insn, *prev_insn;
4499 struct symbol *call_dest;
5e3992fe 4500 int warnings = 0;
baa41469
JP		 4501
4502 if (file->ignore_unreachables)
4503 return 0;
dcc914f4
JP		 4504
4505 for_each_insn(file, insn) {
14db1f0a 4506 if (insn->visited || ignore_unreachable_insn(file, insn))
baa41469
JP		 4507 continue;
4508
fedb724c
JP		 4509 prev_insn = prev_insn_same_sec(file, insn);
4510 if (prev_insn && prev_insn->dead_end) {
4511 call_dest = insn_call_dest(prev_insn);
55eeab2a 4512 if (call_dest && !ignore_noreturn_call(prev_insn)) {
fedb724c
JP		 4513 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4514 call_dest->name);
4515 warnings++;
4516 continue;
4517 }
4518 }
4519
246b2c85 4520 WARN_INSN(insn, "unreachable instruction");
5e3992fe 4521 warnings++;
dcc914f4
JP		 4522 }
4523
5e3992fe 4524 return warnings;
dcc914f4
JP		 4525}
4526
ca653464
JP		 4527/* 'funcs' is a space-separated list of function names */
4528static int disas_funcs(const char *funcs)
4529{
4530 const char *objdump_str, *cross_compile;
4531 int size, ret;
4532 char *cmd;
4533
4534 cross_compile = getenv("CROSS_COMPILE");
4535
4536 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4537 "BEGIN { split(_funcs, funcs); }"
4538 "/^$/ { func_match = 0; }"
4539 "/<.*>:/ { "
4540 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4541 "for (i in funcs) {"
4542 "if (funcs[i] == f) {"
4543 "func_match = 1;"
4544 "base = strtonum(\"0x\" $1);"
4545 "break;"
4546 "}"
4547 "}"
4548 "}"
4549 "{"
4550 "if (func_match) {"
4551 "addr = strtonum(\"0x\" $1);"
4552 "printf(\"%%04x \", addr - base);"
4553 "print;"
4554 "}"
4555 "}' 1>&2";
4556
4557 /* fake snprintf() to calculate the size */
4558 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4559 if (size <= 0) {
4560 WARN("objdump string size calculation failed");
4561 return -1;
4562 }
4563
4564 cmd = malloc(size);
4565
4566 /* real snprintf() */
4567 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4568 ret = system(cmd);
4569 if (ret) {
4570 WARN("disassembly failed: %d", ret);
4571 return -1;
4572 }
4573
4574 return 0;
4575}
4576
4577static int disas_warned_funcs(struct objtool_file *file)
4578{
4579 struct symbol *sym;
4580 char *funcs = NULL, *tmp;
4581
4582 for_each_sym(file, sym) {
4583 if (sym->warned) {
4584 if (!funcs) {
4585 funcs = malloc(strlen(sym->name) + 1);
4586 strcpy(funcs, sym->name);
4587 } else {
4588 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4589 sprintf(tmp, "%s %s", funcs, sym->name);
4590 free(funcs);
4591 funcs = tmp;
4592 }
4593 }
4594 }
4595
4596 if (funcs)
4597 disas_funcs(funcs);
4598
4599 return 0;
4600}
4601
d44becb9 4602int check(struct objtool_file *file)
dcc914f4 4603{
dcc914f4
JP		 4604 int ret, warnings = 0;
4605
baa41469 4606 arch_initial_func_cfi_state(&initial_func_cfi);
8b946cc3
PZ		 4607 init_cfi_state(&init_cfi);
4608 init_cfi_state(&func_cfi);
4609 set_func_state(&func_cfi);
4610
4611 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4612 goto out;
4613
4614 cfi_hash_add(&init_cfi);
4615 cfi_hash_add(&func_cfi);
baa41469 4616
6545eb03 4617 ret = decode_sections(file);
dcc914f4
JP		 4618 if (ret < 0)
4619 goto out;
8b946cc3 4620
dcc914f4
JP		 4621 warnings += ret;
4622
1c34496e 4623 if (!nr_insns)
dcc914f4 4624 goto out;
dcc914f4 4625
2daf7fab 4626 if (opts.retpoline) {
6545eb03 4627 ret = validate_retpoline(file);
b5bc2231
PZ		 4628 if (ret < 0)
4629 return ret;
4630 warnings += ret;
4631 }
4632
c2bdd61c 4633 if (opts.stackval || opts.orc || opts.uaccess) {
7dce6204
JP		 4634 ret = validate_functions(file);
4635 if (ret < 0)
4636 goto out;
4637 warnings += ret;
39358a03 4638
7dce6204 4639 ret = validate_unwind_hints(file, NULL);
08f87a93
PZ		 4640 if (ret < 0)
4641 goto out;
4642 warnings += ret;
7dce6204
JP		 4643
4644 if (!warnings) {
4645 ret = validate_reachable_instructions(file);
4646 if (ret < 0)
4647 goto out;
4648 warnings += ret;
4649 }
753da417
JP		 4650
4651 } else if (opts.noinstr) {
4652 ret = validate_noinstr_sections(file);
4653 if (ret < 0)
4654 goto out;
4655 warnings += ret;
08f87a93
PZ		 4656 }
4657
a09a6e23
PZ		 4658 if (opts.unret) {
4659 /*
4660 * Must be after validate_branch() and friends, it plays
4661 * further games with insn->visited.
4662 */
4708ea14 4663 ret = validate_unrets(file);
a09a6e23
PZ		 4664 if (ret < 0)
4665 return ret;
4666 warnings += ret;
4667 }
4668
7dce6204
JP		 4669 if (opts.ibt) {
4670 ret = validate_ibt(file);
baa41469
JP		 4671 if (ret < 0)
4672 goto out;
c2bdd61c
JP		 4673 warnings += ret;
4674 }
4675
4676 if (opts.sls) {
4677 ret = validate_sls(file);
4678 if (ret < 0)
4679 goto out;
baa41469
JP		 4680 warnings += ret;
4681 }
4682
26e17689
JP		 4683 if (opts.static_call) {
4684 ret = create_static_call_sections(file);
4685 if (ret < 0)
4686 goto out;
4687 warnings += ret;
4688 }
1e7e4788 4689
2daf7fab 4690 if (opts.retpoline) {
134ab5bd
PZ		 4691 ret = create_retpoline_sites_sections(file);
4692 if (ret < 0)
4693 goto out;
4694 warnings += ret;
4695 }
4696
9a479f76
PZ		 4697 if (opts.cfi) {
4698 ret = create_cfi_sections(file);
4699 if (ret < 0)
4700 goto out;
4701 warnings += ret;
4702 }
4703
f43b9876 4704 if (opts.rethunk) {
d9e9d230
PZ		 4705 ret = create_return_sites_sections(file);
4706 if (ret < 0)
4707 goto out;
4708 warnings += ret;
00abd384 4709
0c0a6d89
PZ		 4710 if (opts.hack_skylake) {
4711 ret = create_direct_call_sections(file);
4712 if (ret < 0)
4713 goto out;
4714 warnings += ret;
4715 }
134ab5bd
PZ		 4716 }
4717
2daf7fab 4718 if (opts.mcount) {
99d00215
PZ		 4719 ret = create_mcount_loc_sections(file);
4720 if (ret < 0)
4721 goto out;
4722 warnings += ret;
4723 }
4724
bd456a1b
JP		 4725 if (opts.prefix) {
4726 ret = add_prefix_symbols(file);
4727 if (ret < 0)
4728 return ret;
4729 warnings += ret;
4730 }
4731
2daf7fab 4732 if (opts.ibt) {
89bc853e
PZ		 4733 ret = create_ibt_endbr_seal_sections(file);
4734 if (ret < 0)
4735 goto out;
4736 warnings += ret;
4737 }
4738
1c34496e 4739 if (opts.orc && nr_insns) {
b51277eb
JP		 4740 ret = orc_create(file);
4741 if (ret < 0)
4742 goto out;
4743 warnings += ret;
4744 }
4745
ca653464
JP		 4746 if (opts.verbose)
4747 disas_warned_funcs(file);
b51277eb 4748
2daf7fab 4749 if (opts.stats) {
8b946cc3
PZ		 4750 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4751 printf("nr_cfi: %ld\n", nr_cfi);
4752 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4753 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4754 }
4755
dcc914f4 4756out:
655cf865
JP		 4757 /*
4758 * For now, don't fail the kernel build on fatal warnings. These
4759 * errors are still fairly common due to the growing matrix of
4760 * supported toolchains and their recent pace of change.
4761 */
dcc914f4
JP		 4762 return 0;
4763}
A mirror of Linus' kernel repository