]> git.ipfire.org Git - thirdparty/linux.git/blob - tools/lib/bpf/libbpf.c
projects / thirdparty / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'drm/tegra/for-5.7-fixes' of git://anongit.freedesktop.org/tegra/linux...
[thirdparty/linux.git] / tools / lib / bpf / libbpf.c
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3 /*
4 * Common eBPF ELF object loading operations.
5 *
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
9 * Copyright (C) 2017 Nicira, Inc.
10 * Copyright (C) 2019 Isovalent, Inc.
11 */
12
13 #ifndef _GNU_SOURCE
14 #define _GNU_SOURCE
15 #endif
16 #include <stdlib.h>
17 #include <stdio.h>
18 #include <stdarg.h>
19 #include <libgen.h>
20 #include <inttypes.h>
21 #include <limits.h>
22 #include <string.h>
23 #include <unistd.h>
24 #include <endian.h>
25 #include <fcntl.h>
26 #include <errno.h>
27 #include <ctype.h>
28 #include <asm/unistd.h>
29 #include <linux/err.h>
30 #include <linux/kernel.h>
31 #include <linux/bpf.h>
32 #include <linux/btf.h>
33 #include <linux/filter.h>
34 #include <linux/list.h>
35 #include <linux/limits.h>
36 #include <linux/perf_event.h>
37 #include <linux/ring_buffer.h>
38 #include <linux/version.h>
39 #include <sys/epoll.h>
40 #include <sys/ioctl.h>
41 #include <sys/mman.h>
42 #include <sys/stat.h>
43 #include <sys/types.h>
44 #include <sys/vfs.h>
45 #include <sys/utsname.h>
46 #include <sys/resource.h>
47 #include <tools/libc_compat.h>
48 #include <libelf.h>
49 #include <gelf.h>
50 #include <zlib.h>
51
52 #include "libbpf.h"
53 #include "bpf.h"
54 #include "btf.h"
55 #include "str_error.h"
56 #include "libbpf_internal.h"
57 #include "hashmap.h"
58
59 /* make sure libbpf doesn't use kernel-only integer typedefs */
60 #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64
61
62 #ifndef EM_BPF
63 #define EM_BPF 247
64 #endif
65
66 #ifndef BPF_FS_MAGIC
67 #define BPF_FS_MAGIC 0xcafe4a11
68 #endif
69
70 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
71 * compilation if user enables corresponding warning. Disable it explicitly.
72 */
73 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
74
75 #define __printf(a, b) __attribute__((format(printf, a, b)))
76
77 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
78 static struct bpf_program *bpf_object__find_prog_by_idx(struct bpf_object *obj,
79 int idx);
80 static const struct btf_type *
81 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
82
83 static int __base_pr(enum libbpf_print_level level, const char *format,
84 va_list args)
85 {
86 if (level == LIBBPF_DEBUG)
87 return 0;
88
89 return vfprintf(stderr, format, args);
90 }
91
92 static libbpf_print_fn_t __libbpf_pr = __base_pr;
93
94 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
95 {
96 libbpf_print_fn_t old_print_fn = __libbpf_pr;
97
98 __libbpf_pr = fn;
99 return old_print_fn;
100 }
101
102 __printf(2, 3)
103 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
104 {
105 va_list args;
106
107 if (!__libbpf_pr)
108 return;
109
110 va_start(args, format);
111 __libbpf_pr(level, format, args);
112 va_end(args);
113 }
114
115 static void pr_perm_msg(int err)
116 {
117 struct rlimit limit;
118 char buf[100];
119
120 if (err != -EPERM || geteuid() != 0)
121 return;
122
123 err = getrlimit(RLIMIT_MEMLOCK, &limit);
124 if (err)
125 return;
126
127 if (limit.rlim_cur == RLIM_INFINITY)
128 return;
129
130 if (limit.rlim_cur < 1024)
131 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
132 else if (limit.rlim_cur < 1024*1024)
133 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
134 else
135 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
136
137 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
138 buf);
139 }
140
141 #define STRERR_BUFSIZE 128
142
143 /* Copied from tools/perf/util/util.h */
144 #ifndef zfree
145 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
146 #endif
147
148 #ifndef zclose
149 # define zclose(fd) ({ \
150 int ___err = 0; \
151 if ((fd) >= 0) \
152 ___err = close((fd)); \
153 fd = -1; \
154 ___err; })
155 #endif
156
157 #ifdef HAVE_LIBELF_MMAP_SUPPORT
158 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
159 #else
160 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
161 #endif
162
163 static inline __u64 ptr_to_u64(const void *ptr)
164 {
165 return (__u64) (unsigned long) ptr;
166 }
167
168 struct bpf_capabilities {
169 /* v4.14: kernel support for program & map names. */
170 __u32 name:1;
171 /* v5.2: kernel support for global data sections. */
172 __u32 global_data:1;
173 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
174 __u32 btf_func:1;
175 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
176 __u32 btf_datasec:1;
177 /* BPF_F_MMAPABLE is supported for arrays */
178 __u32 array_mmap:1;
179 /* BTF_FUNC_GLOBAL is supported */
180 __u32 btf_func_global:1;
181 /* kernel support for expected_attach_type in BPF_PROG_LOAD */
182 __u32 exp_attach_type:1;
183 };
184
185 enum reloc_type {
186 RELO_LD64,
187 RELO_CALL,
188 RELO_DATA,
189 RELO_EXTERN,
190 };
191
192 struct reloc_desc {
193 enum reloc_type type;
194 int insn_idx;
195 int map_idx;
196 int sym_off;
197 };
198
199 struct bpf_sec_def;
200
201 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
202 struct bpf_program *prog);
203
204 struct bpf_sec_def {
205 const char *sec;
206 size_t len;
207 enum bpf_prog_type prog_type;
208 enum bpf_attach_type expected_attach_type;
209 bool is_exp_attach_type_optional;
210 bool is_attachable;
211 bool is_attach_btf;
212 attach_fn_t attach_fn;
213 };
214
215 /*
216 * bpf_prog should be a better name but it has been used in
217 * linux/filter.h.
218 */
219 struct bpf_program {
220 /* Index in elf obj file, for relocation use. */
221 int idx;
222 char *name;
223 int prog_ifindex;
224 char *section_name;
225 const struct bpf_sec_def *sec_def;
226 /* section_name with / replaced by _; makes recursive pinning
227 * in bpf_object__pin_programs easier
228 */
229 char *pin_name;
230 struct bpf_insn *insns;
231 size_t insns_cnt, main_prog_cnt;
232 enum bpf_prog_type type;
233
234 struct reloc_desc *reloc_desc;
235 int nr_reloc;
236 int log_level;
237
238 struct {
239 int nr;
240 int *fds;
241 } instances;
242 bpf_program_prep_t preprocessor;
243
244 struct bpf_object *obj;
245 void *priv;
246 bpf_program_clear_priv_t clear_priv;
247
248 enum bpf_attach_type expected_attach_type;
249 __u32 attach_btf_id;
250 __u32 attach_prog_fd;
251 void *func_info;
252 __u32 func_info_rec_size;
253 __u32 func_info_cnt;
254
255 struct bpf_capabilities *caps;
256
257 void *line_info;
258 __u32 line_info_rec_size;
259 __u32 line_info_cnt;
260 __u32 prog_flags;
261 };
262
263 struct bpf_struct_ops {
264 const char *tname;
265 const struct btf_type *type;
266 struct bpf_program **progs;
267 __u32 *kern_func_off;
268 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
269 void *data;
270 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
271 * btf_vmlinux's format.
272 * struct bpf_struct_ops_tcp_congestion_ops {
273 * [... some other kernel fields ...]
274 * struct tcp_congestion_ops data;
275 * }
276 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
277 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
278 * from "data".
279 */
280 void *kern_vdata;
281 __u32 type_id;
282 };
283
284 #define DATA_SEC ".data"
285 #define BSS_SEC ".bss"
286 #define RODATA_SEC ".rodata"
287 #define KCONFIG_SEC ".kconfig"
288 #define STRUCT_OPS_SEC ".struct_ops"
289
290 enum libbpf_map_type {
291 LIBBPF_MAP_UNSPEC,
292 LIBBPF_MAP_DATA,
293 LIBBPF_MAP_BSS,
294 LIBBPF_MAP_RODATA,
295 LIBBPF_MAP_KCONFIG,
296 };
297
298 static const char * const libbpf_type_to_btf_name[] = {
299 [LIBBPF_MAP_DATA] = DATA_SEC,
300 [LIBBPF_MAP_BSS] = BSS_SEC,
301 [LIBBPF_MAP_RODATA] = RODATA_SEC,
302 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC,
303 };
304
305 struct bpf_map {
306 char *name;
307 int fd;
308 int sec_idx;
309 size_t sec_offset;
310 int map_ifindex;
311 int inner_map_fd;
312 struct bpf_map_def def;
313 __u32 btf_key_type_id;
314 __u32 btf_value_type_id;
315 __u32 btf_vmlinux_value_type_id;
316 void *priv;
317 bpf_map_clear_priv_t clear_priv;
318 enum libbpf_map_type libbpf_type;
319 void *mmaped;
320 struct bpf_struct_ops *st_ops;
321 char *pin_path;
322 bool pinned;
323 bool reused;
324 };
325
326 enum extern_type {
327 EXT_UNKNOWN,
328 EXT_CHAR,
329 EXT_BOOL,
330 EXT_INT,
331 EXT_TRISTATE,
332 EXT_CHAR_ARR,
333 };
334
335 struct extern_desc {
336 const char *name;
337 int sym_idx;
338 int btf_id;
339 enum extern_type type;
340 int sz;
341 int align;
342 int data_off;
343 bool is_signed;
344 bool is_weak;
345 bool is_set;
346 };
347
348 static LIST_HEAD(bpf_objects_list);
349
350 struct bpf_object {
351 char name[BPF_OBJ_NAME_LEN];
352 char license[64];
353 __u32 kern_version;
354
355 struct bpf_program *programs;
356 size_t nr_programs;
357 struct bpf_map *maps;
358 size_t nr_maps;
359 size_t maps_cap;
360
361 char *kconfig;
362 struct extern_desc *externs;
363 int nr_extern;
364 int kconfig_map_idx;
365
366 bool loaded;
367 bool has_pseudo_calls;
368
369 /*
370 * Information when doing elf related work. Only valid if fd
371 * is valid.
372 */
373 struct {
374 int fd;
375 const void *obj_buf;
376 size_t obj_buf_sz;
377 Elf *elf;
378 GElf_Ehdr ehdr;
379 Elf_Data *symbols;
380 Elf_Data *data;
381 Elf_Data *rodata;
382 Elf_Data *bss;
383 Elf_Data *st_ops_data;
384 size_t strtabidx;
385 struct {
386 GElf_Shdr shdr;
387 Elf_Data *data;
388 } *reloc_sects;
389 int nr_reloc_sects;
390 int maps_shndx;
391 int btf_maps_shndx;
392 int text_shndx;
393 int symbols_shndx;
394 int data_shndx;
395 int rodata_shndx;
396 int bss_shndx;
397 int st_ops_shndx;
398 } efile;
399 /*
400 * All loaded bpf_object is linked in a list, which is
401 * hidden to caller. bpf_objects__<func> handlers deal with
402 * all objects.
403 */
404 struct list_head list;
405
406 struct btf *btf;
407 /* Parse and load BTF vmlinux if any of the programs in the object need
408 * it at load time.
409 */
410 struct btf *btf_vmlinux;
411 struct btf_ext *btf_ext;
412
413 void *priv;
414 bpf_object_clear_priv_t clear_priv;
415
416 struct bpf_capabilities caps;
417
418 char path[];
419 };
420 #define obj_elf_valid(o) ((o)->efile.elf)
421
422 void bpf_program__unload(struct bpf_program *prog)
423 {
424 int i;
425
426 if (!prog)
427 return;
428
429 /*
430 * If the object is opened but the program was never loaded,
431 * it is possible that prog->instances.nr == -1.
432 */
433 if (prog->instances.nr > 0) {
434 for (i = 0; i < prog->instances.nr; i++)
435 zclose(prog->instances.fds[i]);
436 } else if (prog->instances.nr != -1) {
437 pr_warn("Internal error: instances.nr is %d\n",
438 prog->instances.nr);
439 }
440
441 prog->instances.nr = -1;
442 zfree(&prog->instances.fds);
443
444 zfree(&prog->func_info);
445 zfree(&prog->line_info);
446 }
447
448 static void bpf_program__exit(struct bpf_program *prog)
449 {
450 if (!prog)
451 return;
452
453 if (prog->clear_priv)
454 prog->clear_priv(prog, prog->priv);
455
456 prog->priv = NULL;
457 prog->clear_priv = NULL;
458
459 bpf_program__unload(prog);
460 zfree(&prog->name);
461 zfree(&prog->section_name);
462 zfree(&prog->pin_name);
463 zfree(&prog->insns);
464 zfree(&prog->reloc_desc);
465
466 prog->nr_reloc = 0;
467 prog->insns_cnt = 0;
468 prog->idx = -1;
469 }
470
471 static char *__bpf_program__pin_name(struct bpf_program *prog)
472 {
473 char *name, *p;
474
475 name = p = strdup(prog->section_name);
476 while ((p = strchr(p, '/')))
477 *p = '_';
478
479 return name;
480 }
481
482 static int
483 bpf_program__init(void *data, size_t size, char *section_name, int idx,
484 struct bpf_program *prog)
485 {
486 const size_t bpf_insn_sz = sizeof(struct bpf_insn);
487
488 if (size == 0 || size % bpf_insn_sz) {
489 pr_warn("corrupted section '%s', size: %zu\n",
490 section_name, size);
491 return -EINVAL;
492 }
493
494 memset(prog, 0, sizeof(*prog));
495
496 prog->section_name = strdup(section_name);
497 if (!prog->section_name) {
498 pr_warn("failed to alloc name for prog under section(%d) %s\n",
499 idx, section_name);
500 goto errout;
501 }
502
503 prog->pin_name = __bpf_program__pin_name(prog);
504 if (!prog->pin_name) {
505 pr_warn("failed to alloc pin name for prog under section(%d) %s\n",
506 idx, section_name);
507 goto errout;
508 }
509
510 prog->insns = malloc(size);
511 if (!prog->insns) {
512 pr_warn("failed to alloc insns for prog under section %s\n",
513 section_name);
514 goto errout;
515 }
516 prog->insns_cnt = size / bpf_insn_sz;
517 memcpy(prog->insns, data, size);
518 prog->idx = idx;
519 prog->instances.fds = NULL;
520 prog->instances.nr = -1;
521 prog->type = BPF_PROG_TYPE_UNSPEC;
522
523 return 0;
524 errout:
525 bpf_program__exit(prog);
526 return -ENOMEM;
527 }
528
529 static int
530 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
531 char *section_name, int idx)
532 {
533 struct bpf_program prog, *progs;
534 int nr_progs, err;
535
536 err = bpf_program__init(data, size, section_name, idx, &prog);
537 if (err)
538 return err;
539
540 prog.caps = &obj->caps;
541 progs = obj->programs;
542 nr_progs = obj->nr_programs;
543
544 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
545 if (!progs) {
546 /*
547 * In this case the original obj->programs
548 * is still valid, so don't need special treat for
549 * bpf_close_object().
550 */
551 pr_warn("failed to alloc a new program under section '%s'\n",
552 section_name);
553 bpf_program__exit(&prog);
554 return -ENOMEM;
555 }
556
557 pr_debug("found program %s\n", prog.section_name);
558 obj->programs = progs;
559 obj->nr_programs = nr_progs + 1;
560 prog.obj = obj;
561 progs[nr_progs] = prog;
562 return 0;
563 }
564
565 static int
566 bpf_object__init_prog_names(struct bpf_object *obj)
567 {
568 Elf_Data *symbols = obj->efile.symbols;
569 struct bpf_program *prog;
570 size_t pi, si;
571
572 for (pi = 0; pi < obj->nr_programs; pi++) {
573 const char *name = NULL;
574
575 prog = &obj->programs[pi];
576
577 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
578 si++) {
579 GElf_Sym sym;
580
581 if (!gelf_getsym(symbols, si, &sym))
582 continue;
583 if (sym.st_shndx != prog->idx)
584 continue;
585 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
586 continue;
587
588 name = elf_strptr(obj->efile.elf,
589 obj->efile.strtabidx,
590 sym.st_name);
591 if (!name) {
592 pr_warn("failed to get sym name string for prog %s\n",
593 prog->section_name);
594 return -LIBBPF_ERRNO__LIBELF;
595 }
596 }
597
598 if (!name && prog->idx == obj->efile.text_shndx)
599 name = ".text";
600
601 if (!name) {
602 pr_warn("failed to find sym for prog %s\n",
603 prog->section_name);
604 return -EINVAL;
605 }
606
607 prog->name = strdup(name);
608 if (!prog->name) {
609 pr_warn("failed to allocate memory for prog sym %s\n",
610 name);
611 return -ENOMEM;
612 }
613 }
614
615 return 0;
616 }
617
618 static __u32 get_kernel_version(void)
619 {
620 __u32 major, minor, patch;
621 struct utsname info;
622
623 uname(&info);
624 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
625 return 0;
626 return KERNEL_VERSION(major, minor, patch);
627 }
628
629 static const struct btf_member *
630 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
631 {
632 struct btf_member *m;
633 int i;
634
635 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
636 if (btf_member_bit_offset(t, i) == bit_offset)
637 return m;
638 }
639
640 return NULL;
641 }
642
643 static const struct btf_member *
644 find_member_by_name(const struct btf *btf, const struct btf_type *t,
645 const char *name)
646 {
647 struct btf_member *m;
648 int i;
649
650 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
651 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
652 return m;
653 }
654
655 return NULL;
656 }
657
658 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
659 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
660 const char *name, __u32 kind);
661
662 static int
663 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
664 const struct btf_type **type, __u32 *type_id,
665 const struct btf_type **vtype, __u32 *vtype_id,
666 const struct btf_member **data_member)
667 {
668 const struct btf_type *kern_type, *kern_vtype;
669 const struct btf_member *kern_data_member;
670 __s32 kern_vtype_id, kern_type_id;
671 __u32 i;
672
673 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
674 if (kern_type_id < 0) {
675 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
676 tname);
677 return kern_type_id;
678 }
679 kern_type = btf__type_by_id(btf, kern_type_id);
680
681 /* Find the corresponding "map_value" type that will be used
682 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example,
683 * find "struct bpf_struct_ops_tcp_congestion_ops" from the
684 * btf_vmlinux.
685 */
686 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
687 tname, BTF_KIND_STRUCT);
688 if (kern_vtype_id < 0) {
689 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
690 STRUCT_OPS_VALUE_PREFIX, tname);
691 return kern_vtype_id;
692 }
693 kern_vtype = btf__type_by_id(btf, kern_vtype_id);
694
695 /* Find "struct tcp_congestion_ops" from
696 * struct bpf_struct_ops_tcp_congestion_ops {
697 * [ ... ]
698 * struct tcp_congestion_ops data;
699 * }
700 */
701 kern_data_member = btf_members(kern_vtype);
702 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
703 if (kern_data_member->type == kern_type_id)
704 break;
705 }
706 if (i == btf_vlen(kern_vtype)) {
707 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
708 tname, STRUCT_OPS_VALUE_PREFIX, tname);
709 return -EINVAL;
710 }
711
712 *type = kern_type;
713 *type_id = kern_type_id;
714 *vtype = kern_vtype;
715 *vtype_id = kern_vtype_id;
716 *data_member = kern_data_member;
717
718 return 0;
719 }
720
721 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
722 {
723 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
724 }
725
726 /* Init the map's fields that depend on kern_btf */
727 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
728 const struct btf *btf,
729 const struct btf *kern_btf)
730 {
731 const struct btf_member *member, *kern_member, *kern_data_member;
732 const struct btf_type *type, *kern_type, *kern_vtype;
733 __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
734 struct bpf_struct_ops *st_ops;
735 void *data, *kern_data;
736 const char *tname;
737 int err;
738
739 st_ops = map->st_ops;
740 type = st_ops->type;
741 tname = st_ops->tname;
742 err = find_struct_ops_kern_types(kern_btf, tname,
743 &kern_type, &kern_type_id,
744 &kern_vtype, &kern_vtype_id,
745 &kern_data_member);
746 if (err)
747 return err;
748
749 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
750 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
751
752 map->def.value_size = kern_vtype->size;
753 map->btf_vmlinux_value_type_id = kern_vtype_id;
754
755 st_ops->kern_vdata = calloc(1, kern_vtype->size);
756 if (!st_ops->kern_vdata)
757 return -ENOMEM;
758
759 data = st_ops->data;
760 kern_data_off = kern_data_member->offset / 8;
761 kern_data = st_ops->kern_vdata + kern_data_off;
762
763 member = btf_members(type);
764 for (i = 0; i < btf_vlen(type); i++, member++) {
765 const struct btf_type *mtype, *kern_mtype;
766 __u32 mtype_id, kern_mtype_id;
767 void *mdata, *kern_mdata;
768 __s64 msize, kern_msize;
769 __u32 moff, kern_moff;
770 __u32 kern_member_idx;
771 const char *mname;
772
773 mname = btf__name_by_offset(btf, member->name_off);
774 kern_member = find_member_by_name(kern_btf, kern_type, mname);
775 if (!kern_member) {
776 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
777 map->name, mname);
778 return -ENOTSUP;
779 }
780
781 kern_member_idx = kern_member - btf_members(kern_type);
782 if (btf_member_bitfield_size(type, i) ||
783 btf_member_bitfield_size(kern_type, kern_member_idx)) {
784 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
785 map->name, mname);
786 return -ENOTSUP;
787 }
788
789 moff = member->offset / 8;
790 kern_moff = kern_member->offset / 8;
791
792 mdata = data + moff;
793 kern_mdata = kern_data + kern_moff;
794
795 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
796 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
797 &kern_mtype_id);
798 if (BTF_INFO_KIND(mtype->info) !=
799 BTF_INFO_KIND(kern_mtype->info)) {
800 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
801 map->name, mname, BTF_INFO_KIND(mtype->info),
802 BTF_INFO_KIND(kern_mtype->info));
803 return -ENOTSUP;
804 }
805
806 if (btf_is_ptr(mtype)) {
807 struct bpf_program *prog;
808
809 mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
810 kern_mtype = skip_mods_and_typedefs(kern_btf,
811 kern_mtype->type,
812 &kern_mtype_id);
813 if (!btf_is_func_proto(mtype) ||
814 !btf_is_func_proto(kern_mtype)) {
815 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
816 map->name, mname);
817 return -ENOTSUP;
818 }
819
820 prog = st_ops->progs[i];
821 if (!prog) {
822 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
823 map->name, mname);
824 continue;
825 }
826
827 prog->attach_btf_id = kern_type_id;
828 prog->expected_attach_type = kern_member_idx;
829
830 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
831
832 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
833 map->name, mname, prog->name, moff,
834 kern_moff);
835
836 continue;
837 }
838
839 msize = btf__resolve_size(btf, mtype_id);
840 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
841 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
842 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
843 map->name, mname, (ssize_t)msize,
844 (ssize_t)kern_msize);
845 return -ENOTSUP;
846 }
847
848 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
849 map->name, mname, (unsigned int)msize,
850 moff, kern_moff);
851 memcpy(kern_mdata, mdata, msize);
852 }
853
854 return 0;
855 }
856
857 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
858 {
859 struct bpf_map *map;
860 size_t i;
861 int err;
862
863 for (i = 0; i < obj->nr_maps; i++) {
864 map = &obj->maps[i];
865
866 if (!bpf_map__is_struct_ops(map))
867 continue;
868
869 err = bpf_map__init_kern_struct_ops(map, obj->btf,
870 obj->btf_vmlinux);
871 if (err)
872 return err;
873 }
874
875 return 0;
876 }
877
878 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
879 {
880 const struct btf_type *type, *datasec;
881 const struct btf_var_secinfo *vsi;
882 struct bpf_struct_ops *st_ops;
883 const char *tname, *var_name;
884 __s32 type_id, datasec_id;
885 const struct btf *btf;
886 struct bpf_map *map;
887 __u32 i;
888
889 if (obj->efile.st_ops_shndx == -1)
890 return 0;
891
892 btf = obj->btf;
893 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
894 BTF_KIND_DATASEC);
895 if (datasec_id < 0) {
896 pr_warn("struct_ops init: DATASEC %s not found\n",
897 STRUCT_OPS_SEC);
898 return -EINVAL;
899 }
900
901 datasec = btf__type_by_id(btf, datasec_id);
902 vsi = btf_var_secinfos(datasec);
903 for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
904 type = btf__type_by_id(obj->btf, vsi->type);
905 var_name = btf__name_by_offset(obj->btf, type->name_off);
906
907 type_id = btf__resolve_type(obj->btf, vsi->type);
908 if (type_id < 0) {
909 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
910 vsi->type, STRUCT_OPS_SEC);
911 return -EINVAL;
912 }
913
914 type = btf__type_by_id(obj->btf, type_id);
915 tname = btf__name_by_offset(obj->btf, type->name_off);
916 if (!tname[0]) {
917 pr_warn("struct_ops init: anonymous type is not supported\n");
918 return -ENOTSUP;
919 }
920 if (!btf_is_struct(type)) {
921 pr_warn("struct_ops init: %s is not a struct\n", tname);
922 return -EINVAL;
923 }
924
925 map = bpf_object__add_map(obj);
926 if (IS_ERR(map))
927 return PTR_ERR(map);
928
929 map->sec_idx = obj->efile.st_ops_shndx;
930 map->sec_offset = vsi->offset;
931 map->name = strdup(var_name);
932 if (!map->name)
933 return -ENOMEM;
934
935 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
936 map->def.key_size = sizeof(int);
937 map->def.value_size = type->size;
938 map->def.max_entries = 1;
939
940 map->st_ops = calloc(1, sizeof(*map->st_ops));
941 if (!map->st_ops)
942 return -ENOMEM;
943 st_ops = map->st_ops;
944 st_ops->data = malloc(type->size);
945 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
946 st_ops->kern_func_off = malloc(btf_vlen(type) *
947 sizeof(*st_ops->kern_func_off));
948 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
949 return -ENOMEM;
950
951 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
952 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
953 var_name, STRUCT_OPS_SEC);
954 return -EINVAL;
955 }
956
957 memcpy(st_ops->data,
958 obj->efile.st_ops_data->d_buf + vsi->offset,
959 type->size);
960 st_ops->tname = tname;
961 st_ops->type = type;
962 st_ops->type_id = type_id;
963
964 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
965 tname, type_id, var_name, vsi->offset);
966 }
967
968 return 0;
969 }
970
971 static struct bpf_object *bpf_object__new(const char *path,
972 const void *obj_buf,
973 size_t obj_buf_sz,
974 const char *obj_name)
975 {
976 struct bpf_object *obj;
977 char *end;
978
979 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
980 if (!obj) {
981 pr_warn("alloc memory failed for %s\n", path);
982 return ERR_PTR(-ENOMEM);
983 }
984
985 strcpy(obj->path, path);
986 if (obj_name) {
987 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
988 obj->name[sizeof(obj->name) - 1] = 0;
989 } else {
990 /* Using basename() GNU version which doesn't modify arg. */
991 strncpy(obj->name, basename((void *)path),
992 sizeof(obj->name) - 1);
993 end = strchr(obj->name, '.');
994 if (end)
995 *end = 0;
996 }
997
998 obj->efile.fd = -1;
999 /*
1000 * Caller of this function should also call
1001 * bpf_object__elf_finish() after data collection to return
1002 * obj_buf to user. If not, we should duplicate the buffer to
1003 * avoid user freeing them before elf finish.
1004 */
1005 obj->efile.obj_buf = obj_buf;
1006 obj->efile.obj_buf_sz = obj_buf_sz;
1007 obj->efile.maps_shndx = -1;
1008 obj->efile.btf_maps_shndx = -1;
1009 obj->efile.data_shndx = -1;
1010 obj->efile.rodata_shndx = -1;
1011 obj->efile.bss_shndx = -1;
1012 obj->efile.st_ops_shndx = -1;
1013 obj->kconfig_map_idx = -1;
1014
1015 obj->kern_version = get_kernel_version();
1016 obj->loaded = false;
1017
1018 INIT_LIST_HEAD(&obj->list);
1019 list_add(&obj->list, &bpf_objects_list);
1020 return obj;
1021 }
1022
1023 static void bpf_object__elf_finish(struct bpf_object *obj)
1024 {
1025 if (!obj_elf_valid(obj))
1026 return;
1027
1028 if (obj->efile.elf) {
1029 elf_end(obj->efile.elf);
1030 obj->efile.elf = NULL;
1031 }
1032 obj->efile.symbols = NULL;
1033 obj->efile.data = NULL;
1034 obj->efile.rodata = NULL;
1035 obj->efile.bss = NULL;
1036 obj->efile.st_ops_data = NULL;
1037
1038 zfree(&obj->efile.reloc_sects);
1039 obj->efile.nr_reloc_sects = 0;
1040 zclose(obj->efile.fd);
1041 obj->efile.obj_buf = NULL;
1042 obj->efile.obj_buf_sz = 0;
1043 }
1044
1045 static int bpf_object__elf_init(struct bpf_object *obj)
1046 {
1047 int err = 0;
1048 GElf_Ehdr *ep;
1049
1050 if (obj_elf_valid(obj)) {
1051 pr_warn("elf init: internal error\n");
1052 return -LIBBPF_ERRNO__LIBELF;
1053 }
1054
1055 if (obj->efile.obj_buf_sz > 0) {
1056 /*
1057 * obj_buf should have been validated by
1058 * bpf_object__open_buffer().
1059 */
1060 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1061 obj->efile.obj_buf_sz);
1062 } else {
1063 obj->efile.fd = open(obj->path, O_RDONLY);
1064 if (obj->efile.fd < 0) {
1065 char errmsg[STRERR_BUFSIZE], *cp;
1066
1067 err = -errno;
1068 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1069 pr_warn("failed to open %s: %s\n", obj->path, cp);
1070 return err;
1071 }
1072
1073 obj->efile.elf = elf_begin(obj->efile.fd,
1074 LIBBPF_ELF_C_READ_MMAP, NULL);
1075 }
1076
1077 if (!obj->efile.elf) {
1078 pr_warn("failed to open %s as ELF file\n", obj->path);
1079 err = -LIBBPF_ERRNO__LIBELF;
1080 goto errout;
1081 }
1082
1083 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1084 pr_warn("failed to get EHDR from %s\n", obj->path);
1085 err = -LIBBPF_ERRNO__FORMAT;
1086 goto errout;
1087 }
1088 ep = &obj->efile.ehdr;
1089
1090 /* Old LLVM set e_machine to EM_NONE */
1091 if (ep->e_type != ET_REL ||
1092 (ep->e_machine && ep->e_machine != EM_BPF)) {
1093 pr_warn("%s is not an eBPF object file\n", obj->path);
1094 err = -LIBBPF_ERRNO__FORMAT;
1095 goto errout;
1096 }
1097
1098 return 0;
1099 errout:
1100 bpf_object__elf_finish(obj);
1101 return err;
1102 }
1103
1104 static int bpf_object__check_endianness(struct bpf_object *obj)
1105 {
1106 #if __BYTE_ORDER == __LITTLE_ENDIAN
1107 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1108 return 0;
1109 #elif __BYTE_ORDER == __BIG_ENDIAN
1110 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1111 return 0;
1112 #else
1113 # error "Unrecognized __BYTE_ORDER__"
1114 #endif
1115 pr_warn("endianness mismatch.\n");
1116 return -LIBBPF_ERRNO__ENDIAN;
1117 }
1118
1119 static int
1120 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1121 {
1122 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1123 pr_debug("license of %s is %s\n", obj->path, obj->license);
1124 return 0;
1125 }
1126
1127 static int
1128 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1129 {
1130 __u32 kver;
1131
1132 if (size != sizeof(kver)) {
1133 pr_warn("invalid kver section in %s\n", obj->path);
1134 return -LIBBPF_ERRNO__FORMAT;
1135 }
1136 memcpy(&kver, data, sizeof(kver));
1137 obj->kern_version = kver;
1138 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1139 return 0;
1140 }
1141
1142 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1143 {
1144 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1145 type == BPF_MAP_TYPE_HASH_OF_MAPS)
1146 return true;
1147 return false;
1148 }
1149
1150 static int bpf_object_search_section_size(const struct bpf_object *obj,
1151 const char *name, size_t *d_size)
1152 {
1153 const GElf_Ehdr *ep = &obj->efile.ehdr;
1154 Elf *elf = obj->efile.elf;
1155 Elf_Scn *scn = NULL;
1156 int idx = 0;
1157
1158 while ((scn = elf_nextscn(elf, scn)) != NULL) {
1159 const char *sec_name;
1160 Elf_Data *data;
1161 GElf_Shdr sh;
1162
1163 idx++;
1164 if (gelf_getshdr(scn, &sh) != &sh) {
1165 pr_warn("failed to get section(%d) header from %s\n",
1166 idx, obj->path);
1167 return -EIO;
1168 }
1169
1170 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
1171 if (!sec_name) {
1172 pr_warn("failed to get section(%d) name from %s\n",
1173 idx, obj->path);
1174 return -EIO;
1175 }
1176
1177 if (strcmp(name, sec_name))
1178 continue;
1179
1180 data = elf_getdata(scn, 0);
1181 if (!data) {
1182 pr_warn("failed to get section(%d) data from %s(%s)\n",
1183 idx, name, obj->path);
1184 return -EIO;
1185 }
1186
1187 *d_size = data->d_size;
1188 return 0;
1189 }
1190
1191 return -ENOENT;
1192 }
1193
1194 int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1195 __u32 *size)
1196 {
1197 int ret = -ENOENT;
1198 size_t d_size;
1199
1200 *size = 0;
1201 if (!name) {
1202 return -EINVAL;
1203 } else if (!strcmp(name, DATA_SEC)) {
1204 if (obj->efile.data)
1205 *size = obj->efile.data->d_size;
1206 } else if (!strcmp(name, BSS_SEC)) {
1207 if (obj->efile.bss)
1208 *size = obj->efile.bss->d_size;
1209 } else if (!strcmp(name, RODATA_SEC)) {
1210 if (obj->efile.rodata)
1211 *size = obj->efile.rodata->d_size;
1212 } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1213 if (obj->efile.st_ops_data)
1214 *size = obj->efile.st_ops_data->d_size;
1215 } else {
1216 ret = bpf_object_search_section_size(obj, name, &d_size);
1217 if (!ret)
1218 *size = d_size;
1219 }
1220
1221 return *size ? 0 : ret;
1222 }
1223
1224 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1225 __u32 *off)
1226 {
1227 Elf_Data *symbols = obj->efile.symbols;
1228 const char *sname;
1229 size_t si;
1230
1231 if (!name || !off)
1232 return -EINVAL;
1233
1234 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1235 GElf_Sym sym;
1236
1237 if (!gelf_getsym(symbols, si, &sym))
1238 continue;
1239 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1240 GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1241 continue;
1242
1243 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1244 sym.st_name);
1245 if (!sname) {
1246 pr_warn("failed to get sym name string for var %s\n",
1247 name);
1248 return -EIO;
1249 }
1250 if (strcmp(name, sname) == 0) {
1251 *off = sym.st_value;
1252 return 0;
1253 }
1254 }
1255
1256 return -ENOENT;
1257 }
1258
1259 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1260 {
1261 struct bpf_map *new_maps;
1262 size_t new_cap;
1263 int i;
1264
1265 if (obj->nr_maps < obj->maps_cap)
1266 return &obj->maps[obj->nr_maps++];
1267
1268 new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1269 new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps));
1270 if (!new_maps) {
1271 pr_warn("alloc maps for object failed\n");
1272 return ERR_PTR(-ENOMEM);
1273 }
1274
1275 obj->maps_cap = new_cap;
1276 obj->maps = new_maps;
1277
1278 /* zero out new maps */
1279 memset(obj->maps + obj->nr_maps, 0,
1280 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1281 /*
1282 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
1283 * when failure (zclose won't close negative fd)).
1284 */
1285 for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1286 obj->maps[i].fd = -1;
1287 obj->maps[i].inner_map_fd = -1;
1288 }
1289
1290 return &obj->maps[obj->nr_maps++];
1291 }
1292
1293 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1294 {
1295 long page_sz = sysconf(_SC_PAGE_SIZE);
1296 size_t map_sz;
1297
1298 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1299 map_sz = roundup(map_sz, page_sz);
1300 return map_sz;
1301 }
1302
1303 static char *internal_map_name(struct bpf_object *obj,
1304 enum libbpf_map_type type)
1305 {
1306 char map_name[BPF_OBJ_NAME_LEN], *p;
1307 const char *sfx = libbpf_type_to_btf_name[type];
1308 int sfx_len = max((size_t)7, strlen(sfx));
1309 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1310 strlen(obj->name));
1311
1312 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1313 sfx_len, libbpf_type_to_btf_name[type]);
1314
1315 /* sanitise map name to characters allowed by kernel */
1316 for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1317 if (!isalnum(*p) && *p != '_' && *p != '.')
1318 *p = '_';
1319
1320 return strdup(map_name);
1321 }
1322
1323 static int
1324 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1325 int sec_idx, void *data, size_t data_sz)
1326 {
1327 struct bpf_map_def *def;
1328 struct bpf_map *map;
1329 int err;
1330
1331 map = bpf_object__add_map(obj);
1332 if (IS_ERR(map))
1333 return PTR_ERR(map);
1334
1335 map->libbpf_type = type;
1336 map->sec_idx = sec_idx;
1337 map->sec_offset = 0;
1338 map->name = internal_map_name(obj, type);
1339 if (!map->name) {
1340 pr_warn("failed to alloc map name\n");
1341 return -ENOMEM;
1342 }
1343
1344 def = &map->def;
1345 def->type = BPF_MAP_TYPE_ARRAY;
1346 def->key_size = sizeof(int);
1347 def->value_size = data_sz;
1348 def->max_entries = 1;
1349 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1350 ? BPF_F_RDONLY_PROG : 0;
1351 def->map_flags |= BPF_F_MMAPABLE;
1352
1353 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1354 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1355
1356 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1357 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1358 if (map->mmaped == MAP_FAILED) {
1359 err = -errno;
1360 map->mmaped = NULL;
1361 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1362 map->name, err);
1363 zfree(&map->name);
1364 return err;
1365 }
1366
1367 if (data)
1368 memcpy(map->mmaped, data, data_sz);
1369
1370 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1371 return 0;
1372 }
1373
1374 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1375 {
1376 int err;
1377
1378 /*
1379 * Populate obj->maps with libbpf internal maps.
1380 */
1381 if (obj->efile.data_shndx >= 0) {
1382 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1383 obj->efile.data_shndx,
1384 obj->efile.data->d_buf,
1385 obj->efile.data->d_size);
1386 if (err)
1387 return err;
1388 }
1389 if (obj->efile.rodata_shndx >= 0) {
1390 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1391 obj->efile.rodata_shndx,
1392 obj->efile.rodata->d_buf,
1393 obj->efile.rodata->d_size);
1394 if (err)
1395 return err;
1396 }
1397 if (obj->efile.bss_shndx >= 0) {
1398 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1399 obj->efile.bss_shndx,
1400 NULL,
1401 obj->efile.bss->d_size);
1402 if (err)
1403 return err;
1404 }
1405 return 0;
1406 }
1407
1408
1409 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1410 const void *name)
1411 {
1412 int i;
1413
1414 for (i = 0; i < obj->nr_extern; i++) {
1415 if (strcmp(obj->externs[i].name, name) == 0)
1416 return &obj->externs[i];
1417 }
1418 return NULL;
1419 }
1420
1421 static int set_ext_value_tri(struct extern_desc *ext, void *ext_val,
1422 char value)
1423 {
1424 switch (ext->type) {
1425 case EXT_BOOL:
1426 if (value == 'm') {
1427 pr_warn("extern %s=%c should be tristate or char\n",
1428 ext->name, value);
1429 return -EINVAL;
1430 }
1431 *(bool *)ext_val = value == 'y' ? true : false;
1432 break;
1433 case EXT_TRISTATE:
1434 if (value == 'y')
1435 *(enum libbpf_tristate *)ext_val = TRI_YES;
1436 else if (value == 'm')
1437 *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1438 else /* value == 'n' */
1439 *(enum libbpf_tristate *)ext_val = TRI_NO;
1440 break;
1441 case EXT_CHAR:
1442 *(char *)ext_val = value;
1443 break;
1444 case EXT_UNKNOWN:
1445 case EXT_INT:
1446 case EXT_CHAR_ARR:
1447 default:
1448 pr_warn("extern %s=%c should be bool, tristate, or char\n",
1449 ext->name, value);
1450 return -EINVAL;
1451 }
1452 ext->is_set = true;
1453 return 0;
1454 }
1455
1456 static int set_ext_value_str(struct extern_desc *ext, char *ext_val,
1457 const char *value)
1458 {
1459 size_t len;
1460
1461 if (ext->type != EXT_CHAR_ARR) {
1462 pr_warn("extern %s=%s should char array\n", ext->name, value);
1463 return -EINVAL;
1464 }
1465
1466 len = strlen(value);
1467 if (value[len - 1] != '"') {
1468 pr_warn("extern '%s': invalid string config '%s'\n",
1469 ext->name, value);
1470 return -EINVAL;
1471 }
1472
1473 /* strip quotes */
1474 len -= 2;
1475 if (len >= ext->sz) {
1476 pr_warn("extern '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1477 ext->name, value, len, ext->sz - 1);
1478 len = ext->sz - 1;
1479 }
1480 memcpy(ext_val, value + 1, len);
1481 ext_val[len] = '\0';
1482 ext->is_set = true;
1483 return 0;
1484 }
1485
1486 static int parse_u64(const char *value, __u64 *res)
1487 {
1488 char *value_end;
1489 int err;
1490
1491 errno = 0;
1492 *res = strtoull(value, &value_end, 0);
1493 if (errno) {
1494 err = -errno;
1495 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1496 return err;
1497 }
1498 if (*value_end) {
1499 pr_warn("failed to parse '%s' as integer completely\n", value);
1500 return -EINVAL;
1501 }
1502 return 0;
1503 }
1504
1505 static bool is_ext_value_in_range(const struct extern_desc *ext, __u64 v)
1506 {
1507 int bit_sz = ext->sz * 8;
1508
1509 if (ext->sz == 8)
1510 return true;
1511
1512 /* Validate that value stored in u64 fits in integer of `ext->sz`
1513 * bytes size without any loss of information. If the target integer
1514 * is signed, we rely on the following limits of integer type of
1515 * Y bits and subsequent transformation:
1516 *
1517 * -2^(Y-1) <= X <= 2^(Y-1) - 1
1518 * 0 <= X + 2^(Y-1) <= 2^Y - 1
1519 * 0 <= X + 2^(Y-1) < 2^Y
1520 *
1521 * For unsigned target integer, check that all the (64 - Y) bits are
1522 * zero.
1523 */
1524 if (ext->is_signed)
1525 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1526 else
1527 return (v >> bit_sz) == 0;
1528 }
1529
1530 static int set_ext_value_num(struct extern_desc *ext, void *ext_val,
1531 __u64 value)
1532 {
1533 if (ext->type != EXT_INT && ext->type != EXT_CHAR) {
1534 pr_warn("extern %s=%llu should be integer\n",
1535 ext->name, (unsigned long long)value);
1536 return -EINVAL;
1537 }
1538 if (!is_ext_value_in_range(ext, value)) {
1539 pr_warn("extern %s=%llu value doesn't fit in %d bytes\n",
1540 ext->name, (unsigned long long)value, ext->sz);
1541 return -ERANGE;
1542 }
1543 switch (ext->sz) {
1544 case 1: *(__u8 *)ext_val = value; break;
1545 case 2: *(__u16 *)ext_val = value; break;
1546 case 4: *(__u32 *)ext_val = value; break;
1547 case 8: *(__u64 *)ext_val = value; break;
1548 default:
1549 return -EINVAL;
1550 }
1551 ext->is_set = true;
1552 return 0;
1553 }
1554
1555 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1556 char *buf, void *data)
1557 {
1558 struct extern_desc *ext;
1559 char *sep, *value;
1560 int len, err = 0;
1561 void *ext_val;
1562 __u64 num;
1563
1564 if (strncmp(buf, "CONFIG_", 7))
1565 return 0;
1566
1567 sep = strchr(buf, '=');
1568 if (!sep) {
1569 pr_warn("failed to parse '%s': no separator\n", buf);
1570 return -EINVAL;
1571 }
1572
1573 /* Trim ending '\n' */
1574 len = strlen(buf);
1575 if (buf[len - 1] == '\n')
1576 buf[len - 1] = '\0';
1577 /* Split on '=' and ensure that a value is present. */
1578 *sep = '\0';
1579 if (!sep[1]) {
1580 *sep = '=';
1581 pr_warn("failed to parse '%s': no value\n", buf);
1582 return -EINVAL;
1583 }
1584
1585 ext = find_extern_by_name(obj, buf);
1586 if (!ext || ext->is_set)
1587 return 0;
1588
1589 ext_val = data + ext->data_off;
1590 value = sep + 1;
1591
1592 switch (*value) {
1593 case 'y': case 'n': case 'm':
1594 err = set_ext_value_tri(ext, ext_val, *value);
1595 break;
1596 case '"':
1597 err = set_ext_value_str(ext, ext_val, value);
1598 break;
1599 default:
1600 /* assume integer */
1601 err = parse_u64(value, &num);
1602 if (err) {
1603 pr_warn("extern %s=%s should be integer\n",
1604 ext->name, value);
1605 return err;
1606 }
1607 err = set_ext_value_num(ext, ext_val, num);
1608 break;
1609 }
1610 if (err)
1611 return err;
1612 pr_debug("extern %s=%s\n", ext->name, value);
1613 return 0;
1614 }
1615
1616 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1617 {
1618 char buf[PATH_MAX];
1619 struct utsname uts;
1620 int len, err = 0;
1621 gzFile file;
1622
1623 uname(&uts);
1624 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1625 if (len < 0)
1626 return -EINVAL;
1627 else if (len >= PATH_MAX)
1628 return -ENAMETOOLONG;
1629
1630 /* gzopen also accepts uncompressed files. */
1631 file = gzopen(buf, "r");
1632 if (!file)
1633 file = gzopen("/proc/config.gz", "r");
1634
1635 if (!file) {
1636 pr_warn("failed to open system Kconfig\n");
1637 return -ENOENT;
1638 }
1639
1640 while (gzgets(file, buf, sizeof(buf))) {
1641 err = bpf_object__process_kconfig_line(obj, buf, data);
1642 if (err) {
1643 pr_warn("error parsing system Kconfig line '%s': %d\n",
1644 buf, err);
1645 goto out;
1646 }
1647 }
1648
1649 out:
1650 gzclose(file);
1651 return err;
1652 }
1653
1654 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1655 const char *config, void *data)
1656 {
1657 char buf[PATH_MAX];
1658 int err = 0;
1659 FILE *file;
1660
1661 file = fmemopen((void *)config, strlen(config), "r");
1662 if (!file) {
1663 err = -errno;
1664 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1665 return err;
1666 }
1667
1668 while (fgets(buf, sizeof(buf), file)) {
1669 err = bpf_object__process_kconfig_line(obj, buf, data);
1670 if (err) {
1671 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1672 buf, err);
1673 break;
1674 }
1675 }
1676
1677 fclose(file);
1678 return err;
1679 }
1680
1681 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1682 {
1683 struct extern_desc *last_ext;
1684 size_t map_sz;
1685 int err;
1686
1687 if (obj->nr_extern == 0)
1688 return 0;
1689
1690 last_ext = &obj->externs[obj->nr_extern - 1];
1691 map_sz = last_ext->data_off + last_ext->sz;
1692
1693 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1694 obj->efile.symbols_shndx,
1695 NULL, map_sz);
1696 if (err)
1697 return err;
1698
1699 obj->kconfig_map_idx = obj->nr_maps - 1;
1700
1701 return 0;
1702 }
1703
1704 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1705 {
1706 Elf_Data *symbols = obj->efile.symbols;
1707 int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1708 Elf_Data *data = NULL;
1709 Elf_Scn *scn;
1710
1711 if (obj->efile.maps_shndx < 0)
1712 return 0;
1713
1714 if (!symbols)
1715 return -EINVAL;
1716
1717 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
1718 if (scn)
1719 data = elf_getdata(scn, NULL);
1720 if (!scn || !data) {
1721 pr_warn("failed to get Elf_Data from map section %d\n",
1722 obj->efile.maps_shndx);
1723 return -EINVAL;
1724 }
1725
1726 /*
1727 * Count number of maps. Each map has a name.
1728 * Array of maps is not supported: only the first element is
1729 * considered.
1730 *
1731 * TODO: Detect array of map and report error.
1732 */
1733 nr_syms = symbols->d_size / sizeof(GElf_Sym);
1734 for (i = 0; i < nr_syms; i++) {
1735 GElf_Sym sym;
1736
1737 if (!gelf_getsym(symbols, i, &sym))
1738 continue;
1739 if (sym.st_shndx != obj->efile.maps_shndx)
1740 continue;
1741 nr_maps++;
1742 }
1743 /* Assume equally sized map definitions */
1744 pr_debug("maps in %s: %d maps in %zd bytes\n",
1745 obj->path, nr_maps, data->d_size);
1746
1747 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1748 pr_warn("unable to determine map definition size section %s, %d maps in %zd bytes\n",
1749 obj->path, nr_maps, data->d_size);
1750 return -EINVAL;
1751 }
1752 map_def_sz = data->d_size / nr_maps;
1753
1754 /* Fill obj->maps using data in "maps" section. */
1755 for (i = 0; i < nr_syms; i++) {
1756 GElf_Sym sym;
1757 const char *map_name;
1758 struct bpf_map_def *def;
1759 struct bpf_map *map;
1760
1761 if (!gelf_getsym(symbols, i, &sym))
1762 continue;
1763 if (sym.st_shndx != obj->efile.maps_shndx)
1764 continue;
1765
1766 map = bpf_object__add_map(obj);
1767 if (IS_ERR(map))
1768 return PTR_ERR(map);
1769
1770 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1771 sym.st_name);
1772 if (!map_name) {
1773 pr_warn("failed to get map #%d name sym string for obj %s\n",
1774 i, obj->path);
1775 return -LIBBPF_ERRNO__FORMAT;
1776 }
1777
1778 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1779 map->sec_idx = sym.st_shndx;
1780 map->sec_offset = sym.st_value;
1781 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1782 map_name, map->sec_idx, map->sec_offset);
1783 if (sym.st_value + map_def_sz > data->d_size) {
1784 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1785 obj->path, map_name);
1786 return -EINVAL;
1787 }
1788
1789 map->name = strdup(map_name);
1790 if (!map->name) {
1791 pr_warn("failed to alloc map name\n");
1792 return -ENOMEM;
1793 }
1794 pr_debug("map %d is \"%s\"\n", i, map->name);
1795 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1796 /*
1797 * If the definition of the map in the object file fits in
1798 * bpf_map_def, copy it. Any extra fields in our version
1799 * of bpf_map_def will default to zero as a result of the
1800 * calloc above.
1801 */
1802 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1803 memcpy(&map->def, def, map_def_sz);
1804 } else {
1805 /*
1806 * Here the map structure being read is bigger than what
1807 * we expect, truncate if the excess bits are all zero.
1808 * If they are not zero, reject this map as
1809 * incompatible.
1810 */
1811 char *b;
1812
1813 for (b = ((char *)def) + sizeof(struct bpf_map_def);
1814 b < ((char *)def) + map_def_sz; b++) {
1815 if (*b != 0) {
1816 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1817 obj->path, map_name);
1818 if (strict)
1819 return -EINVAL;
1820 }
1821 }
1822 memcpy(&map->def, def, sizeof(struct bpf_map_def));
1823 }
1824 }
1825 return 0;
1826 }
1827
1828 static const struct btf_type *
1829 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1830 {
1831 const struct btf_type *t = btf__type_by_id(btf, id);
1832
1833 if (res_id)
1834 *res_id = id;
1835
1836 while (btf_is_mod(t) || btf_is_typedef(t)) {
1837 if (res_id)
1838 *res_id = t->type;
1839 t = btf__type_by_id(btf, t->type);
1840 }
1841
1842 return t;
1843 }
1844
1845 static const struct btf_type *
1846 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1847 {
1848 const struct btf_type *t;
1849
1850 t = skip_mods_and_typedefs(btf, id, NULL);
1851 if (!btf_is_ptr(t))
1852 return NULL;
1853
1854 t = skip_mods_and_typedefs(btf, t->type, res_id);
1855
1856 return btf_is_func_proto(t) ? t : NULL;
1857 }
1858
1859 /*
1860 * Fetch integer attribute of BTF map definition. Such attributes are
1861 * represented using a pointer to an array, in which dimensionality of array
1862 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
1863 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
1864 * type definition, while using only sizeof(void *) space in ELF data section.
1865 */
1866 static bool get_map_field_int(const char *map_name, const struct btf *btf,
1867 const struct btf_member *m, __u32 *res)
1868 {
1869 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1870 const char *name = btf__name_by_offset(btf, m->name_off);
1871 const struct btf_array *arr_info;
1872 const struct btf_type *arr_t;
1873
1874 if (!btf_is_ptr(t)) {
1875 pr_warn("map '%s': attr '%s': expected PTR, got %u.\n",
1876 map_name, name, btf_kind(t));
1877 return false;
1878 }
1879
1880 arr_t = btf__type_by_id(btf, t->type);
1881 if (!arr_t) {
1882 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1883 map_name, name, t->type);
1884 return false;
1885 }
1886 if (!btf_is_array(arr_t)) {
1887 pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n",
1888 map_name, name, btf_kind(arr_t));
1889 return false;
1890 }
1891 arr_info = btf_array(arr_t);
1892 *res = arr_info->nelems;
1893 return true;
1894 }
1895
1896 static int build_map_pin_path(struct bpf_map *map, const char *path)
1897 {
1898 char buf[PATH_MAX];
1899 int err, len;
1900
1901 if (!path)
1902 path = "/sys/fs/bpf";
1903
1904 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
1905 if (len < 0)
1906 return -EINVAL;
1907 else if (len >= PATH_MAX)
1908 return -ENAMETOOLONG;
1909
1910 err = bpf_map__set_pin_path(map, buf);
1911 if (err)
1912 return err;
1913
1914 return 0;
1915 }
1916
1917 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
1918 const struct btf_type *sec,
1919 int var_idx, int sec_idx,
1920 const Elf_Data *data, bool strict,
1921 const char *pin_root_path)
1922 {
1923 const struct btf_type *var, *def, *t;
1924 const struct btf_var_secinfo *vi;
1925 const struct btf_var *var_extra;
1926 const struct btf_member *m;
1927 const char *map_name;
1928 struct bpf_map *map;
1929 int vlen, i;
1930
1931 vi = btf_var_secinfos(sec) + var_idx;
1932 var = btf__type_by_id(obj->btf, vi->type);
1933 var_extra = btf_var(var);
1934 map_name = btf__name_by_offset(obj->btf, var->name_off);
1935 vlen = btf_vlen(var);
1936
1937 if (map_name == NULL || map_name[0] == '\0') {
1938 pr_warn("map #%d: empty name.\n", var_idx);
1939 return -EINVAL;
1940 }
1941 if ((__u64)vi->offset + vi->size > data->d_size) {
1942 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
1943 return -EINVAL;
1944 }
1945 if (!btf_is_var(var)) {
1946 pr_warn("map '%s': unexpected var kind %u.\n",
1947 map_name, btf_kind(var));
1948 return -EINVAL;
1949 }
1950 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
1951 var_extra->linkage != BTF_VAR_STATIC) {
1952 pr_warn("map '%s': unsupported var linkage %u.\n",
1953 map_name, var_extra->linkage);
1954 return -EOPNOTSUPP;
1955 }
1956
1957 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
1958 if (!btf_is_struct(def)) {
1959 pr_warn("map '%s': unexpected def kind %u.\n",
1960 map_name, btf_kind(var));
1961 return -EINVAL;
1962 }
1963 if (def->size > vi->size) {
1964 pr_warn("map '%s': invalid def size.\n", map_name);
1965 return -EINVAL;
1966 }
1967
1968 map = bpf_object__add_map(obj);
1969 if (IS_ERR(map))
1970 return PTR_ERR(map);
1971 map->name = strdup(map_name);
1972 if (!map->name) {
1973 pr_warn("map '%s': failed to alloc map name.\n", map_name);
1974 return -ENOMEM;
1975 }
1976 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1977 map->def.type = BPF_MAP_TYPE_UNSPEC;
1978 map->sec_idx = sec_idx;
1979 map->sec_offset = vi->offset;
1980 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
1981 map_name, map->sec_idx, map->sec_offset);
1982
1983 vlen = btf_vlen(def);
1984 m = btf_members(def);
1985 for (i = 0; i < vlen; i++, m++) {
1986 const char *name = btf__name_by_offset(obj->btf, m->name_off);
1987
1988 if (!name) {
1989 pr_warn("map '%s': invalid field #%d.\n", map_name, i);
1990 return -EINVAL;
1991 }
1992 if (strcmp(name, "type") == 0) {
1993 if (!get_map_field_int(map_name, obj->btf, m,
1994 &map->def.type))
1995 return -EINVAL;
1996 pr_debug("map '%s': found type = %u.\n",
1997 map_name, map->def.type);
1998 } else if (strcmp(name, "max_entries") == 0) {
1999 if (!get_map_field_int(map_name, obj->btf, m,
2000 &map->def.max_entries))
2001 return -EINVAL;
2002 pr_debug("map '%s': found max_entries = %u.\n",
2003 map_name, map->def.max_entries);
2004 } else if (strcmp(name, "map_flags") == 0) {
2005 if (!get_map_field_int(map_name, obj->btf, m,
2006 &map->def.map_flags))
2007 return -EINVAL;
2008 pr_debug("map '%s': found map_flags = %u.\n",
2009 map_name, map->def.map_flags);
2010 } else if (strcmp(name, "key_size") == 0) {
2011 __u32 sz;
2012
2013 if (!get_map_field_int(map_name, obj->btf, m, &sz))
2014 return -EINVAL;
2015 pr_debug("map '%s': found key_size = %u.\n",
2016 map_name, sz);
2017 if (map->def.key_size && map->def.key_size != sz) {
2018 pr_warn("map '%s': conflicting key size %u != %u.\n",
2019 map_name, map->def.key_size, sz);
2020 return -EINVAL;
2021 }
2022 map->def.key_size = sz;
2023 } else if (strcmp(name, "key") == 0) {
2024 __s64 sz;
2025
2026 t = btf__type_by_id(obj->btf, m->type);
2027 if (!t) {
2028 pr_warn("map '%s': key type [%d] not found.\n",
2029 map_name, m->type);
2030 return -EINVAL;
2031 }
2032 if (!btf_is_ptr(t)) {
2033 pr_warn("map '%s': key spec is not PTR: %u.\n",
2034 map_name, btf_kind(t));
2035 return -EINVAL;
2036 }
2037 sz = btf__resolve_size(obj->btf, t->type);
2038 if (sz < 0) {
2039 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2040 map_name, t->type, (ssize_t)sz);
2041 return sz;
2042 }
2043 pr_debug("map '%s': found key [%u], sz = %zd.\n",
2044 map_name, t->type, (ssize_t)sz);
2045 if (map->def.key_size && map->def.key_size != sz) {
2046 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2047 map_name, map->def.key_size, (ssize_t)sz);
2048 return -EINVAL;
2049 }
2050 map->def.key_size = sz;
2051 map->btf_key_type_id = t->type;
2052 } else if (strcmp(name, "value_size") == 0) {
2053 __u32 sz;
2054
2055 if (!get_map_field_int(map_name, obj->btf, m, &sz))
2056 return -EINVAL;
2057 pr_debug("map '%s': found value_size = %u.\n",
2058 map_name, sz);
2059 if (map->def.value_size && map->def.value_size != sz) {
2060 pr_warn("map '%s': conflicting value size %u != %u.\n",
2061 map_name, map->def.value_size, sz);
2062 return -EINVAL;
2063 }
2064 map->def.value_size = sz;
2065 } else if (strcmp(name, "value") == 0) {
2066 __s64 sz;
2067
2068 t = btf__type_by_id(obj->btf, m->type);
2069 if (!t) {
2070 pr_warn("map '%s': value type [%d] not found.\n",
2071 map_name, m->type);
2072 return -EINVAL;
2073 }
2074 if (!btf_is_ptr(t)) {
2075 pr_warn("map '%s': value spec is not PTR: %u.\n",
2076 map_name, btf_kind(t));
2077 return -EINVAL;
2078 }
2079 sz = btf__resolve_size(obj->btf, t->type);
2080 if (sz < 0) {
2081 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2082 map_name, t->type, (ssize_t)sz);
2083 return sz;
2084 }
2085 pr_debug("map '%s': found value [%u], sz = %zd.\n",
2086 map_name, t->type, (ssize_t)sz);
2087 if (map->def.value_size && map->def.value_size != sz) {
2088 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2089 map_name, map->def.value_size, (ssize_t)sz);
2090 return -EINVAL;
2091 }
2092 map->def.value_size = sz;
2093 map->btf_value_type_id = t->type;
2094 } else if (strcmp(name, "pinning") == 0) {
2095 __u32 val;
2096 int err;
2097
2098 if (!get_map_field_int(map_name, obj->btf, m, &val))
2099 return -EINVAL;
2100 pr_debug("map '%s': found pinning = %u.\n",
2101 map_name, val);
2102
2103 if (val != LIBBPF_PIN_NONE &&
2104 val != LIBBPF_PIN_BY_NAME) {
2105 pr_warn("map '%s': invalid pinning value %u.\n",
2106 map_name, val);
2107 return -EINVAL;
2108 }
2109 if (val == LIBBPF_PIN_BY_NAME) {
2110 err = build_map_pin_path(map, pin_root_path);
2111 if (err) {
2112 pr_warn("map '%s': couldn't build pin path.\n",
2113 map_name);
2114 return err;
2115 }
2116 }
2117 } else {
2118 if (strict) {
2119 pr_warn("map '%s': unknown field '%s'.\n",
2120 map_name, name);
2121 return -ENOTSUP;
2122 }
2123 pr_debug("map '%s': ignoring unknown field '%s'.\n",
2124 map_name, name);
2125 }
2126 }
2127
2128 if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
2129 pr_warn("map '%s': map type isn't specified.\n", map_name);
2130 return -EINVAL;
2131 }
2132
2133 return 0;
2134 }
2135
2136 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2137 const char *pin_root_path)
2138 {
2139 const struct btf_type *sec = NULL;
2140 int nr_types, i, vlen, err;
2141 const struct btf_type *t;
2142 const char *name;
2143 Elf_Data *data;
2144 Elf_Scn *scn;
2145
2146 if (obj->efile.btf_maps_shndx < 0)
2147 return 0;
2148
2149 scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx);
2150 if (scn)
2151 data = elf_getdata(scn, NULL);
2152 if (!scn || !data) {
2153 pr_warn("failed to get Elf_Data from map section %d (%s)\n",
2154 obj->efile.maps_shndx, MAPS_ELF_SEC);
2155 return -EINVAL;
2156 }
2157
2158 nr_types = btf__get_nr_types(obj->btf);
2159 for (i = 1; i <= nr_types; i++) {
2160 t = btf__type_by_id(obj->btf, i);
2161 if (!btf_is_datasec(t))
2162 continue;
2163 name = btf__name_by_offset(obj->btf, t->name_off);
2164 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2165 sec = t;
2166 break;
2167 }
2168 }
2169
2170 if (!sec) {
2171 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2172 return -ENOENT;
2173 }
2174
2175 vlen = btf_vlen(sec);
2176 for (i = 0; i < vlen; i++) {
2177 err = bpf_object__init_user_btf_map(obj, sec, i,
2178 obj->efile.btf_maps_shndx,
2179 data, strict,
2180 pin_root_path);
2181 if (err)
2182 return err;
2183 }
2184
2185 return 0;
2186 }
2187
2188 static int bpf_object__init_maps(struct bpf_object *obj,
2189 const struct bpf_object_open_opts *opts)
2190 {
2191 const char *pin_root_path;
2192 bool strict;
2193 int err;
2194
2195 strict = !OPTS_GET(opts, relaxed_maps, false);
2196 pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2197
2198 err = bpf_object__init_user_maps(obj, strict);
2199 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2200 err = err ?: bpf_object__init_global_data_maps(obj);
2201 err = err ?: bpf_object__init_kconfig_map(obj);
2202 err = err ?: bpf_object__init_struct_ops_maps(obj);
2203 if (err)
2204 return err;
2205
2206 return 0;
2207 }
2208
2209 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2210 {
2211 Elf_Scn *scn;
2212 GElf_Shdr sh;
2213
2214 scn = elf_getscn(obj->efile.elf, idx);
2215 if (!scn)
2216 return false;
2217
2218 if (gelf_getshdr(scn, &sh) != &sh)
2219 return false;
2220
2221 if (sh.sh_flags & SHF_EXECINSTR)
2222 return true;
2223
2224 return false;
2225 }
2226
2227 static void bpf_object__sanitize_btf(struct bpf_object *obj)
2228 {
2229 bool has_func_global = obj->caps.btf_func_global;
2230 bool has_datasec = obj->caps.btf_datasec;
2231 bool has_func = obj->caps.btf_func;
2232 struct btf *btf = obj->btf;
2233 struct btf_type *t;
2234 int i, j, vlen;
2235
2236 if (!obj->btf || (has_func && has_datasec && has_func_global))
2237 return;
2238
2239 for (i = 1; i <= btf__get_nr_types(btf); i++) {
2240 t = (struct btf_type *)btf__type_by_id(btf, i);
2241
2242 if (!has_datasec && btf_is_var(t)) {
2243 /* replace VAR with INT */
2244 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2245 /*
2246 * using size = 1 is the safest choice, 4 will be too
2247 * big and cause kernel BTF validation failure if
2248 * original variable took less than 4 bytes
2249 */
2250 t->size = 1;
2251 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2252 } else if (!has_datasec && btf_is_datasec(t)) {
2253 /* replace DATASEC with STRUCT */
2254 const struct btf_var_secinfo *v = btf_var_secinfos(t);
2255 struct btf_member *m = btf_members(t);
2256 struct btf_type *vt;
2257 char *name;
2258
2259 name = (char *)btf__name_by_offset(btf, t->name_off);
2260 while (*name) {
2261 if (*name == '.')
2262 *name = '_';
2263 name++;
2264 }
2265
2266 vlen = btf_vlen(t);
2267 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2268 for (j = 0; j < vlen; j++, v++, m++) {
2269 /* order of field assignments is important */
2270 m->offset = v->offset * 8;
2271 m->type = v->type;
2272 /* preserve variable name as member name */
2273 vt = (void *)btf__type_by_id(btf, v->type);
2274 m->name_off = vt->name_off;
2275 }
2276 } else if (!has_func && btf_is_func_proto(t)) {
2277 /* replace FUNC_PROTO with ENUM */
2278 vlen = btf_vlen(t);
2279 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2280 t->size = sizeof(__u32); /* kernel enforced */
2281 } else if (!has_func && btf_is_func(t)) {
2282 /* replace FUNC with TYPEDEF */
2283 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2284 } else if (!has_func_global && btf_is_func(t)) {
2285 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2286 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2287 }
2288 }
2289 }
2290
2291 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj)
2292 {
2293 if (!obj->btf_ext)
2294 return;
2295
2296 if (!obj->caps.btf_func) {
2297 btf_ext__free(obj->btf_ext);
2298 obj->btf_ext = NULL;
2299 }
2300 }
2301
2302 static bool libbpf_needs_btf(const struct bpf_object *obj)
2303 {
2304 return obj->efile.btf_maps_shndx >= 0 ||
2305 obj->efile.st_ops_shndx >= 0 ||
2306 obj->nr_extern > 0;
2307 }
2308
2309 static bool kernel_needs_btf(const struct bpf_object *obj)
2310 {
2311 return obj->efile.st_ops_shndx >= 0;
2312 }
2313
2314 static int bpf_object__init_btf(struct bpf_object *obj,
2315 Elf_Data *btf_data,
2316 Elf_Data *btf_ext_data)
2317 {
2318 int err = -ENOENT;
2319
2320 if (btf_data) {
2321 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2322 if (IS_ERR(obj->btf)) {
2323 err = PTR_ERR(obj->btf);
2324 obj->btf = NULL;
2325 pr_warn("Error loading ELF section %s: %d.\n",
2326 BTF_ELF_SEC, err);
2327 goto out;
2328 }
2329 err = 0;
2330 }
2331 if (btf_ext_data) {
2332 if (!obj->btf) {
2333 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2334 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2335 goto out;
2336 }
2337 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
2338 btf_ext_data->d_size);
2339 if (IS_ERR(obj->btf_ext)) {
2340 pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
2341 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
2342 obj->btf_ext = NULL;
2343 goto out;
2344 }
2345 }
2346 out:
2347 if (err && libbpf_needs_btf(obj)) {
2348 pr_warn("BTF is required, but is missing or corrupted.\n");
2349 return err;
2350 }
2351 return 0;
2352 }
2353
2354 static int bpf_object__finalize_btf(struct bpf_object *obj)
2355 {
2356 int err;
2357
2358 if (!obj->btf)
2359 return 0;
2360
2361 err = btf__finalize_data(obj, obj->btf);
2362 if (!err)
2363 return 0;
2364
2365 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2366 btf__free(obj->btf);
2367 obj->btf = NULL;
2368 btf_ext__free(obj->btf_ext);
2369 obj->btf_ext = NULL;
2370
2371 if (libbpf_needs_btf(obj)) {
2372 pr_warn("BTF is required, but is missing or corrupted.\n");
2373 return -ENOENT;
2374 }
2375 return 0;
2376 }
2377
2378 static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog)
2379 {
2380 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2381 prog->type == BPF_PROG_TYPE_LSM)
2382 return true;
2383
2384 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2385 * also need vmlinux BTF
2386 */
2387 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2388 return true;
2389
2390 return false;
2391 }
2392
2393 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj)
2394 {
2395 struct bpf_program *prog;
2396 int err;
2397
2398 bpf_object__for_each_program(prog, obj) {
2399 if (libbpf_prog_needs_vmlinux_btf(prog)) {
2400 obj->btf_vmlinux = libbpf_find_kernel_btf();
2401 if (IS_ERR(obj->btf_vmlinux)) {
2402 err = PTR_ERR(obj->btf_vmlinux);
2403 pr_warn("Error loading vmlinux BTF: %d\n", err);
2404 obj->btf_vmlinux = NULL;
2405 return err;
2406 }
2407 return 0;
2408 }
2409 }
2410
2411 return 0;
2412 }
2413
2414 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2415 {
2416 int err = 0;
2417
2418 if (!obj->btf)
2419 return 0;
2420
2421 bpf_object__sanitize_btf(obj);
2422 bpf_object__sanitize_btf_ext(obj);
2423
2424 err = btf__load(obj->btf);
2425 if (err) {
2426 pr_warn("Error loading %s into kernel: %d.\n",
2427 BTF_ELF_SEC, err);
2428 btf__free(obj->btf);
2429 obj->btf = NULL;
2430 /* btf_ext can't exist without btf, so free it as well */
2431 if (obj->btf_ext) {
2432 btf_ext__free(obj->btf_ext);
2433 obj->btf_ext = NULL;
2434 }
2435
2436 if (kernel_needs_btf(obj))
2437 return err;
2438 }
2439 return 0;
2440 }
2441
2442 static int bpf_object__elf_collect(struct bpf_object *obj)
2443 {
2444 Elf *elf = obj->efile.elf;
2445 GElf_Ehdr *ep = &obj->efile.ehdr;
2446 Elf_Data *btf_ext_data = NULL;
2447 Elf_Data *btf_data = NULL;
2448 Elf_Scn *scn = NULL;
2449 int idx = 0, err = 0;
2450
2451 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
2452 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
2453 pr_warn("failed to get e_shstrndx from %s\n", obj->path);
2454 return -LIBBPF_ERRNO__FORMAT;
2455 }
2456
2457 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2458 char *name;
2459 GElf_Shdr sh;
2460 Elf_Data *data;
2461
2462 idx++;
2463 if (gelf_getshdr(scn, &sh) != &sh) {
2464 pr_warn("failed to get section(%d) header from %s\n",
2465 idx, obj->path);
2466 return -LIBBPF_ERRNO__FORMAT;
2467 }
2468
2469 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
2470 if (!name) {
2471 pr_warn("failed to get section(%d) name from %s\n",
2472 idx, obj->path);
2473 return -LIBBPF_ERRNO__FORMAT;
2474 }
2475
2476 data = elf_getdata(scn, 0);
2477 if (!data) {
2478 pr_warn("failed to get section(%d) data from %s(%s)\n",
2479 idx, name, obj->path);
2480 return -LIBBPF_ERRNO__FORMAT;
2481 }
2482 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
2483 idx, name, (unsigned long)data->d_size,
2484 (int)sh.sh_link, (unsigned long)sh.sh_flags,
2485 (int)sh.sh_type);
2486
2487 if (strcmp(name, "license") == 0) {
2488 err = bpf_object__init_license(obj,
2489 data->d_buf,
2490 data->d_size);
2491 if (err)
2492 return err;
2493 } else if (strcmp(name, "version") == 0) {
2494 err = bpf_object__init_kversion(obj,
2495 data->d_buf,
2496 data->d_size);
2497 if (err)
2498 return err;
2499 } else if (strcmp(name, "maps") == 0) {
2500 obj->efile.maps_shndx = idx;
2501 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
2502 obj->efile.btf_maps_shndx = idx;
2503 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
2504 btf_data = data;
2505 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
2506 btf_ext_data = data;
2507 } else if (sh.sh_type == SHT_SYMTAB) {
2508 if (obj->efile.symbols) {
2509 pr_warn("bpf: multiple SYMTAB in %s\n",
2510 obj->path);
2511 return -LIBBPF_ERRNO__FORMAT;
2512 }
2513 obj->efile.symbols = data;
2514 obj->efile.symbols_shndx = idx;
2515 obj->efile.strtabidx = sh.sh_link;
2516 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
2517 if (sh.sh_flags & SHF_EXECINSTR) {
2518 if (strcmp(name, ".text") == 0)
2519 obj->efile.text_shndx = idx;
2520 err = bpf_object__add_program(obj, data->d_buf,
2521 data->d_size,
2522 name, idx);
2523 if (err) {
2524 char errmsg[STRERR_BUFSIZE];
2525 char *cp;
2526
2527 cp = libbpf_strerror_r(-err, errmsg,
2528 sizeof(errmsg));
2529 pr_warn("failed to alloc program %s (%s): %s",
2530 name, obj->path, cp);
2531 return err;
2532 }
2533 } else if (strcmp(name, DATA_SEC) == 0) {
2534 obj->efile.data = data;
2535 obj->efile.data_shndx = idx;
2536 } else if (strcmp(name, RODATA_SEC) == 0) {
2537 obj->efile.rodata = data;
2538 obj->efile.rodata_shndx = idx;
2539 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
2540 obj->efile.st_ops_data = data;
2541 obj->efile.st_ops_shndx = idx;
2542 } else {
2543 pr_debug("skip section(%d) %s\n", idx, name);
2544 }
2545 } else if (sh.sh_type == SHT_REL) {
2546 int nr_sects = obj->efile.nr_reloc_sects;
2547 void *sects = obj->efile.reloc_sects;
2548 int sec = sh.sh_info; /* points to other section */
2549
2550 /* Only do relo for section with exec instructions */
2551 if (!section_have_execinstr(obj, sec) &&
2552 strcmp(name, ".rel" STRUCT_OPS_SEC)) {
2553 pr_debug("skip relo %s(%d) for section(%d)\n",
2554 name, idx, sec);
2555 continue;
2556 }
2557
2558 sects = reallocarray(sects, nr_sects + 1,
2559 sizeof(*obj->efile.reloc_sects));
2560 if (!sects) {
2561 pr_warn("reloc_sects realloc failed\n");
2562 return -ENOMEM;
2563 }
2564
2565 obj->efile.reloc_sects = sects;
2566 obj->efile.nr_reloc_sects++;
2567
2568 obj->efile.reloc_sects[nr_sects].shdr = sh;
2569 obj->efile.reloc_sects[nr_sects].data = data;
2570 } else if (sh.sh_type == SHT_NOBITS &&
2571 strcmp(name, BSS_SEC) == 0) {
2572 obj->efile.bss = data;
2573 obj->efile.bss_shndx = idx;
2574 } else {
2575 pr_debug("skip section(%d) %s\n", idx, name);
2576 }
2577 }
2578
2579 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
2580 pr_warn("Corrupted ELF file: index of strtab invalid\n");
2581 return -LIBBPF_ERRNO__FORMAT;
2582 }
2583 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
2584 }
2585
2586 static bool sym_is_extern(const GElf_Sym *sym)
2587 {
2588 int bind = GELF_ST_BIND(sym->st_info);
2589 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
2590 return sym->st_shndx == SHN_UNDEF &&
2591 (bind == STB_GLOBAL || bind == STB_WEAK) &&
2592 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
2593 }
2594
2595 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
2596 {
2597 const struct btf_type *t;
2598 const char *var_name;
2599 int i, n;
2600
2601 if (!btf)
2602 return -ESRCH;
2603
2604 n = btf__get_nr_types(btf);
2605 for (i = 1; i <= n; i++) {
2606 t = btf__type_by_id(btf, i);
2607
2608 if (!btf_is_var(t))
2609 continue;
2610
2611 var_name = btf__name_by_offset(btf, t->name_off);
2612 if (strcmp(var_name, ext_name))
2613 continue;
2614
2615 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
2616 return -EINVAL;
2617
2618 return i;
2619 }
2620
2621 return -ENOENT;
2622 }
2623
2624 static enum extern_type find_extern_type(const struct btf *btf, int id,
2625 bool *is_signed)
2626 {
2627 const struct btf_type *t;
2628 const char *name;
2629
2630 t = skip_mods_and_typedefs(btf, id, NULL);
2631 name = btf__name_by_offset(btf, t->name_off);
2632
2633 if (is_signed)
2634 *is_signed = false;
2635 switch (btf_kind(t)) {
2636 case BTF_KIND_INT: {
2637 int enc = btf_int_encoding(t);
2638
2639 if (enc & BTF_INT_BOOL)
2640 return t->size == 1 ? EXT_BOOL : EXT_UNKNOWN;
2641 if (is_signed)
2642 *is_signed = enc & BTF_INT_SIGNED;
2643 if (t->size == 1)
2644 return EXT_CHAR;
2645 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
2646 return EXT_UNKNOWN;
2647 return EXT_INT;
2648 }
2649 case BTF_KIND_ENUM:
2650 if (t->size != 4)
2651 return EXT_UNKNOWN;
2652 if (strcmp(name, "libbpf_tristate"))
2653 return EXT_UNKNOWN;
2654 return EXT_TRISTATE;
2655 case BTF_KIND_ARRAY:
2656 if (btf_array(t)->nelems == 0)
2657 return EXT_UNKNOWN;
2658 if (find_extern_type(btf, btf_array(t)->type, NULL) != EXT_CHAR)
2659 return EXT_UNKNOWN;
2660 return EXT_CHAR_ARR;
2661 default:
2662 return EXT_UNKNOWN;
2663 }
2664 }
2665
2666 static int cmp_externs(const void *_a, const void *_b)
2667 {
2668 const struct extern_desc *a = _a;
2669 const struct extern_desc *b = _b;
2670
2671 /* descending order by alignment requirements */
2672 if (a->align != b->align)
2673 return a->align > b->align ? -1 : 1;
2674 /* ascending order by size, within same alignment class */
2675 if (a->sz != b->sz)
2676 return a->sz < b->sz ? -1 : 1;
2677 /* resolve ties by name */
2678 return strcmp(a->name, b->name);
2679 }
2680
2681 static int bpf_object__collect_externs(struct bpf_object *obj)
2682 {
2683 const struct btf_type *t;
2684 struct extern_desc *ext;
2685 int i, n, off, btf_id;
2686 struct btf_type *sec;
2687 const char *ext_name;
2688 Elf_Scn *scn;
2689 GElf_Shdr sh;
2690
2691 if (!obj->efile.symbols)
2692 return 0;
2693
2694 scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx);
2695 if (!scn)
2696 return -LIBBPF_ERRNO__FORMAT;
2697 if (gelf_getshdr(scn, &sh) != &sh)
2698 return -LIBBPF_ERRNO__FORMAT;
2699 n = sh.sh_size / sh.sh_entsize;
2700
2701 pr_debug("looking for externs among %d symbols...\n", n);
2702 for (i = 0; i < n; i++) {
2703 GElf_Sym sym;
2704
2705 if (!gelf_getsym(obj->efile.symbols, i, &sym))
2706 return -LIBBPF_ERRNO__FORMAT;
2707 if (!sym_is_extern(&sym))
2708 continue;
2709 ext_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
2710 sym.st_name);
2711 if (!ext_name || !ext_name[0])
2712 continue;
2713
2714 ext = obj->externs;
2715 ext = reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
2716 if (!ext)
2717 return -ENOMEM;
2718 obj->externs = ext;
2719 ext = &ext[obj->nr_extern];
2720 memset(ext, 0, sizeof(*ext));
2721 obj->nr_extern++;
2722
2723 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
2724 if (ext->btf_id <= 0) {
2725 pr_warn("failed to find BTF for extern '%s': %d\n",
2726 ext_name, ext->btf_id);
2727 return ext->btf_id;
2728 }
2729 t = btf__type_by_id(obj->btf, ext->btf_id);
2730 ext->name = btf__name_by_offset(obj->btf, t->name_off);
2731 ext->sym_idx = i;
2732 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
2733 ext->sz = btf__resolve_size(obj->btf, t->type);
2734 if (ext->sz <= 0) {
2735 pr_warn("failed to resolve size of extern '%s': %d\n",
2736 ext_name, ext->sz);
2737 return ext->sz;
2738 }
2739 ext->align = btf__align_of(obj->btf, t->type);
2740 if (ext->align <= 0) {
2741 pr_warn("failed to determine alignment of extern '%s': %d\n",
2742 ext_name, ext->align);
2743 return -EINVAL;
2744 }
2745 ext->type = find_extern_type(obj->btf, t->type,
2746 &ext->is_signed);
2747 if (ext->type == EXT_UNKNOWN) {
2748 pr_warn("extern '%s' type is unsupported\n", ext_name);
2749 return -ENOTSUP;
2750 }
2751 }
2752 pr_debug("collected %d externs total\n", obj->nr_extern);
2753
2754 if (!obj->nr_extern)
2755 return 0;
2756
2757 /* sort externs by (alignment, size, name) and calculate their offsets
2758 * within a map */
2759 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
2760 off = 0;
2761 for (i = 0; i < obj->nr_extern; i++) {
2762 ext = &obj->externs[i];
2763 ext->data_off = roundup(off, ext->align);
2764 off = ext->data_off + ext->sz;
2765 pr_debug("extern #%d: symbol %d, off %u, name %s\n",
2766 i, ext->sym_idx, ext->data_off, ext->name);
2767 }
2768
2769 btf_id = btf__find_by_name(obj->btf, KCONFIG_SEC);
2770 if (btf_id <= 0) {
2771 pr_warn("no BTF info found for '%s' datasec\n", KCONFIG_SEC);
2772 return -ESRCH;
2773 }
2774
2775 sec = (struct btf_type *)btf__type_by_id(obj->btf, btf_id);
2776 sec->size = off;
2777 n = btf_vlen(sec);
2778 for (i = 0; i < n; i++) {
2779 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
2780
2781 t = btf__type_by_id(obj->btf, vs->type);
2782 ext_name = btf__name_by_offset(obj->btf, t->name_off);
2783 ext = find_extern_by_name(obj, ext_name);
2784 if (!ext) {
2785 pr_warn("failed to find extern definition for BTF var '%s'\n",
2786 ext_name);
2787 return -ESRCH;
2788 }
2789 vs->offset = ext->data_off;
2790 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
2791 }
2792
2793 return 0;
2794 }
2795
2796 static struct bpf_program *
2797 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
2798 {
2799 struct bpf_program *prog;
2800 size_t i;
2801
2802 for (i = 0; i < obj->nr_programs; i++) {
2803 prog = &obj->programs[i];
2804 if (prog->idx == idx)
2805 return prog;
2806 }
2807 return NULL;
2808 }
2809
2810 struct bpf_program *
2811 bpf_object__find_program_by_title(const struct bpf_object *obj,
2812 const char *title)
2813 {
2814 struct bpf_program *pos;
2815
2816 bpf_object__for_each_program(pos, obj) {
2817 if (pos->section_name && !strcmp(pos->section_name, title))
2818 return pos;
2819 }
2820 return NULL;
2821 }
2822
2823 struct bpf_program *
2824 bpf_object__find_program_by_name(const struct bpf_object *obj,
2825 const char *name)
2826 {
2827 struct bpf_program *prog;
2828
2829 bpf_object__for_each_program(prog, obj) {
2830 if (!strcmp(prog->name, name))
2831 return prog;
2832 }
2833 return NULL;
2834 }
2835
2836 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
2837 int shndx)
2838 {
2839 return shndx == obj->efile.data_shndx ||
2840 shndx == obj->efile.bss_shndx ||
2841 shndx == obj->efile.rodata_shndx;
2842 }
2843
2844 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
2845 int shndx)
2846 {
2847 return shndx == obj->efile.maps_shndx ||
2848 shndx == obj->efile.btf_maps_shndx;
2849 }
2850
2851 static enum libbpf_map_type
2852 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
2853 {
2854 if (shndx == obj->efile.data_shndx)
2855 return LIBBPF_MAP_DATA;
2856 else if (shndx == obj->efile.bss_shndx)
2857 return LIBBPF_MAP_BSS;
2858 else if (shndx == obj->efile.rodata_shndx)
2859 return LIBBPF_MAP_RODATA;
2860 else if (shndx == obj->efile.symbols_shndx)
2861 return LIBBPF_MAP_KCONFIG;
2862 else
2863 return LIBBPF_MAP_UNSPEC;
2864 }
2865
2866 static int bpf_program__record_reloc(struct bpf_program *prog,
2867 struct reloc_desc *reloc_desc,
2868 __u32 insn_idx, const char *name,
2869 const GElf_Sym *sym, const GElf_Rel *rel)
2870 {
2871 struct bpf_insn *insn = &prog->insns[insn_idx];
2872 size_t map_idx, nr_maps = prog->obj->nr_maps;
2873 struct bpf_object *obj = prog->obj;
2874 __u32 shdr_idx = sym->st_shndx;
2875 enum libbpf_map_type type;
2876 struct bpf_map *map;
2877
2878 /* sub-program call relocation */
2879 if (insn->code == (BPF_JMP | BPF_CALL)) {
2880 if (insn->src_reg != BPF_PSEUDO_CALL) {
2881 pr_warn("incorrect bpf_call opcode\n");
2882 return -LIBBPF_ERRNO__RELOC;
2883 }
2884 /* text_shndx can be 0, if no default "main" program exists */
2885 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
2886 pr_warn("bad call relo against section %u\n", shdr_idx);
2887 return -LIBBPF_ERRNO__RELOC;
2888 }
2889 if (sym->st_value % 8) {
2890 pr_warn("bad call relo offset: %zu\n",
2891 (size_t)sym->st_value);
2892 return -LIBBPF_ERRNO__RELOC;
2893 }
2894 reloc_desc->type = RELO_CALL;
2895 reloc_desc->insn_idx = insn_idx;
2896 reloc_desc->sym_off = sym->st_value;
2897 obj->has_pseudo_calls = true;
2898 return 0;
2899 }
2900
2901 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
2902 pr_warn("invalid relo for insns[%d].code 0x%x\n",
2903 insn_idx, insn->code);
2904 return -LIBBPF_ERRNO__RELOC;
2905 }
2906
2907 if (sym_is_extern(sym)) {
2908 int sym_idx = GELF_R_SYM(rel->r_info);
2909 int i, n = obj->nr_extern;
2910 struct extern_desc *ext;
2911
2912 for (i = 0; i < n; i++) {
2913 ext = &obj->externs[i];
2914 if (ext->sym_idx == sym_idx)
2915 break;
2916 }
2917 if (i >= n) {
2918 pr_warn("extern relo failed to find extern for sym %d\n",
2919 sym_idx);
2920 return -LIBBPF_ERRNO__RELOC;
2921 }
2922 pr_debug("found extern #%d '%s' (sym %d, off %u) for insn %u\n",
2923 i, ext->name, ext->sym_idx, ext->data_off, insn_idx);
2924 reloc_desc->type = RELO_EXTERN;
2925 reloc_desc->insn_idx = insn_idx;
2926 reloc_desc->sym_off = ext->data_off;
2927 return 0;
2928 }
2929
2930 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
2931 pr_warn("invalid relo for \'%s\' in special section 0x%x; forgot to initialize global var?..\n",
2932 name, shdr_idx);
2933 return -LIBBPF_ERRNO__RELOC;
2934 }
2935
2936 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
2937
2938 /* generic map reference relocation */
2939 if (type == LIBBPF_MAP_UNSPEC) {
2940 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
2941 pr_warn("bad map relo against section %u\n",
2942 shdr_idx);
2943 return -LIBBPF_ERRNO__RELOC;
2944 }
2945 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
2946 map = &obj->maps[map_idx];
2947 if (map->libbpf_type != type ||
2948 map->sec_idx != sym->st_shndx ||
2949 map->sec_offset != sym->st_value)
2950 continue;
2951 pr_debug("found map %zd (%s, sec %d, off %zu) for insn %u\n",
2952 map_idx, map->name, map->sec_idx,
2953 map->sec_offset, insn_idx);
2954 break;
2955 }
2956 if (map_idx >= nr_maps) {
2957 pr_warn("map relo failed to find map for sec %u, off %zu\n",
2958 shdr_idx, (size_t)sym->st_value);
2959 return -LIBBPF_ERRNO__RELOC;
2960 }
2961 reloc_desc->type = RELO_LD64;
2962 reloc_desc->insn_idx = insn_idx;
2963 reloc_desc->map_idx = map_idx;
2964 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
2965 return 0;
2966 }
2967
2968 /* global data map relocation */
2969 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
2970 pr_warn("bad data relo against section %u\n", shdr_idx);
2971 return -LIBBPF_ERRNO__RELOC;
2972 }
2973 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
2974 map = &obj->maps[map_idx];
2975 if (map->libbpf_type != type)
2976 continue;
2977 pr_debug("found data map %zd (%s, sec %d, off %zu) for insn %u\n",
2978 map_idx, map->name, map->sec_idx, map->sec_offset,
2979 insn_idx);
2980 break;
2981 }
2982 if (map_idx >= nr_maps) {
2983 pr_warn("data relo failed to find map for sec %u\n",
2984 shdr_idx);
2985 return -LIBBPF_ERRNO__RELOC;
2986 }
2987
2988 reloc_desc->type = RELO_DATA;
2989 reloc_desc->insn_idx = insn_idx;
2990 reloc_desc->map_idx = map_idx;
2991 reloc_desc->sym_off = sym->st_value;
2992 return 0;
2993 }
2994
2995 static int
2996 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
2997 Elf_Data *data, struct bpf_object *obj)
2998 {
2999 Elf_Data *symbols = obj->efile.symbols;
3000 int err, i, nrels;
3001
3002 pr_debug("collecting relocating info for: '%s'\n", prog->section_name);
3003 nrels = shdr->sh_size / shdr->sh_entsize;
3004
3005 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
3006 if (!prog->reloc_desc) {
3007 pr_warn("failed to alloc memory in relocation\n");
3008 return -ENOMEM;
3009 }
3010 prog->nr_reloc = nrels;
3011
3012 for (i = 0; i < nrels; i++) {
3013 const char *name;
3014 __u32 insn_idx;
3015 GElf_Sym sym;
3016 GElf_Rel rel;
3017
3018 if (!gelf_getrel(data, i, &rel)) {
3019 pr_warn("relocation: failed to get %d reloc\n", i);
3020 return -LIBBPF_ERRNO__FORMAT;
3021 }
3022 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3023 pr_warn("relocation: symbol %"PRIx64" not found\n",
3024 GELF_R_SYM(rel.r_info));
3025 return -LIBBPF_ERRNO__FORMAT;
3026 }
3027 if (rel.r_offset % sizeof(struct bpf_insn))
3028 return -LIBBPF_ERRNO__FORMAT;
3029
3030 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
3031 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
3032 sym.st_name) ? : "<?>";
3033
3034 pr_debug("relo for shdr %u, symb %zu, value %zu, type %d, bind %d, name %d (\'%s\'), insn %u\n",
3035 (__u32)sym.st_shndx, (size_t)GELF_R_SYM(rel.r_info),
3036 (size_t)sym.st_value, GELF_ST_TYPE(sym.st_info),
3037 GELF_ST_BIND(sym.st_info), sym.st_name, name,
3038 insn_idx);
3039
3040 err = bpf_program__record_reloc(prog, &prog->reloc_desc[i],
3041 insn_idx, name, &sym, &rel);
3042 if (err)
3043 return err;
3044 }
3045 return 0;
3046 }
3047
3048 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3049 {
3050 struct bpf_map_def *def = &map->def;
3051 __u32 key_type_id = 0, value_type_id = 0;
3052 int ret;
3053
3054 /* if it's BTF-defined map, we don't need to search for type IDs.
3055 * For struct_ops map, it does not need btf_key_type_id and
3056 * btf_value_type_id.
3057 */
3058 if (map->sec_idx == obj->efile.btf_maps_shndx ||
3059 bpf_map__is_struct_ops(map))
3060 return 0;
3061
3062 if (!bpf_map__is_internal(map)) {
3063 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3064 def->value_size, &key_type_id,
3065 &value_type_id);
3066 } else {
3067 /*
3068 * LLVM annotates global data differently in BTF, that is,
3069 * only as '.data', '.bss' or '.rodata'.
3070 */
3071 ret = btf__find_by_name(obj->btf,
3072 libbpf_type_to_btf_name[map->libbpf_type]);
3073 }
3074 if (ret < 0)
3075 return ret;
3076
3077 map->btf_key_type_id = key_type_id;
3078 map->btf_value_type_id = bpf_map__is_internal(map) ?
3079 ret : value_type_id;
3080 return 0;
3081 }
3082
3083 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3084 {
3085 struct bpf_map_info info = {};
3086 __u32 len = sizeof(info);
3087 int new_fd, err;
3088 char *new_name;
3089
3090 err = bpf_obj_get_info_by_fd(fd, &info, &len);
3091 if (err)
3092 return err;
3093
3094 new_name = strdup(info.name);
3095 if (!new_name)
3096 return -errno;
3097
3098 new_fd = open("/", O_RDONLY | O_CLOEXEC);
3099 if (new_fd < 0) {
3100 err = -errno;
3101 goto err_free_new_name;
3102 }
3103
3104 new_fd = dup3(fd, new_fd, O_CLOEXEC);
3105 if (new_fd < 0) {
3106 err = -errno;
3107 goto err_close_new_fd;
3108 }
3109
3110 err = zclose(map->fd);
3111 if (err) {
3112 err = -errno;
3113 goto err_close_new_fd;
3114 }
3115 free(map->name);
3116
3117 map->fd = new_fd;
3118 map->name = new_name;
3119 map->def.type = info.type;
3120 map->def.key_size = info.key_size;
3121 map->def.value_size = info.value_size;
3122 map->def.max_entries = info.max_entries;
3123 map->def.map_flags = info.map_flags;
3124 map->btf_key_type_id = info.btf_key_type_id;
3125 map->btf_value_type_id = info.btf_value_type_id;
3126 map->reused = true;
3127
3128 return 0;
3129
3130 err_close_new_fd:
3131 close(new_fd);
3132 err_free_new_name:
3133 free(new_name);
3134 return err;
3135 }
3136
3137 int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
3138 {
3139 if (!map || !max_entries)
3140 return -EINVAL;
3141
3142 /* If map already created, its attributes can't be changed. */
3143 if (map->fd >= 0)
3144 return -EBUSY;
3145
3146 map->def.max_entries = max_entries;
3147
3148 return 0;
3149 }
3150
3151 static int
3152 bpf_object__probe_name(struct bpf_object *obj)
3153 {
3154 struct bpf_load_program_attr attr;
3155 char *cp, errmsg[STRERR_BUFSIZE];
3156 struct bpf_insn insns[] = {
3157 BPF_MOV64_IMM(BPF_REG_0, 0),
3158 BPF_EXIT_INSN(),
3159 };
3160 int ret;
3161
3162 /* make sure basic loading works */
3163
3164 memset(&attr, 0, sizeof(attr));
3165 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3166 attr.insns = insns;
3167 attr.insns_cnt = ARRAY_SIZE(insns);
3168 attr.license = "GPL";
3169
3170 ret = bpf_load_program_xattr(&attr, NULL, 0);
3171 if (ret < 0) {
3172 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3173 pr_warn("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n",
3174 __func__, cp, errno);
3175 return -errno;
3176 }
3177 close(ret);
3178
3179 /* now try the same program, but with the name */
3180
3181 attr.name = "test";
3182 ret = bpf_load_program_xattr(&attr, NULL, 0);
3183 if (ret >= 0) {
3184 obj->caps.name = 1;
3185 close(ret);
3186 }
3187
3188 return 0;
3189 }
3190
3191 static int
3192 bpf_object__probe_global_data(struct bpf_object *obj)
3193 {
3194 struct bpf_load_program_attr prg_attr;
3195 struct bpf_create_map_attr map_attr;
3196 char *cp, errmsg[STRERR_BUFSIZE];
3197 struct bpf_insn insns[] = {
3198 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
3199 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
3200 BPF_MOV64_IMM(BPF_REG_0, 0),
3201 BPF_EXIT_INSN(),
3202 };
3203 int ret, map;
3204
3205 memset(&map_attr, 0, sizeof(map_attr));
3206 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3207 map_attr.key_size = sizeof(int);
3208 map_attr.value_size = 32;
3209 map_attr.max_entries = 1;
3210
3211 map = bpf_create_map_xattr(&map_attr);
3212 if (map < 0) {
3213 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3214 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3215 __func__, cp, errno);
3216 return -errno;
3217 }
3218
3219 insns[0].imm = map;
3220
3221 memset(&prg_attr, 0, sizeof(prg_attr));
3222 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3223 prg_attr.insns = insns;
3224 prg_attr.insns_cnt = ARRAY_SIZE(insns);
3225 prg_attr.license = "GPL";
3226
3227 ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
3228 if (ret >= 0) {
3229 obj->caps.global_data = 1;
3230 close(ret);
3231 }
3232
3233 close(map);
3234 return 0;
3235 }
3236
3237 static int bpf_object__probe_btf_func(struct bpf_object *obj)
3238 {
3239 static const char strs[] = "\0int\0x\0a";
3240 /* void x(int a) {} */
3241 __u32 types[] = {
3242 /* int */
3243 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3244 /* FUNC_PROTO */ /* [2] */
3245 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3246 BTF_PARAM_ENC(7, 1),
3247 /* FUNC x */ /* [3] */
3248 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
3249 };
3250 int btf_fd;
3251
3252 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3253 strs, sizeof(strs));
3254 if (btf_fd >= 0) {
3255 obj->caps.btf_func = 1;
3256 close(btf_fd);
3257 return 1;
3258 }
3259
3260 return 0;
3261 }
3262
3263 static int bpf_object__probe_btf_func_global(struct bpf_object *obj)
3264 {
3265 static const char strs[] = "\0int\0x\0a";
3266 /* static void x(int a) {} */
3267 __u32 types[] = {
3268 /* int */
3269 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3270 /* FUNC_PROTO */ /* [2] */
3271 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3272 BTF_PARAM_ENC(7, 1),
3273 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */
3274 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
3275 };
3276 int btf_fd;
3277
3278 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3279 strs, sizeof(strs));
3280 if (btf_fd >= 0) {
3281 obj->caps.btf_func_global = 1;
3282 close(btf_fd);
3283 return 1;
3284 }
3285
3286 return 0;
3287 }
3288
3289 static int bpf_object__probe_btf_datasec(struct bpf_object *obj)
3290 {
3291 static const char strs[] = "\0x\0.data";
3292 /* static int a; */
3293 __u32 types[] = {
3294 /* int */
3295 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3296 /* VAR x */ /* [2] */
3297 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
3298 BTF_VAR_STATIC,
3299 /* DATASEC val */ /* [3] */
3300 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
3301 BTF_VAR_SECINFO_ENC(2, 0, 4),
3302 };
3303 int btf_fd;
3304
3305 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3306 strs, sizeof(strs));
3307 if (btf_fd >= 0) {
3308 obj->caps.btf_datasec = 1;
3309 close(btf_fd);
3310 return 1;
3311 }
3312
3313 return 0;
3314 }
3315
3316 static int bpf_object__probe_array_mmap(struct bpf_object *obj)
3317 {
3318 struct bpf_create_map_attr attr = {
3319 .map_type = BPF_MAP_TYPE_ARRAY,
3320 .map_flags = BPF_F_MMAPABLE,
3321 .key_size = sizeof(int),
3322 .value_size = sizeof(int),
3323 .max_entries = 1,
3324 };
3325 int fd;
3326
3327 fd = bpf_create_map_xattr(&attr);
3328 if (fd >= 0) {
3329 obj->caps.array_mmap = 1;
3330 close(fd);
3331 return 1;
3332 }
3333
3334 return 0;
3335 }
3336
3337 static int
3338 bpf_object__probe_exp_attach_type(struct bpf_object *obj)
3339 {
3340 struct bpf_load_program_attr attr;
3341 struct bpf_insn insns[] = {
3342 BPF_MOV64_IMM(BPF_REG_0, 0),
3343 BPF_EXIT_INSN(),
3344 };
3345 int fd;
3346
3347 memset(&attr, 0, sizeof(attr));
3348 /* use any valid combination of program type and (optional)
3349 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
3350 * to see if kernel supports expected_attach_type field for
3351 * BPF_PROG_LOAD command
3352 */
3353 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK;
3354 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE;
3355 attr.insns = insns;
3356 attr.insns_cnt = ARRAY_SIZE(insns);
3357 attr.license = "GPL";
3358
3359 fd = bpf_load_program_xattr(&attr, NULL, 0);
3360 if (fd >= 0) {
3361 obj->caps.exp_attach_type = 1;
3362 close(fd);
3363 return 1;
3364 }
3365 return 0;
3366 }
3367
3368 static int
3369 bpf_object__probe_caps(struct bpf_object *obj)
3370 {
3371 int (*probe_fn[])(struct bpf_object *obj) = {
3372 bpf_object__probe_name,
3373 bpf_object__probe_global_data,
3374 bpf_object__probe_btf_func,
3375 bpf_object__probe_btf_func_global,
3376 bpf_object__probe_btf_datasec,
3377 bpf_object__probe_array_mmap,
3378 bpf_object__probe_exp_attach_type,
3379 };
3380 int i, ret;
3381
3382 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) {
3383 ret = probe_fn[i](obj);
3384 if (ret < 0)
3385 pr_debug("Probe #%d failed with %d.\n", i, ret);
3386 }
3387
3388 return 0;
3389 }
3390
3391 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
3392 {
3393 struct bpf_map_info map_info = {};
3394 char msg[STRERR_BUFSIZE];
3395 __u32 map_info_len;
3396
3397 map_info_len = sizeof(map_info);
3398
3399 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
3400 pr_warn("failed to get map info for map FD %d: %s\n",
3401 map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
3402 return false;
3403 }
3404
3405 return (map_info.type == map->def.type &&
3406 map_info.key_size == map->def.key_size &&
3407 map_info.value_size == map->def.value_size &&
3408 map_info.max_entries == map->def.max_entries &&
3409 map_info.map_flags == map->def.map_flags);
3410 }
3411
3412 static int
3413 bpf_object__reuse_map(struct bpf_map *map)
3414 {
3415 char *cp, errmsg[STRERR_BUFSIZE];
3416 int err, pin_fd;
3417
3418 pin_fd = bpf_obj_get(map->pin_path);
3419 if (pin_fd < 0) {
3420 err = -errno;
3421 if (err == -ENOENT) {
3422 pr_debug("found no pinned map to reuse at '%s'\n",
3423 map->pin_path);
3424 return 0;
3425 }
3426
3427 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
3428 pr_warn("couldn't retrieve pinned map '%s': %s\n",
3429 map->pin_path, cp);
3430 return err;
3431 }
3432
3433 if (!map_is_reuse_compat(map, pin_fd)) {
3434 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
3435 map->pin_path);
3436 close(pin_fd);
3437 return -EINVAL;
3438 }
3439
3440 err = bpf_map__reuse_fd(map, pin_fd);
3441 if (err) {
3442 close(pin_fd);
3443 return err;
3444 }
3445 map->pinned = true;
3446 pr_debug("reused pinned map at '%s'\n", map->pin_path);
3447
3448 return 0;
3449 }
3450
3451 static int
3452 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
3453 {
3454 enum libbpf_map_type map_type = map->libbpf_type;
3455 char *cp, errmsg[STRERR_BUFSIZE];
3456 int err, zero = 0;
3457
3458 /* kernel already zero-initializes .bss map. */
3459 if (map_type == LIBBPF_MAP_BSS)
3460 return 0;
3461
3462 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
3463 if (err) {
3464 err = -errno;
3465 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3466 pr_warn("Error setting initial map(%s) contents: %s\n",
3467 map->name, cp);
3468 return err;
3469 }
3470
3471 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
3472 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
3473 err = bpf_map_freeze(map->fd);
3474 if (err) {
3475 err = -errno;
3476 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3477 pr_warn("Error freezing map(%s) as read-only: %s\n",
3478 map->name, cp);
3479 return err;
3480 }
3481 }
3482 return 0;
3483 }
3484
3485 static int
3486 bpf_object__create_maps(struct bpf_object *obj)
3487 {
3488 struct bpf_create_map_attr create_attr = {};
3489 int nr_cpus = 0;
3490 unsigned int i;
3491 int err;
3492
3493 for (i = 0; i < obj->nr_maps; i++) {
3494 struct bpf_map *map = &obj->maps[i];
3495 struct bpf_map_def *def = &map->def;
3496 char *cp, errmsg[STRERR_BUFSIZE];
3497 int *pfd = &map->fd;
3498
3499 if (map->pin_path) {
3500 err = bpf_object__reuse_map(map);
3501 if (err) {
3502 pr_warn("error reusing pinned map %s\n",
3503 map->name);
3504 return err;
3505 }
3506 }
3507
3508 if (map->fd >= 0) {
3509 pr_debug("skip map create (preset) %s: fd=%d\n",
3510 map->name, map->fd);
3511 continue;
3512 }
3513
3514 if (obj->caps.name)
3515 create_attr.name = map->name;
3516 create_attr.map_ifindex = map->map_ifindex;
3517 create_attr.map_type = def->type;
3518 create_attr.map_flags = def->map_flags;
3519 create_attr.key_size = def->key_size;
3520 create_attr.value_size = def->value_size;
3521 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
3522 !def->max_entries) {
3523 if (!nr_cpus)
3524 nr_cpus = libbpf_num_possible_cpus();
3525 if (nr_cpus < 0) {
3526 pr_warn("failed to determine number of system CPUs: %d\n",
3527 nr_cpus);
3528 err = nr_cpus;
3529 goto err_out;
3530 }
3531 pr_debug("map '%s': setting size to %d\n",
3532 map->name, nr_cpus);
3533 create_attr.max_entries = nr_cpus;
3534 } else {
3535 create_attr.max_entries = def->max_entries;
3536 }
3537 create_attr.btf_fd = 0;
3538 create_attr.btf_key_type_id = 0;
3539 create_attr.btf_value_type_id = 0;
3540 if (bpf_map_type__is_map_in_map(def->type) &&
3541 map->inner_map_fd >= 0)
3542 create_attr.inner_map_fd = map->inner_map_fd;
3543 if (bpf_map__is_struct_ops(map))
3544 create_attr.btf_vmlinux_value_type_id =
3545 map->btf_vmlinux_value_type_id;
3546
3547 if (obj->btf && !bpf_map_find_btf_info(obj, map)) {
3548 create_attr.btf_fd = btf__fd(obj->btf);
3549 create_attr.btf_key_type_id = map->btf_key_type_id;
3550 create_attr.btf_value_type_id = map->btf_value_type_id;
3551 }
3552
3553 *pfd = bpf_create_map_xattr(&create_attr);
3554 if (*pfd < 0 && (create_attr.btf_key_type_id ||
3555 create_attr.btf_value_type_id)) {
3556 err = -errno;
3557 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3558 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
3559 map->name, cp, err);
3560 create_attr.btf_fd = 0;
3561 create_attr.btf_key_type_id = 0;
3562 create_attr.btf_value_type_id = 0;
3563 map->btf_key_type_id = 0;
3564 map->btf_value_type_id = 0;
3565 *pfd = bpf_create_map_xattr(&create_attr);
3566 }
3567
3568 if (*pfd < 0) {
3569 size_t j;
3570
3571 err = -errno;
3572 err_out:
3573 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3574 pr_warn("failed to create map (name: '%s'): %s(%d)\n",
3575 map->name, cp, err);
3576 pr_perm_msg(err);
3577 for (j = 0; j < i; j++)
3578 zclose(obj->maps[j].fd);
3579 return err;
3580 }
3581
3582 if (bpf_map__is_internal(map)) {
3583 err = bpf_object__populate_internal_map(obj, map);
3584 if (err < 0) {
3585 zclose(*pfd);
3586 goto err_out;
3587 }
3588 }
3589
3590 if (map->pin_path && !map->pinned) {
3591 err = bpf_map__pin(map, NULL);
3592 if (err) {
3593 pr_warn("failed to auto-pin map name '%s' at '%s'\n",
3594 map->name, map->pin_path);
3595 return err;
3596 }
3597 }
3598
3599 pr_debug("created map %s: fd=%d\n", map->name, *pfd);
3600 }
3601
3602 return 0;
3603 }
3604
3605 static int
3606 check_btf_ext_reloc_err(struct bpf_program *prog, int err,
3607 void *btf_prog_info, const char *info_name)
3608 {
3609 if (err != -ENOENT) {
3610 pr_warn("Error in loading %s for sec %s.\n",
3611 info_name, prog->section_name);
3612 return err;
3613 }
3614
3615 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */
3616
3617 if (btf_prog_info) {
3618 /*
3619 * Some info has already been found but has problem
3620 * in the last btf_ext reloc. Must have to error out.
3621 */
3622 pr_warn("Error in relocating %s for sec %s.\n",
3623 info_name, prog->section_name);
3624 return err;
3625 }
3626
3627 /* Have problem loading the very first info. Ignore the rest. */
3628 pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n",
3629 info_name, prog->section_name, info_name);
3630 return 0;
3631 }
3632
3633 static int
3634 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
3635 const char *section_name, __u32 insn_offset)
3636 {
3637 int err;
3638
3639 if (!insn_offset || prog->func_info) {
3640 /*
3641 * !insn_offset => main program
3642 *
3643 * For sub prog, the main program's func_info has to
3644 * be loaded first (i.e. prog->func_info != NULL)
3645 */
3646 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext,
3647 section_name, insn_offset,
3648 &prog->func_info,
3649 &prog->func_info_cnt);
3650 if (err)
3651 return check_btf_ext_reloc_err(prog, err,
3652 prog->func_info,
3653 "bpf_func_info");
3654
3655 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
3656 }
3657
3658 if (!insn_offset || prog->line_info) {
3659 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
3660 section_name, insn_offset,
3661 &prog->line_info,
3662 &prog->line_info_cnt);
3663 if (err)
3664 return check_btf_ext_reloc_err(prog, err,
3665 prog->line_info,
3666 "bpf_line_info");
3667
3668 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
3669 }
3670
3671 return 0;
3672 }
3673
3674 #define BPF_CORE_SPEC_MAX_LEN 64
3675
3676 /* represents BPF CO-RE field or array element accessor */
3677 struct bpf_core_accessor {
3678 __u32 type_id; /* struct/union type or array element type */
3679 __u32 idx; /* field index or array index */
3680 const char *name; /* field name or NULL for array accessor */
3681 };
3682
3683 struct bpf_core_spec {
3684 const struct btf *btf;
3685 /* high-level spec: named fields and array indices only */
3686 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
3687 /* high-level spec length */
3688 int len;
3689 /* raw, low-level spec: 1-to-1 with accessor spec string */
3690 int raw_spec[BPF_CORE_SPEC_MAX_LEN];
3691 /* raw spec length */
3692 int raw_len;
3693 /* field bit offset represented by spec */
3694 __u32 bit_offset;
3695 };
3696
3697 static bool str_is_empty(const char *s)
3698 {
3699 return !s || !s[0];
3700 }
3701
3702 static bool is_flex_arr(const struct btf *btf,
3703 const struct bpf_core_accessor *acc,
3704 const struct btf_array *arr)
3705 {
3706 const struct btf_type *t;
3707
3708 /* not a flexible array, if not inside a struct or has non-zero size */
3709 if (!acc->name || arr->nelems > 0)
3710 return false;
3711
3712 /* has to be the last member of enclosing struct */
3713 t = btf__type_by_id(btf, acc->type_id);
3714 return acc->idx == btf_vlen(t) - 1;
3715 }
3716
3717 /*
3718 * Turn bpf_field_reloc into a low- and high-level spec representation,
3719 * validating correctness along the way, as well as calculating resulting
3720 * field bit offset, specified by accessor string. Low-level spec captures
3721 * every single level of nestedness, including traversing anonymous
3722 * struct/union members. High-level one only captures semantically meaningful
3723 * "turning points": named fields and array indicies.
3724 * E.g., for this case:
3725 *
3726 * struct sample {
3727 * int __unimportant;
3728 * struct {
3729 * int __1;
3730 * int __2;
3731 * int a[7];
3732 * };
3733 * };
3734 *
3735 * struct sample *s = ...;
3736 *
3737 * int x = &s->a[3]; // access string = '0:1:2:3'
3738 *
3739 * Low-level spec has 1:1 mapping with each element of access string (it's
3740 * just a parsed access string representation): [0, 1, 2, 3].
3741 *
3742 * High-level spec will capture only 3 points:
3743 * - intial zero-index access by pointer (&s->... is the same as &s[0]...);
3744 * - field 'a' access (corresponds to '2' in low-level spec);
3745 * - array element #3 access (corresponds to '3' in low-level spec).
3746 *
3747 */
3748 static int bpf_core_spec_parse(const struct btf *btf,
3749 __u32 type_id,
3750 const char *spec_str,
3751 struct bpf_core_spec *spec)
3752 {
3753 int access_idx, parsed_len, i;
3754 struct bpf_core_accessor *acc;
3755 const struct btf_type *t;
3756 const char *name;
3757 __u32 id;
3758 __s64 sz;
3759
3760 if (str_is_empty(spec_str) || *spec_str == ':')
3761 return -EINVAL;
3762
3763 memset(spec, 0, sizeof(*spec));
3764 spec->btf = btf;
3765
3766 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
3767 while (*spec_str) {
3768 if (*spec_str == ':')
3769 ++spec_str;
3770 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
3771 return -EINVAL;
3772 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
3773 return -E2BIG;
3774 spec_str += parsed_len;
3775 spec->raw_spec[spec->raw_len++] = access_idx;
3776 }
3777
3778 if (spec->raw_len == 0)
3779 return -EINVAL;
3780
3781 /* first spec value is always reloc type array index */
3782 t = skip_mods_and_typedefs(btf, type_id, &id);
3783 if (!t)
3784 return -EINVAL;
3785
3786 access_idx = spec->raw_spec[0];
3787 spec->spec[0].type_id = id;
3788 spec->spec[0].idx = access_idx;
3789 spec->len++;
3790
3791 sz = btf__resolve_size(btf, id);
3792 if (sz < 0)
3793 return sz;
3794 spec->bit_offset = access_idx * sz * 8;
3795
3796 for (i = 1; i < spec->raw_len; i++) {
3797 t = skip_mods_and_typedefs(btf, id, &id);
3798 if (!t)
3799 return -EINVAL;
3800
3801 access_idx = spec->raw_spec[i];
3802 acc = &spec->spec[spec->len];
3803
3804 if (btf_is_composite(t)) {
3805 const struct btf_member *m;
3806 __u32 bit_offset;
3807
3808 if (access_idx >= btf_vlen(t))
3809 return -EINVAL;
3810
3811 bit_offset = btf_member_bit_offset(t, access_idx);
3812 spec->bit_offset += bit_offset;
3813
3814 m = btf_members(t) + access_idx;
3815 if (m->name_off) {
3816 name = btf__name_by_offset(btf, m->name_off);
3817 if (str_is_empty(name))
3818 return -EINVAL;
3819
3820 acc->type_id = id;
3821 acc->idx = access_idx;
3822 acc->name = name;
3823 spec->len++;
3824 }
3825
3826 id = m->type;
3827 } else if (btf_is_array(t)) {
3828 const struct btf_array *a = btf_array(t);
3829 bool flex;
3830
3831 t = skip_mods_and_typedefs(btf, a->type, &id);
3832 if (!t)
3833 return -EINVAL;
3834
3835 flex = is_flex_arr(btf, acc - 1, a);
3836 if (!flex && access_idx >= a->nelems)
3837 return -EINVAL;
3838
3839 spec->spec[spec->len].type_id = id;
3840 spec->spec[spec->len].idx = access_idx;
3841 spec->len++;
3842
3843 sz = btf__resolve_size(btf, id);
3844 if (sz < 0)
3845 return sz;
3846 spec->bit_offset += access_idx * sz * 8;
3847 } else {
3848 pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n",
3849 type_id, spec_str, i, id, btf_kind(t));
3850 return -EINVAL;
3851 }
3852 }
3853
3854 return 0;
3855 }
3856
3857 static bool bpf_core_is_flavor_sep(const char *s)
3858 {
3859 /* check X___Y name pattern, where X and Y are not underscores */
3860 return s[0] != '_' && /* X */
3861 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */
3862 s[4] != '_'; /* Y */
3863 }
3864
3865 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
3866 * before last triple underscore. Struct name part after last triple
3867 * underscore is ignored by BPF CO-RE relocation during relocation matching.
3868 */
3869 static size_t bpf_core_essential_name_len(const char *name)
3870 {
3871 size_t n = strlen(name);
3872 int i;
3873
3874 for (i = n - 5; i >= 0; i--) {
3875 if (bpf_core_is_flavor_sep(name + i))
3876 return i + 1;
3877 }
3878 return n;
3879 }
3880
3881 /* dynamically sized list of type IDs */
3882 struct ids_vec {
3883 __u32 *data;
3884 int len;
3885 };
3886
3887 static void bpf_core_free_cands(struct ids_vec *cand_ids)
3888 {
3889 free(cand_ids->data);
3890 free(cand_ids);
3891 }
3892
3893 static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf,
3894 __u32 local_type_id,
3895 const struct btf *targ_btf)
3896 {
3897 size_t local_essent_len, targ_essent_len;
3898 const char *local_name, *targ_name;
3899 const struct btf_type *t;
3900 struct ids_vec *cand_ids;
3901 __u32 *new_ids;
3902 int i, err, n;
3903
3904 t = btf__type_by_id(local_btf, local_type_id);
3905 if (!t)
3906 return ERR_PTR(-EINVAL);
3907
3908 local_name = btf__name_by_offset(local_btf, t->name_off);
3909 if (str_is_empty(local_name))
3910 return ERR_PTR(-EINVAL);
3911 local_essent_len = bpf_core_essential_name_len(local_name);
3912
3913 cand_ids = calloc(1, sizeof(*cand_ids));
3914 if (!cand_ids)
3915 return ERR_PTR(-ENOMEM);
3916
3917 n = btf__get_nr_types(targ_btf);
3918 for (i = 1; i <= n; i++) {
3919 t = btf__type_by_id(targ_btf, i);
3920 targ_name = btf__name_by_offset(targ_btf, t->name_off);
3921 if (str_is_empty(targ_name))
3922 continue;
3923
3924 t = skip_mods_and_typedefs(targ_btf, i, NULL);
3925 if (!btf_is_composite(t) && !btf_is_array(t))
3926 continue;
3927
3928 targ_essent_len = bpf_core_essential_name_len(targ_name);
3929 if (targ_essent_len != local_essent_len)
3930 continue;
3931
3932 if (strncmp(local_name, targ_name, local_essent_len) == 0) {
3933 pr_debug("[%d] %s: found candidate [%d] %s\n",
3934 local_type_id, local_name, i, targ_name);
3935 new_ids = reallocarray(cand_ids->data,
3936 cand_ids->len + 1,
3937 sizeof(*cand_ids->data));
3938 if (!new_ids) {
3939 err = -ENOMEM;
3940 goto err_out;
3941 }
3942 cand_ids->data = new_ids;
3943 cand_ids->data[cand_ids->len++] = i;
3944 }
3945 }
3946 return cand_ids;
3947 err_out:
3948 bpf_core_free_cands(cand_ids);
3949 return ERR_PTR(err);
3950 }
3951
3952 /* Check two types for compatibility, skipping const/volatile/restrict and
3953 * typedefs, to ensure we are relocating compatible entities:
3954 * - any two STRUCTs/UNIONs are compatible and can be mixed;
3955 * - any two FWDs are compatible, if their names match (modulo flavor suffix);
3956 * - any two PTRs are always compatible;
3957 * - for ENUMs, names should be the same (ignoring flavor suffix) or at
3958 * least one of enums should be anonymous;
3959 * - for ENUMs, check sizes, names are ignored;
3960 * - for INT, size and signedness are ignored;
3961 * - for ARRAY, dimensionality is ignored, element types are checked for
3962 * compatibility recursively;
3963 * - everything else shouldn't be ever a target of relocation.
3964 * These rules are not set in stone and probably will be adjusted as we get
3965 * more experience with using BPF CO-RE relocations.
3966 */
3967 static int bpf_core_fields_are_compat(const struct btf *local_btf,
3968 __u32 local_id,
3969 const struct btf *targ_btf,
3970 __u32 targ_id)
3971 {
3972 const struct btf_type *local_type, *targ_type;
3973
3974 recur:
3975 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
3976 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
3977 if (!local_type || !targ_type)
3978 return -EINVAL;
3979
3980 if (btf_is_composite(local_type) && btf_is_composite(targ_type))
3981 return 1;
3982 if (btf_kind(local_type) != btf_kind(targ_type))
3983 return 0;
3984
3985 switch (btf_kind(local_type)) {
3986 case BTF_KIND_PTR:
3987 return 1;
3988 case BTF_KIND_FWD:
3989 case BTF_KIND_ENUM: {
3990 const char *local_name, *targ_name;
3991 size_t local_len, targ_len;
3992
3993 local_name = btf__name_by_offset(local_btf,
3994 local_type->name_off);
3995 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
3996 local_len = bpf_core_essential_name_len(local_name);
3997 targ_len = bpf_core_essential_name_len(targ_name);
3998 /* one of them is anonymous or both w/ same flavor-less names */
3999 return local_len == 0 || targ_len == 0 ||
4000 (local_len == targ_len &&
4001 strncmp(local_name, targ_name, local_len) == 0);
4002 }
4003 case BTF_KIND_INT:
4004 /* just reject deprecated bitfield-like integers; all other
4005 * integers are by default compatible between each other
4006 */
4007 return btf_int_offset(local_type) == 0 &&
4008 btf_int_offset(targ_type) == 0;
4009 case BTF_KIND_ARRAY:
4010 local_id = btf_array(local_type)->type;
4011 targ_id = btf_array(targ_type)->type;
4012 goto recur;
4013 default:
4014 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
4015 btf_kind(local_type), local_id, targ_id);
4016 return 0;
4017 }
4018 }
4019
4020 /*
4021 * Given single high-level named field accessor in local type, find
4022 * corresponding high-level accessor for a target type. Along the way,
4023 * maintain low-level spec for target as well. Also keep updating target
4024 * bit offset.
4025 *
4026 * Searching is performed through recursive exhaustive enumeration of all
4027 * fields of a struct/union. If there are any anonymous (embedded)
4028 * structs/unions, they are recursively searched as well. If field with
4029 * desired name is found, check compatibility between local and target types,
4030 * before returning result.
4031 *
4032 * 1 is returned, if field is found.
4033 * 0 is returned if no compatible field is found.
4034 * <0 is returned on error.
4035 */
4036 static int bpf_core_match_member(const struct btf *local_btf,
4037 const struct bpf_core_accessor *local_acc,
4038 const struct btf *targ_btf,
4039 __u32 targ_id,
4040 struct bpf_core_spec *spec,
4041 __u32 *next_targ_id)
4042 {
4043 const struct btf_type *local_type, *targ_type;
4044 const struct btf_member *local_member, *m;
4045 const char *local_name, *targ_name;
4046 __u32 local_id;
4047 int i, n, found;
4048
4049 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4050 if (!targ_type)
4051 return -EINVAL;
4052 if (!btf_is_composite(targ_type))
4053 return 0;
4054
4055 local_id = local_acc->type_id;
4056 local_type = btf__type_by_id(local_btf, local_id);
4057 local_member = btf_members(local_type) + local_acc->idx;
4058 local_name = btf__name_by_offset(local_btf, local_member->name_off);
4059
4060 n = btf_vlen(targ_type);
4061 m = btf_members(targ_type);
4062 for (i = 0; i < n; i++, m++) {
4063 __u32 bit_offset;
4064
4065 bit_offset = btf_member_bit_offset(targ_type, i);
4066
4067 /* too deep struct/union/array nesting */
4068 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4069 return -E2BIG;
4070
4071 /* speculate this member will be the good one */
4072 spec->bit_offset += bit_offset;
4073 spec->raw_spec[spec->raw_len++] = i;
4074
4075 targ_name = btf__name_by_offset(targ_btf, m->name_off);
4076 if (str_is_empty(targ_name)) {
4077 /* embedded struct/union, we need to go deeper */
4078 found = bpf_core_match_member(local_btf, local_acc,
4079 targ_btf, m->type,
4080 spec, next_targ_id);
4081 if (found) /* either found or error */
4082 return found;
4083 } else if (strcmp(local_name, targ_name) == 0) {
4084 /* matching named field */
4085 struct bpf_core_accessor *targ_acc;
4086
4087 targ_acc = &spec->spec[spec->len++];
4088 targ_acc->type_id = targ_id;
4089 targ_acc->idx = i;
4090 targ_acc->name = targ_name;
4091
4092 *next_targ_id = m->type;
4093 found = bpf_core_fields_are_compat(local_btf,
4094 local_member->type,
4095 targ_btf, m->type);
4096 if (!found)
4097 spec->len--; /* pop accessor */
4098 return found;
4099 }
4100 /* member turned out not to be what we looked for */
4101 spec->bit_offset -= bit_offset;
4102 spec->raw_len--;
4103 }
4104
4105 return 0;
4106 }
4107
4108 /*
4109 * Try to match local spec to a target type and, if successful, produce full
4110 * target spec (high-level, low-level + bit offset).
4111 */
4112 static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
4113 const struct btf *targ_btf, __u32 targ_id,
4114 struct bpf_core_spec *targ_spec)
4115 {
4116 const struct btf_type *targ_type;
4117 const struct bpf_core_accessor *local_acc;
4118 struct bpf_core_accessor *targ_acc;
4119 int i, sz, matched;
4120
4121 memset(targ_spec, 0, sizeof(*targ_spec));
4122 targ_spec->btf = targ_btf;
4123
4124 local_acc = &local_spec->spec[0];
4125 targ_acc = &targ_spec->spec[0];
4126
4127 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
4128 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
4129 &targ_id);
4130 if (!targ_type)
4131 return -EINVAL;
4132
4133 if (local_acc->name) {
4134 matched = bpf_core_match_member(local_spec->btf,
4135 local_acc,
4136 targ_btf, targ_id,
4137 targ_spec, &targ_id);
4138 if (matched <= 0)
4139 return matched;
4140 } else {
4141 /* for i=0, targ_id is already treated as array element
4142 * type (because it's the original struct), for others
4143 * we should find array element type first
4144 */
4145 if (i > 0) {
4146 const struct btf_array *a;
4147 bool flex;
4148
4149 if (!btf_is_array(targ_type))
4150 return 0;
4151
4152 a = btf_array(targ_type);
4153 flex = is_flex_arr(targ_btf, targ_acc - 1, a);
4154 if (!flex && local_acc->idx >= a->nelems)
4155 return 0;
4156 if (!skip_mods_and_typedefs(targ_btf, a->type,
4157 &targ_id))
4158 return -EINVAL;
4159 }
4160
4161 /* too deep struct/union/array nesting */
4162 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4163 return -E2BIG;
4164
4165 targ_acc->type_id = targ_id;
4166 targ_acc->idx = local_acc->idx;
4167 targ_acc->name = NULL;
4168 targ_spec->len++;
4169 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
4170 targ_spec->raw_len++;
4171
4172 sz = btf__resolve_size(targ_btf, targ_id);
4173 if (sz < 0)
4174 return sz;
4175 targ_spec->bit_offset += local_acc->idx * sz * 8;
4176 }
4177 }
4178
4179 return 1;
4180 }
4181
4182 static int bpf_core_calc_field_relo(const struct bpf_program *prog,
4183 const struct bpf_field_reloc *relo,
4184 const struct bpf_core_spec *spec,
4185 __u32 *val, bool *validate)
4186 {
4187 const struct bpf_core_accessor *acc = &spec->spec[spec->len - 1];
4188 const struct btf_type *t = btf__type_by_id(spec->btf, acc->type_id);
4189 __u32 byte_off, byte_sz, bit_off, bit_sz;
4190 const struct btf_member *m;
4191 const struct btf_type *mt;
4192 bool bitfield;
4193 __s64 sz;
4194
4195 /* a[n] accessor needs special handling */
4196 if (!acc->name) {
4197 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
4198 *val = spec->bit_offset / 8;
4199 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
4200 sz = btf__resolve_size(spec->btf, acc->type_id);
4201 if (sz < 0)
4202 return -EINVAL;
4203 *val = sz;
4204 } else {
4205 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
4206 bpf_program__title(prog, false),
4207 relo->kind, relo->insn_off / 8);
4208 return -EINVAL;
4209 }
4210 if (validate)
4211 *validate = true;
4212 return 0;
4213 }
4214
4215 m = btf_members(t) + acc->idx;
4216 mt = skip_mods_and_typedefs(spec->btf, m->type, NULL);
4217 bit_off = spec->bit_offset;
4218 bit_sz = btf_member_bitfield_size(t, acc->idx);
4219
4220 bitfield = bit_sz > 0;
4221 if (bitfield) {
4222 byte_sz = mt->size;
4223 byte_off = bit_off / 8 / byte_sz * byte_sz;
4224 /* figure out smallest int size necessary for bitfield load */
4225 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
4226 if (byte_sz >= 8) {
4227 /* bitfield can't be read with 64-bit read */
4228 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
4229 bpf_program__title(prog, false),
4230 relo->kind, relo->insn_off / 8);
4231 return -E2BIG;
4232 }
4233 byte_sz *= 2;
4234 byte_off = bit_off / 8 / byte_sz * byte_sz;
4235 }
4236 } else {
4237 sz = btf__resolve_size(spec->btf, m->type);
4238 if (sz < 0)
4239 return -EINVAL;
4240 byte_sz = sz;
4241 byte_off = spec->bit_offset / 8;
4242 bit_sz = byte_sz * 8;
4243 }
4244
4245 /* for bitfields, all the relocatable aspects are ambiguous and we
4246 * might disagree with compiler, so turn off validation of expected
4247 * value, except for signedness
4248 */
4249 if (validate)
4250 *validate = !bitfield;
4251
4252 switch (relo->kind) {
4253 case BPF_FIELD_BYTE_OFFSET:
4254 *val = byte_off;
4255 break;
4256 case BPF_FIELD_BYTE_SIZE:
4257 *val = byte_sz;
4258 break;
4259 case BPF_FIELD_SIGNED:
4260 /* enums will be assumed unsigned */
4261 *val = btf_is_enum(mt) ||
4262 (btf_int_encoding(mt) & BTF_INT_SIGNED);
4263 if (validate)
4264 *validate = true; /* signedness is never ambiguous */
4265 break;
4266 case BPF_FIELD_LSHIFT_U64:
4267 #if __BYTE_ORDER == __LITTLE_ENDIAN
4268 *val = 64 - (bit_off + bit_sz - byte_off * 8);
4269 #else
4270 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
4271 #endif
4272 break;
4273 case BPF_FIELD_RSHIFT_U64:
4274 *val = 64 - bit_sz;
4275 if (validate)
4276 *validate = true; /* right shift is never ambiguous */
4277 break;
4278 case BPF_FIELD_EXISTS:
4279 default:
4280 pr_warn("prog '%s': unknown relo %d at insn #%d\n",
4281 bpf_program__title(prog, false),
4282 relo->kind, relo->insn_off / 8);
4283 return -EINVAL;
4284 }
4285
4286 return 0;
4287 }
4288
4289 /*
4290 * Patch relocatable BPF instruction.
4291 *
4292 * Patched value is determined by relocation kind and target specification.
4293 * For field existence relocation target spec will be NULL if field is not
4294 * found.
4295 * Expected insn->imm value is determined using relocation kind and local
4296 * spec, and is checked before patching instruction. If actual insn->imm value
4297 * is wrong, bail out with error.
4298 *
4299 * Currently three kinds of BPF instructions are supported:
4300 * 1. rX = <imm> (assignment with immediate operand);
4301 * 2. rX += <imm> (arithmetic operations with immediate operand);
4302 */
4303 static int bpf_core_reloc_insn(struct bpf_program *prog,
4304 const struct bpf_field_reloc *relo,
4305 int relo_idx,
4306 const struct bpf_core_spec *local_spec,
4307 const struct bpf_core_spec *targ_spec)
4308 {
4309 __u32 orig_val, new_val;
4310 struct bpf_insn *insn;
4311 bool validate = true;
4312 int insn_idx, err;
4313 __u8 class;
4314
4315 if (relo->insn_off % sizeof(struct bpf_insn))
4316 return -EINVAL;
4317 insn_idx = relo->insn_off / sizeof(struct bpf_insn);
4318 insn = &prog->insns[insn_idx];
4319 class = BPF_CLASS(insn->code);
4320
4321 if (relo->kind == BPF_FIELD_EXISTS) {
4322 orig_val = 1; /* can't generate EXISTS relo w/o local field */
4323 new_val = targ_spec ? 1 : 0;
4324 } else if (!targ_spec) {
4325 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
4326 bpf_program__title(prog, false), relo_idx, insn_idx);
4327 insn->code = BPF_JMP | BPF_CALL;
4328 insn->dst_reg = 0;
4329 insn->src_reg = 0;
4330 insn->off = 0;
4331 /* if this instruction is reachable (not a dead code),
4332 * verifier will complain with the following message:
4333 * invalid func unknown#195896080
4334 */
4335 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
4336 return 0;
4337 } else {
4338 err = bpf_core_calc_field_relo(prog, relo, local_spec,
4339 &orig_val, &validate);
4340 if (err)
4341 return err;
4342 err = bpf_core_calc_field_relo(prog, relo, targ_spec,
4343 &new_val, NULL);
4344 if (err)
4345 return err;
4346 }
4347
4348 switch (class) {
4349 case BPF_ALU:
4350 case BPF_ALU64:
4351 if (BPF_SRC(insn->code) != BPF_K)
4352 return -EINVAL;
4353 if (validate && insn->imm != orig_val) {
4354 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
4355 bpf_program__title(prog, false), relo_idx,
4356 insn_idx, insn->imm, orig_val, new_val);
4357 return -EINVAL;
4358 }
4359 orig_val = insn->imm;
4360 insn->imm = new_val;
4361 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
4362 bpf_program__title(prog, false), relo_idx, insn_idx,
4363 orig_val, new_val);
4364 break;
4365 case BPF_LDX:
4366 case BPF_ST:
4367 case BPF_STX:
4368 if (validate && insn->off != orig_val) {
4369 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LD/LDX/ST/STX) value: got %u, exp %u -> %u\n",
4370 bpf_program__title(prog, false), relo_idx,
4371 insn_idx, insn->off, orig_val, new_val);
4372 return -EINVAL;
4373 }
4374 if (new_val > SHRT_MAX) {
4375 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
4376 bpf_program__title(prog, false), relo_idx,
4377 insn_idx, new_val);
4378 return -ERANGE;
4379 }
4380 orig_val = insn->off;
4381 insn->off = new_val;
4382 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
4383 bpf_program__title(prog, false), relo_idx, insn_idx,
4384 orig_val, new_val);
4385 break;
4386 default:
4387 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",
4388 bpf_program__title(prog, false), relo_idx,
4389 insn_idx, insn->code, insn->src_reg, insn->dst_reg,
4390 insn->off, insn->imm);
4391 return -EINVAL;
4392 }
4393
4394 return 0;
4395 }
4396
4397 /* Output spec definition in the format:
4398 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
4399 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
4400 */
4401 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
4402 {
4403 const struct btf_type *t;
4404 const char *s;
4405 __u32 type_id;
4406 int i;
4407
4408 type_id = spec->spec[0].type_id;
4409 t = btf__type_by_id(spec->btf, type_id);
4410 s = btf__name_by_offset(spec->btf, t->name_off);
4411 libbpf_print(level, "[%u] %s + ", type_id, s);
4412
4413 for (i = 0; i < spec->raw_len; i++)
4414 libbpf_print(level, "%d%s", spec->raw_spec[i],
4415 i == spec->raw_len - 1 ? " => " : ":");
4416
4417 libbpf_print(level, "%u.%u @ &x",
4418 spec->bit_offset / 8, spec->bit_offset % 8);
4419
4420 for (i = 0; i < spec->len; i++) {
4421 if (spec->spec[i].name)
4422 libbpf_print(level, ".%s", spec->spec[i].name);
4423 else
4424 libbpf_print(level, "[%u]", spec->spec[i].idx);
4425 }
4426
4427 }
4428
4429 static size_t bpf_core_hash_fn(const void *key, void *ctx)
4430 {
4431 return (size_t)key;
4432 }
4433
4434 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
4435 {
4436 return k1 == k2;
4437 }
4438
4439 static void *u32_as_hash_key(__u32 x)
4440 {
4441 return (void *)(uintptr_t)x;
4442 }
4443
4444 /*
4445 * CO-RE relocate single instruction.
4446 *
4447 * The outline and important points of the algorithm:
4448 * 1. For given local type, find corresponding candidate target types.
4449 * Candidate type is a type with the same "essential" name, ignoring
4450 * everything after last triple underscore (___). E.g., `sample`,
4451 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
4452 * for each other. Names with triple underscore are referred to as
4453 * "flavors" and are useful, among other things, to allow to
4454 * specify/support incompatible variations of the same kernel struct, which
4455 * might differ between different kernel versions and/or build
4456 * configurations.
4457 *
4458 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
4459 * converter, when deduplicated BTF of a kernel still contains more than
4460 * one different types with the same name. In that case, ___2, ___3, etc
4461 * are appended starting from second name conflict. But start flavors are
4462 * also useful to be defined "locally", in BPF program, to extract same
4463 * data from incompatible changes between different kernel
4464 * versions/configurations. For instance, to handle field renames between
4465 * kernel versions, one can use two flavors of the struct name with the
4466 * same common name and use conditional relocations to extract that field,
4467 * depending on target kernel version.
4468 * 2. For each candidate type, try to match local specification to this
4469 * candidate target type. Matching involves finding corresponding
4470 * high-level spec accessors, meaning that all named fields should match,
4471 * as well as all array accesses should be within the actual bounds. Also,
4472 * types should be compatible (see bpf_core_fields_are_compat for details).
4473 * 3. It is supported and expected that there might be multiple flavors
4474 * matching the spec. As long as all the specs resolve to the same set of
4475 * offsets across all candidates, there is no error. If there is any
4476 * ambiguity, CO-RE relocation will fail. This is necessary to accomodate
4477 * imprefection of BTF deduplication, which can cause slight duplication of
4478 * the same BTF type, if some directly or indirectly referenced (by
4479 * pointer) type gets resolved to different actual types in different
4480 * object files. If such situation occurs, deduplicated BTF will end up
4481 * with two (or more) structurally identical types, which differ only in
4482 * types they refer to through pointer. This should be OK in most cases and
4483 * is not an error.
4484 * 4. Candidate types search is performed by linearly scanning through all
4485 * types in target BTF. It is anticipated that this is overall more
4486 * efficient memory-wise and not significantly worse (if not better)
4487 * CPU-wise compared to prebuilding a map from all local type names to
4488 * a list of candidate type names. It's also sped up by caching resolved
4489 * list of matching candidates per each local "root" type ID, that has at
4490 * least one bpf_field_reloc associated with it. This list is shared
4491 * between multiple relocations for the same type ID and is updated as some
4492 * of the candidates are pruned due to structural incompatibility.
4493 */
4494 static int bpf_core_reloc_field(struct bpf_program *prog,
4495 const struct bpf_field_reloc *relo,
4496 int relo_idx,
4497 const struct btf *local_btf,
4498 const struct btf *targ_btf,
4499 struct hashmap *cand_cache)
4500 {
4501 const char *prog_name = bpf_program__title(prog, false);
4502 struct bpf_core_spec local_spec, cand_spec, targ_spec;
4503 const void *type_key = u32_as_hash_key(relo->type_id);
4504 const struct btf_type *local_type, *cand_type;
4505 const char *local_name, *cand_name;
4506 struct ids_vec *cand_ids;
4507 __u32 local_id, cand_id;
4508 const char *spec_str;
4509 int i, j, err;
4510
4511 local_id = relo->type_id;
4512 local_type = btf__type_by_id(local_btf, local_id);
4513 if (!local_type)
4514 return -EINVAL;
4515
4516 local_name = btf__name_by_offset(local_btf, local_type->name_off);
4517 if (str_is_empty(local_name))
4518 return -EINVAL;
4519
4520 spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
4521 if (str_is_empty(spec_str))
4522 return -EINVAL;
4523
4524 err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec);
4525 if (err) {
4526 pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n",
4527 prog_name, relo_idx, local_id, local_name, spec_str,
4528 err);
4529 return -EINVAL;
4530 }
4531
4532 pr_debug("prog '%s': relo #%d: kind %d, spec is ", prog_name, relo_idx,
4533 relo->kind);
4534 bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
4535 libbpf_print(LIBBPF_DEBUG, "\n");
4536
4537 if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) {
4538 cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf);
4539 if (IS_ERR(cand_ids)) {
4540 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld",
4541 prog_name, relo_idx, local_id, local_name,
4542 PTR_ERR(cand_ids));
4543 return PTR_ERR(cand_ids);
4544 }
4545 err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL);
4546 if (err) {
4547 bpf_core_free_cands(cand_ids);
4548 return err;
4549 }
4550 }
4551
4552 for (i = 0, j = 0; i < cand_ids->len; i++) {
4553 cand_id = cand_ids->data[i];
4554 cand_type = btf__type_by_id(targ_btf, cand_id);
4555 cand_name = btf__name_by_offset(targ_btf, cand_type->name_off);
4556
4557 err = bpf_core_spec_match(&local_spec, targ_btf,
4558 cand_id, &cand_spec);
4559 pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ",
4560 prog_name, relo_idx, i, cand_name);
4561 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
4562 libbpf_print(LIBBPF_DEBUG, ": %d\n", err);
4563 if (err < 0) {
4564 pr_warn("prog '%s': relo #%d: matching error: %d\n",
4565 prog_name, relo_idx, err);
4566 return err;
4567 }
4568 if (err == 0)
4569 continue;
4570
4571 if (j == 0) {
4572 targ_spec = cand_spec;
4573 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
4574 /* if there are many candidates, they should all
4575 * resolve to the same bit offset
4576 */
4577 pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n",
4578 prog_name, relo_idx, cand_spec.bit_offset,
4579 targ_spec.bit_offset);
4580 return -EINVAL;
4581 }
4582
4583 cand_ids->data[j++] = cand_spec.spec[0].type_id;
4584 }
4585
4586 /*
4587 * For BPF_FIELD_EXISTS relo or when used BPF program has field
4588 * existence checks or kernel version/config checks, it's expected
4589 * that we might not find any candidates. In this case, if field
4590 * wasn't found in any candidate, the list of candidates shouldn't
4591 * change at all, we'll just handle relocating appropriately,
4592 * depending on relo's kind.
4593 */
4594 if (j > 0)
4595 cand_ids->len = j;
4596
4597 /*
4598 * If no candidates were found, it might be both a programmer error,
4599 * as well as expected case, depending whether instruction w/
4600 * relocation is guarded in some way that makes it unreachable (dead
4601 * code) if relocation can't be resolved. This is handled in
4602 * bpf_core_reloc_insn() uniformly by replacing that instruction with
4603 * BPF helper call insn (using invalid helper ID). If that instruction
4604 * is indeed unreachable, then it will be ignored and eliminated by
4605 * verifier. If it was an error, then verifier will complain and point
4606 * to a specific instruction number in its log.
4607 */
4608 if (j == 0)
4609 pr_debug("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n",
4610 prog_name, relo_idx, local_id, local_name, spec_str);
4611
4612 /* bpf_core_reloc_insn should know how to handle missing targ_spec */
4613 err = bpf_core_reloc_insn(prog, relo, relo_idx, &local_spec,
4614 j ? &targ_spec : NULL);
4615 if (err) {
4616 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
4617 prog_name, relo_idx, relo->insn_off, err);
4618 return -EINVAL;
4619 }
4620
4621 return 0;
4622 }
4623
4624 static int
4625 bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path)
4626 {
4627 const struct btf_ext_info_sec *sec;
4628 const struct bpf_field_reloc *rec;
4629 const struct btf_ext_info *seg;
4630 struct hashmap_entry *entry;
4631 struct hashmap *cand_cache = NULL;
4632 struct bpf_program *prog;
4633 struct btf *targ_btf;
4634 const char *sec_name;
4635 int i, err = 0;
4636
4637 if (targ_btf_path)
4638 targ_btf = btf__parse_elf(targ_btf_path, NULL);
4639 else
4640 targ_btf = libbpf_find_kernel_btf();
4641 if (IS_ERR(targ_btf)) {
4642 pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
4643 return PTR_ERR(targ_btf);
4644 }
4645
4646 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
4647 if (IS_ERR(cand_cache)) {
4648 err = PTR_ERR(cand_cache);
4649 goto out;
4650 }
4651
4652 seg = &obj->btf_ext->field_reloc_info;
4653 for_each_btf_ext_sec(seg, sec) {
4654 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
4655 if (str_is_empty(sec_name)) {
4656 err = -EINVAL;
4657 goto out;
4658 }
4659 prog = bpf_object__find_program_by_title(obj, sec_name);
4660 if (!prog) {
4661 pr_warn("failed to find program '%s' for CO-RE offset relocation\n",
4662 sec_name);
4663 err = -EINVAL;
4664 goto out;
4665 }
4666
4667 pr_debug("prog '%s': performing %d CO-RE offset relocs\n",
4668 sec_name, sec->num_info);
4669
4670 for_each_btf_ext_rec(seg, sec, i, rec) {
4671 err = bpf_core_reloc_field(prog, rec, i, obj->btf,
4672 targ_btf, cand_cache);
4673 if (err) {
4674 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
4675 sec_name, i, err);
4676 goto out;
4677 }
4678 }
4679 }
4680
4681 out:
4682 btf__free(targ_btf);
4683 if (!IS_ERR_OR_NULL(cand_cache)) {
4684 hashmap__for_each_entry(cand_cache, entry, i) {
4685 bpf_core_free_cands(entry->value);
4686 }
4687 hashmap__free(cand_cache);
4688 }
4689 return err;
4690 }
4691
4692 static int
4693 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
4694 {
4695 int err = 0;
4696
4697 if (obj->btf_ext->field_reloc_info.len)
4698 err = bpf_core_reloc_fields(obj, targ_btf_path);
4699
4700 return err;
4701 }
4702
4703 static int
4704 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
4705 struct reloc_desc *relo)
4706 {
4707 struct bpf_insn *insn, *new_insn;
4708 struct bpf_program *text;
4709 size_t new_cnt;
4710 int err;
4711
4712 if (prog->idx != obj->efile.text_shndx && prog->main_prog_cnt == 0) {
4713 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
4714 if (!text) {
4715 pr_warn("no .text section found yet relo into text exist\n");
4716 return -LIBBPF_ERRNO__RELOC;
4717 }
4718 new_cnt = prog->insns_cnt + text->insns_cnt;
4719 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
4720 if (!new_insn) {
4721 pr_warn("oom in prog realloc\n");
4722 return -ENOMEM;
4723 }
4724 prog->insns = new_insn;
4725
4726 if (obj->btf_ext) {
4727 err = bpf_program_reloc_btf_ext(prog, obj,
4728 text->section_name,
4729 prog->insns_cnt);
4730 if (err)
4731 return err;
4732 }
4733
4734 memcpy(new_insn + prog->insns_cnt, text->insns,
4735 text->insns_cnt * sizeof(*insn));
4736 prog->main_prog_cnt = prog->insns_cnt;
4737 prog->insns_cnt = new_cnt;
4738 pr_debug("added %zd insn from %s to prog %s\n",
4739 text->insns_cnt, text->section_name,
4740 prog->section_name);
4741 }
4742
4743 insn = &prog->insns[relo->insn_idx];
4744 insn->imm += relo->sym_off / 8 + prog->main_prog_cnt - relo->insn_idx;
4745 return 0;
4746 }
4747
4748 static int
4749 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
4750 {
4751 int i, err;
4752
4753 if (!prog)
4754 return 0;
4755
4756 if (obj->btf_ext) {
4757 err = bpf_program_reloc_btf_ext(prog, obj,
4758 prog->section_name, 0);
4759 if (err)
4760 return err;
4761 }
4762
4763 if (!prog->reloc_desc)
4764 return 0;
4765
4766 for (i = 0; i < prog->nr_reloc; i++) {
4767 struct reloc_desc *relo = &prog->reloc_desc[i];
4768 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
4769
4770 if (relo->insn_idx + 1 >= (int)prog->insns_cnt) {
4771 pr_warn("relocation out of range: '%s'\n",
4772 prog->section_name);
4773 return -LIBBPF_ERRNO__RELOC;
4774 }
4775
4776 switch (relo->type) {
4777 case RELO_LD64:
4778 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
4779 insn[0].imm = obj->maps[relo->map_idx].fd;
4780 break;
4781 case RELO_DATA:
4782 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
4783 insn[1].imm = insn[0].imm + relo->sym_off;
4784 insn[0].imm = obj->maps[relo->map_idx].fd;
4785 break;
4786 case RELO_EXTERN:
4787 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
4788 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
4789 insn[1].imm = relo->sym_off;
4790 break;
4791 case RELO_CALL:
4792 err = bpf_program__reloc_text(prog, obj, relo);
4793 if (err)
4794 return err;
4795 break;
4796 default:
4797 pr_warn("relo #%d: bad relo type %d\n", i, relo->type);
4798 return -EINVAL;
4799 }
4800 }
4801
4802 zfree(&prog->reloc_desc);
4803 prog->nr_reloc = 0;
4804 return 0;
4805 }
4806
4807 static int
4808 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
4809 {
4810 struct bpf_program *prog;
4811 size_t i;
4812 int err;
4813
4814 if (obj->btf_ext) {
4815 err = bpf_object__relocate_core(obj, targ_btf_path);
4816 if (err) {
4817 pr_warn("failed to perform CO-RE relocations: %d\n",
4818 err);
4819 return err;
4820 }
4821 }
4822 /* ensure .text is relocated first, as it's going to be copied as-is
4823 * later for sub-program calls
4824 */
4825 for (i = 0; i < obj->nr_programs; i++) {
4826 prog = &obj->programs[i];
4827 if (prog->idx != obj->efile.text_shndx)
4828 continue;
4829
4830 err = bpf_program__relocate(prog, obj);
4831 if (err) {
4832 pr_warn("failed to relocate '%s'\n", prog->section_name);
4833 return err;
4834 }
4835 break;
4836 }
4837 /* now relocate everything but .text, which by now is relocated
4838 * properly, so we can copy raw sub-program instructions as is safely
4839 */
4840 for (i = 0; i < obj->nr_programs; i++) {
4841 prog = &obj->programs[i];
4842 if (prog->idx == obj->efile.text_shndx)
4843 continue;
4844
4845 err = bpf_program__relocate(prog, obj);
4846 if (err) {
4847 pr_warn("failed to relocate '%s'\n", prog->section_name);
4848 return err;
4849 }
4850 }
4851 return 0;
4852 }
4853
4854 static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj,
4855 GElf_Shdr *shdr,
4856 Elf_Data *data);
4857
4858 static int bpf_object__collect_reloc(struct bpf_object *obj)
4859 {
4860 int i, err;
4861
4862 if (!obj_elf_valid(obj)) {
4863 pr_warn("Internal error: elf object is closed\n");
4864 return -LIBBPF_ERRNO__INTERNAL;
4865 }
4866
4867 for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
4868 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
4869 Elf_Data *data = obj->efile.reloc_sects[i].data;
4870 int idx = shdr->sh_info;
4871 struct bpf_program *prog;
4872
4873 if (shdr->sh_type != SHT_REL) {
4874 pr_warn("internal error at %d\n", __LINE__);
4875 return -LIBBPF_ERRNO__INTERNAL;
4876 }
4877
4878 if (idx == obj->efile.st_ops_shndx) {
4879 err = bpf_object__collect_struct_ops_map_reloc(obj,
4880 shdr,
4881 data);
4882 if (err)
4883 return err;
4884 continue;
4885 }
4886
4887 prog = bpf_object__find_prog_by_idx(obj, idx);
4888 if (!prog) {
4889 pr_warn("relocation failed: no section(%d)\n", idx);
4890 return -LIBBPF_ERRNO__RELOC;
4891 }
4892
4893 err = bpf_program__collect_reloc(prog, shdr, data, obj);
4894 if (err)
4895 return err;
4896 }
4897 return 0;
4898 }
4899
4900 static int
4901 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
4902 char *license, __u32 kern_version, int *pfd)
4903 {
4904 struct bpf_load_program_attr load_attr;
4905 char *cp, errmsg[STRERR_BUFSIZE];
4906 size_t log_buf_size = 0;
4907 char *log_buf = NULL;
4908 int btf_fd, ret;
4909
4910 if (!insns || !insns_cnt)
4911 return -EINVAL;
4912
4913 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
4914 load_attr.prog_type = prog->type;
4915 /* old kernels might not support specifying expected_attach_type */
4916 if (!prog->caps->exp_attach_type && prog->sec_def &&
4917 prog->sec_def->is_exp_attach_type_optional)
4918 load_attr.expected_attach_type = 0;
4919 else
4920 load_attr.expected_attach_type = prog->expected_attach_type;
4921 if (prog->caps->name)
4922 load_attr.name = prog->name;
4923 load_attr.insns = insns;
4924 load_attr.insns_cnt = insns_cnt;
4925 load_attr.license = license;
4926 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
4927 prog->type == BPF_PROG_TYPE_LSM) {
4928 load_attr.attach_btf_id = prog->attach_btf_id;
4929 } else if (prog->type == BPF_PROG_TYPE_TRACING ||
4930 prog->type == BPF_PROG_TYPE_EXT) {
4931 load_attr.attach_prog_fd = prog->attach_prog_fd;
4932 load_attr.attach_btf_id = prog->attach_btf_id;
4933 } else {
4934 load_attr.kern_version = kern_version;
4935 load_attr.prog_ifindex = prog->prog_ifindex;
4936 }
4937 /* if .BTF.ext was loaded, kernel supports associated BTF for prog */
4938 if (prog->obj->btf_ext)
4939 btf_fd = bpf_object__btf_fd(prog->obj);
4940 else
4941 btf_fd = -1;
4942 load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0;
4943 load_attr.func_info = prog->func_info;
4944 load_attr.func_info_rec_size = prog->func_info_rec_size;
4945 load_attr.func_info_cnt = prog->func_info_cnt;
4946 load_attr.line_info = prog->line_info;
4947 load_attr.line_info_rec_size = prog->line_info_rec_size;
4948 load_attr.line_info_cnt = prog->line_info_cnt;
4949 load_attr.log_level = prog->log_level;
4950 load_attr.prog_flags = prog->prog_flags;
4951
4952 retry_load:
4953 if (log_buf_size) {
4954 log_buf = malloc(log_buf_size);
4955 if (!log_buf)
4956 return -ENOMEM;
4957
4958 *log_buf = 0;
4959 }
4960
4961 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);
4962
4963 if (ret >= 0) {
4964 if (log_buf && load_attr.log_level)
4965 pr_debug("verifier log:\n%s", log_buf);
4966 *pfd = ret;
4967 ret = 0;
4968 goto out;
4969 }
4970
4971 if (!log_buf || errno == ENOSPC) {
4972 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
4973 log_buf_size << 1);
4974
4975 free(log_buf);
4976 goto retry_load;
4977 }
4978 ret = -errno;
4979 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
4980 pr_warn("load bpf program failed: %s\n", cp);
4981 pr_perm_msg(ret);
4982
4983 if (log_buf && log_buf[0] != '\0') {
4984 ret = -LIBBPF_ERRNO__VERIFY;
4985 pr_warn("-- BEGIN DUMP LOG ---\n");
4986 pr_warn("\n%s\n", log_buf);
4987 pr_warn("-- END LOG --\n");
4988 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
4989 pr_warn("Program too large (%zu insns), at most %d insns\n",
4990 load_attr.insns_cnt, BPF_MAXINSNS);
4991 ret = -LIBBPF_ERRNO__PROG2BIG;
4992 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
4993 /* Wrong program type? */
4994 int fd;
4995
4996 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
4997 load_attr.expected_attach_type = 0;
4998 fd = bpf_load_program_xattr(&load_attr, NULL, 0);
4999 if (fd >= 0) {
5000 close(fd);
5001 ret = -LIBBPF_ERRNO__PROGTYPE;
5002 goto out;
5003 }
5004 }
5005
5006 out:
5007 free(log_buf);
5008 return ret;
5009 }
5010
5011 static int libbpf_find_attach_btf_id(struct bpf_program *prog);
5012
5013 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
5014 {
5015 int err = 0, fd, i, btf_id;
5016
5017 if ((prog->type == BPF_PROG_TYPE_TRACING ||
5018 prog->type == BPF_PROG_TYPE_LSM ||
5019 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
5020 btf_id = libbpf_find_attach_btf_id(prog);
5021 if (btf_id <= 0)
5022 return btf_id;
5023 prog->attach_btf_id = btf_id;
5024 }
5025
5026 if (prog->instances.nr < 0 || !prog->instances.fds) {
5027 if (prog->preprocessor) {
5028 pr_warn("Internal error: can't load program '%s'\n",
5029 prog->section_name);
5030 return -LIBBPF_ERRNO__INTERNAL;
5031 }
5032
5033 prog->instances.fds = malloc(sizeof(int));
5034 if (!prog->instances.fds) {
5035 pr_warn("Not enough memory for BPF fds\n");
5036 return -ENOMEM;
5037 }
5038 prog->instances.nr = 1;
5039 prog->instances.fds[0] = -1;
5040 }
5041
5042 if (!prog->preprocessor) {
5043 if (prog->instances.nr != 1) {
5044 pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n",
5045 prog->section_name, prog->instances.nr);
5046 }
5047 err = load_program(prog, prog->insns, prog->insns_cnt,
5048 license, kern_ver, &fd);
5049 if (!err)
5050 prog->instances.fds[0] = fd;
5051 goto out;
5052 }
5053
5054 for (i = 0; i < prog->instances.nr; i++) {
5055 struct bpf_prog_prep_result result;
5056 bpf_program_prep_t preprocessor = prog->preprocessor;
5057
5058 memset(&result, 0, sizeof(result));
5059 err = preprocessor(prog, i, prog->insns,
5060 prog->insns_cnt, &result);
5061 if (err) {
5062 pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
5063 i, prog->section_name);
5064 goto out;
5065 }
5066
5067 if (!result.new_insn_ptr || !result.new_insn_cnt) {
5068 pr_debug("Skip loading the %dth instance of program '%s'\n",
5069 i, prog->section_name);
5070 prog->instances.fds[i] = -1;
5071 if (result.pfd)
5072 *result.pfd = -1;
5073 continue;
5074 }
5075
5076 err = load_program(prog, result.new_insn_ptr,
5077 result.new_insn_cnt, license, kern_ver, &fd);
5078 if (err) {
5079 pr_warn("Loading the %dth instance of program '%s' failed\n",
5080 i, prog->section_name);
5081 goto out;
5082 }
5083
5084 if (result.pfd)
5085 *result.pfd = fd;
5086 prog->instances.fds[i] = fd;
5087 }
5088 out:
5089 if (err)
5090 pr_warn("failed to load program '%s'\n", prog->section_name);
5091 zfree(&prog->insns);
5092 prog->insns_cnt = 0;
5093 return err;
5094 }
5095
5096 static bool bpf_program__is_function_storage(const struct bpf_program *prog,
5097 const struct bpf_object *obj)
5098 {
5099 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
5100 }
5101
5102 static int
5103 bpf_object__load_progs(struct bpf_object *obj, int log_level)
5104 {
5105 size_t i;
5106 int err;
5107
5108 for (i = 0; i < obj->nr_programs; i++) {
5109 if (bpf_program__is_function_storage(&obj->programs[i], obj))
5110 continue;
5111 obj->programs[i].log_level |= log_level;
5112 err = bpf_program__load(&obj->programs[i],
5113 obj->license,
5114 obj->kern_version);
5115 if (err)
5116 return err;
5117 }
5118 return 0;
5119 }
5120
5121 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
5122
5123 static struct bpf_object *
5124 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
5125 const struct bpf_object_open_opts *opts)
5126 {
5127 const char *obj_name, *kconfig;
5128 struct bpf_program *prog;
5129 struct bpf_object *obj;
5130 char tmp_name[64];
5131 int err;
5132
5133 if (elf_version(EV_CURRENT) == EV_NONE) {
5134 pr_warn("failed to init libelf for %s\n",
5135 path ? : "(mem buf)");
5136 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
5137 }
5138
5139 if (!OPTS_VALID(opts, bpf_object_open_opts))
5140 return ERR_PTR(-EINVAL);
5141
5142 obj_name = OPTS_GET(opts, object_name, NULL);
5143 if (obj_buf) {
5144 if (!obj_name) {
5145 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
5146 (unsigned long)obj_buf,
5147 (unsigned long)obj_buf_sz);
5148 obj_name = tmp_name;
5149 }
5150 path = obj_name;
5151 pr_debug("loading object '%s' from buffer\n", obj_name);
5152 }
5153
5154 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
5155 if (IS_ERR(obj))
5156 return obj;
5157
5158 kconfig = OPTS_GET(opts, kconfig, NULL);
5159 if (kconfig) {
5160 obj->kconfig = strdup(kconfig);
5161 if (!obj->kconfig)
5162 return ERR_PTR(-ENOMEM);
5163 }
5164
5165 err = bpf_object__elf_init(obj);
5166 err = err ? : bpf_object__check_endianness(obj);
5167 err = err ? : bpf_object__elf_collect(obj);
5168 err = err ? : bpf_object__collect_externs(obj);
5169 err = err ? : bpf_object__finalize_btf(obj);
5170 err = err ? : bpf_object__init_maps(obj, opts);
5171 err = err ? : bpf_object__init_prog_names(obj);
5172 err = err ? : bpf_object__collect_reloc(obj);
5173 if (err)
5174 goto out;
5175 bpf_object__elf_finish(obj);
5176
5177 bpf_object__for_each_program(prog, obj) {
5178 prog->sec_def = find_sec_def(prog->section_name);
5179 if (!prog->sec_def)
5180 /* couldn't guess, but user might manually specify */
5181 continue;
5182
5183 bpf_program__set_type(prog, prog->sec_def->prog_type);
5184 bpf_program__set_expected_attach_type(prog,
5185 prog->sec_def->expected_attach_type);
5186
5187 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
5188 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
5189 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
5190 }
5191
5192 return obj;
5193 out:
5194 bpf_object__close(obj);
5195 return ERR_PTR(err);
5196 }
5197
5198 static struct bpf_object *
5199 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
5200 {
5201 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
5202 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
5203 );
5204
5205 /* param validation */
5206 if (!attr->file)
5207 return NULL;
5208
5209 pr_debug("loading %s\n", attr->file);
5210 return __bpf_object__open(attr->file, NULL, 0, &opts);
5211 }
5212
5213 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
5214 {
5215 return __bpf_object__open_xattr(attr, 0);
5216 }
5217
5218 struct bpf_object *bpf_object__open(const char *path)
5219 {
5220 struct bpf_object_open_attr attr = {
5221 .file = path,
5222 .prog_type = BPF_PROG_TYPE_UNSPEC,
5223 };
5224
5225 return bpf_object__open_xattr(&attr);
5226 }
5227
5228 struct bpf_object *
5229 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
5230 {
5231 if (!path)
5232 return ERR_PTR(-EINVAL);
5233
5234 pr_debug("loading %s\n", path);
5235
5236 return __bpf_object__open(path, NULL, 0, opts);
5237 }
5238
5239 struct bpf_object *
5240 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
5241 const struct bpf_object_open_opts *opts)
5242 {
5243 if (!obj_buf || obj_buf_sz == 0)
5244 return ERR_PTR(-EINVAL);
5245
5246 return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
5247 }
5248
5249 struct bpf_object *
5250 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
5251 const char *name)
5252 {
5253 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
5254 .object_name = name,
5255 /* wrong default, but backwards-compatible */
5256 .relaxed_maps = true,
5257 );
5258
5259 /* returning NULL is wrong, but backwards-compatible */
5260 if (!obj_buf || obj_buf_sz == 0)
5261 return NULL;
5262
5263 return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
5264 }
5265
5266 int bpf_object__unload(struct bpf_object *obj)
5267 {
5268 size_t i;
5269
5270 if (!obj)
5271 return -EINVAL;
5272
5273 for (i = 0; i < obj->nr_maps; i++) {
5274 zclose(obj->maps[i].fd);
5275 if (obj->maps[i].st_ops)
5276 zfree(&obj->maps[i].st_ops->kern_vdata);
5277 }
5278
5279 for (i = 0; i < obj->nr_programs; i++)
5280 bpf_program__unload(&obj->programs[i]);
5281
5282 return 0;
5283 }
5284
5285 static int bpf_object__sanitize_maps(struct bpf_object *obj)
5286 {
5287 struct bpf_map *m;
5288
5289 bpf_object__for_each_map(m, obj) {
5290 if (!bpf_map__is_internal(m))
5291 continue;
5292 if (!obj->caps.global_data) {
5293 pr_warn("kernel doesn't support global data\n");
5294 return -ENOTSUP;
5295 }
5296 if (!obj->caps.array_mmap)
5297 m->def.map_flags ^= BPF_F_MMAPABLE;
5298 }
5299
5300 return 0;
5301 }
5302
5303 static int bpf_object__resolve_externs(struct bpf_object *obj,
5304 const char *extra_kconfig)
5305 {
5306 bool need_config = false;
5307 struct extern_desc *ext;
5308 int err, i;
5309 void *data;
5310
5311 if (obj->nr_extern == 0)
5312 return 0;
5313
5314 data = obj->maps[obj->kconfig_map_idx].mmaped;
5315
5316 for (i = 0; i < obj->nr_extern; i++) {
5317 ext = &obj->externs[i];
5318
5319 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
5320 void *ext_val = data + ext->data_off;
5321 __u32 kver = get_kernel_version();
5322
5323 if (!kver) {
5324 pr_warn("failed to get kernel version\n");
5325 return -EINVAL;
5326 }
5327 err = set_ext_value_num(ext, ext_val, kver);
5328 if (err)
5329 return err;
5330 pr_debug("extern %s=0x%x\n", ext->name, kver);
5331 } else if (strncmp(ext->name, "CONFIG_", 7) == 0) {
5332 need_config = true;
5333 } else {
5334 pr_warn("unrecognized extern '%s'\n", ext->name);
5335 return -EINVAL;
5336 }
5337 }
5338 if (need_config && extra_kconfig) {
5339 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, data);
5340 if (err)
5341 return -EINVAL;
5342 need_config = false;
5343 for (i = 0; i < obj->nr_extern; i++) {
5344 ext = &obj->externs[i];
5345 if (!ext->is_set) {
5346 need_config = true;
5347 break;
5348 }
5349 }
5350 }
5351 if (need_config) {
5352 err = bpf_object__read_kconfig_file(obj, data);
5353 if (err)
5354 return -EINVAL;
5355 }
5356 for (i = 0; i < obj->nr_extern; i++) {
5357 ext = &obj->externs[i];
5358
5359 if (!ext->is_set && !ext->is_weak) {
5360 pr_warn("extern %s (strong) not resolved\n", ext->name);
5361 return -ESRCH;
5362 } else if (!ext->is_set) {
5363 pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
5364 ext->name);
5365 }
5366 }
5367
5368 return 0;
5369 }
5370
5371 int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
5372 {
5373 struct bpf_object *obj;
5374 int err, i;
5375
5376 if (!attr)
5377 return -EINVAL;
5378 obj = attr->obj;
5379 if (!obj)
5380 return -EINVAL;
5381
5382 if (obj->loaded) {
5383 pr_warn("object should not be loaded twice\n");
5384 return -EINVAL;
5385 }
5386
5387 obj->loaded = true;
5388
5389 err = bpf_object__probe_caps(obj);
5390 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
5391 err = err ? : bpf_object__sanitize_and_load_btf(obj);
5392 err = err ? : bpf_object__sanitize_maps(obj);
5393 err = err ? : bpf_object__load_vmlinux_btf(obj);
5394 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
5395 err = err ? : bpf_object__create_maps(obj);
5396 err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
5397 err = err ? : bpf_object__load_progs(obj, attr->log_level);
5398
5399 btf__free(obj->btf_vmlinux);
5400 obj->btf_vmlinux = NULL;
5401
5402 if (err)
5403 goto out;
5404
5405 return 0;
5406 out:
5407 /* unpin any maps that were auto-pinned during load */
5408 for (i = 0; i < obj->nr_maps; i++)
5409 if (obj->maps[i].pinned && !obj->maps[i].reused)
5410 bpf_map__unpin(&obj->maps[i], NULL);
5411
5412 bpf_object__unload(obj);
5413 pr_warn("failed to load object '%s'\n", obj->path);
5414 return err;
5415 }
5416
5417 int bpf_object__load(struct bpf_object *obj)
5418 {
5419 struct bpf_object_load_attr attr = {
5420 .obj = obj,
5421 };
5422
5423 return bpf_object__load_xattr(&attr);
5424 }
5425
5426 static int make_parent_dir(const char *path)
5427 {
5428 char *cp, errmsg[STRERR_BUFSIZE];
5429 char *dname, *dir;
5430 int err = 0;
5431
5432 dname = strdup(path);
5433 if (dname == NULL)
5434 return -ENOMEM;
5435
5436 dir = dirname(dname);
5437 if (mkdir(dir, 0700) && errno != EEXIST)
5438 err = -errno;
5439
5440 free(dname);
5441 if (err) {
5442 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5443 pr_warn("failed to mkdir %s: %s\n", path, cp);
5444 }
5445 return err;
5446 }
5447
5448 static int check_path(const char *path)
5449 {
5450 char *cp, errmsg[STRERR_BUFSIZE];
5451 struct statfs st_fs;
5452 char *dname, *dir;
5453 int err = 0;
5454
5455 if (path == NULL)
5456 return -EINVAL;
5457
5458 dname = strdup(path);
5459 if (dname == NULL)
5460 return -ENOMEM;
5461
5462 dir = dirname(dname);
5463 if (statfs(dir, &st_fs)) {
5464 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
5465 pr_warn("failed to statfs %s: %s\n", dir, cp);
5466 err = -errno;
5467 }
5468 free(dname);
5469
5470 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
5471 pr_warn("specified path %s is not on BPF FS\n", path);
5472 err = -EINVAL;
5473 }
5474
5475 return err;
5476 }
5477
5478 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
5479 int instance)
5480 {
5481 char *cp, errmsg[STRERR_BUFSIZE];
5482 int err;
5483
5484 err = make_parent_dir(path);
5485 if (err)
5486 return err;
5487
5488 err = check_path(path);
5489 if (err)
5490 return err;
5491
5492 if (prog == NULL) {
5493 pr_warn("invalid program pointer\n");
5494 return -EINVAL;
5495 }
5496
5497 if (instance < 0 || instance >= prog->instances.nr) {
5498 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
5499 instance, prog->section_name, prog->instances.nr);
5500 return -EINVAL;
5501 }
5502
5503 if (bpf_obj_pin(prog->instances.fds[instance], path)) {
5504 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
5505 pr_warn("failed to pin program: %s\n", cp);
5506 return -errno;
5507 }
5508 pr_debug("pinned program '%s'\n", path);
5509
5510 return 0;
5511 }
5512
5513 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
5514 int instance)
5515 {
5516 int err;
5517
5518 err = check_path(path);
5519 if (err)
5520 return err;
5521
5522 if (prog == NULL) {
5523 pr_warn("invalid program pointer\n");
5524 return -EINVAL;
5525 }
5526
5527 if (instance < 0 || instance >= prog->instances.nr) {
5528 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
5529 instance, prog->section_name, prog->instances.nr);
5530 return -EINVAL;
5531 }
5532
5533 err = unlink(path);
5534 if (err != 0)
5535 return -errno;
5536 pr_debug("unpinned program '%s'\n", path);
5537
5538 return 0;
5539 }
5540
5541 int bpf_program__pin(struct bpf_program *prog, const char *path)
5542 {
5543 int i, err;
5544
5545 err = make_parent_dir(path);
5546 if (err)
5547 return err;
5548
5549 err = check_path(path);
5550 if (err)
5551 return err;
5552
5553 if (prog == NULL) {
5554 pr_warn("invalid program pointer\n");
5555 return -EINVAL;
5556 }
5557
5558 if (prog->instances.nr <= 0) {
5559 pr_warn("no instances of prog %s to pin\n",
5560 prog->section_name);
5561 return -EINVAL;
5562 }
5563
5564 if (prog->instances.nr == 1) {
5565 /* don't create subdirs when pinning single instance */
5566 return bpf_program__pin_instance(prog, path, 0);
5567 }
5568
5569 for (i = 0; i < prog->instances.nr; i++) {
5570 char buf[PATH_MAX];
5571 int len;
5572
5573 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
5574 if (len < 0) {
5575 err = -EINVAL;
5576 goto err_unpin;
5577 } else if (len >= PATH_MAX) {
5578 err = -ENAMETOOLONG;
5579 goto err_unpin;
5580 }
5581
5582 err = bpf_program__pin_instance(prog, buf, i);
5583 if (err)
5584 goto err_unpin;
5585 }
5586
5587 return 0;
5588
5589 err_unpin:
5590 for (i = i - 1; i >= 0; i--) {
5591 char buf[PATH_MAX];
5592 int len;
5593
5594 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
5595 if (len < 0)
5596 continue;
5597 else if (len >= PATH_MAX)
5598 continue;
5599
5600 bpf_program__unpin_instance(prog, buf, i);
5601 }
5602
5603 rmdir(path);
5604
5605 return err;
5606 }
5607
5608 int bpf_program__unpin(struct bpf_program *prog, const char *path)
5609 {
5610 int i, err;
5611
5612 err = check_path(path);
5613 if (err)
5614 return err;
5615
5616 if (prog == NULL) {
5617 pr_warn("invalid program pointer\n");
5618 return -EINVAL;
5619 }
5620
5621 if (prog->instances.nr <= 0) {
5622 pr_warn("no instances of prog %s to pin\n",
5623 prog->section_name);
5624 return -EINVAL;
5625 }
5626
5627 if (prog->instances.nr == 1) {
5628 /* don't create subdirs when pinning single instance */
5629 return bpf_program__unpin_instance(prog, path, 0);
5630 }
5631
5632 for (i = 0; i < prog->instances.nr; i++) {
5633 char buf[PATH_MAX];
5634 int len;
5635
5636 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
5637 if (len < 0)
5638 return -EINVAL;
5639 else if (len >= PATH_MAX)
5640 return -ENAMETOOLONG;
5641
5642 err = bpf_program__unpin_instance(prog, buf, i);
5643 if (err)
5644 return err;
5645 }
5646
5647 err = rmdir(path);
5648 if (err)
5649 return -errno;
5650
5651 return 0;
5652 }
5653
5654 int bpf_map__pin(struct bpf_map *map, const char *path)
5655 {
5656 char *cp, errmsg[STRERR_BUFSIZE];
5657 int err;
5658
5659 if (map == NULL) {
5660 pr_warn("invalid map pointer\n");
5661 return -EINVAL;
5662 }
5663
5664 if (map->pin_path) {
5665 if (path && strcmp(path, map->pin_path)) {
5666 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
5667 bpf_map__name(map), map->pin_path, path);
5668 return -EINVAL;
5669 } else if (map->pinned) {
5670 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
5671 bpf_map__name(map), map->pin_path);
5672 return 0;
5673 }
5674 } else {
5675 if (!path) {
5676 pr_warn("missing a path to pin map '%s' at\n",
5677 bpf_map__name(map));
5678 return -EINVAL;
5679 } else if (map->pinned) {
5680 pr_warn("map '%s' already pinned\n", bpf_map__name(map));
5681 return -EEXIST;
5682 }
5683
5684 map->pin_path = strdup(path);
5685 if (!map->pin_path) {
5686 err = -errno;
5687 goto out_err;
5688 }
5689 }
5690
5691 err = make_parent_dir(map->pin_path);
5692 if (err)
5693 return err;
5694
5695 err = check_path(map->pin_path);
5696 if (err)
5697 return err;
5698
5699 if (bpf_obj_pin(map->fd, map->pin_path)) {
5700 err = -errno;
5701 goto out_err;
5702 }
5703
5704 map->pinned = true;
5705 pr_debug("pinned map '%s'\n", map->pin_path);
5706
5707 return 0;
5708
5709 out_err:
5710 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5711 pr_warn("failed to pin map: %s\n", cp);
5712 return err;
5713 }
5714
5715 int bpf_map__unpin(struct bpf_map *map, const char *path)
5716 {
5717 int err;
5718
5719 if (map == NULL) {
5720 pr_warn("invalid map pointer\n");
5721 return -EINVAL;
5722 }
5723
5724 if (map->pin_path) {
5725 if (path && strcmp(path, map->pin_path)) {
5726 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
5727 bpf_map__name(map), map->pin_path, path);
5728 return -EINVAL;
5729 }
5730 path = map->pin_path;
5731 } else if (!path) {
5732 pr_warn("no path to unpin map '%s' from\n",
5733 bpf_map__name(map));
5734 return -EINVAL;
5735 }
5736
5737 err = check_path(path);
5738 if (err)
5739 return err;
5740
5741 err = unlink(path);
5742 if (err != 0)
5743 return -errno;
5744
5745 map->pinned = false;
5746 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
5747
5748 return 0;
5749 }
5750
5751 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
5752 {
5753 char *new = NULL;
5754
5755 if (path) {
5756 new = strdup(path);
5757 if (!new)
5758 return -errno;
5759 }
5760
5761 free(map->pin_path);
5762 map->pin_path = new;
5763 return 0;
5764 }
5765
5766 const char *bpf_map__get_pin_path(const struct bpf_map *map)
5767 {
5768 return map->pin_path;
5769 }
5770
5771 bool bpf_map__is_pinned(const struct bpf_map *map)
5772 {
5773 return map->pinned;
5774 }
5775
5776 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
5777 {
5778 struct bpf_map *map;
5779 int err;
5780
5781 if (!obj)
5782 return -ENOENT;
5783
5784 if (!obj->loaded) {
5785 pr_warn("object not yet loaded; load it first\n");
5786 return -ENOENT;
5787 }
5788
5789 bpf_object__for_each_map(map, obj) {
5790 char *pin_path = NULL;
5791 char buf[PATH_MAX];
5792
5793 if (path) {
5794 int len;
5795
5796 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5797 bpf_map__name(map));
5798 if (len < 0) {
5799 err = -EINVAL;
5800 goto err_unpin_maps;
5801 } else if (len >= PATH_MAX) {
5802 err = -ENAMETOOLONG;
5803 goto err_unpin_maps;
5804 }
5805 pin_path = buf;
5806 } else if (!map->pin_path) {
5807 continue;
5808 }
5809
5810 err = bpf_map__pin(map, pin_path);
5811 if (err)
5812 goto err_unpin_maps;
5813 }
5814
5815 return 0;
5816
5817 err_unpin_maps:
5818 while ((map = bpf_map__prev(map, obj))) {
5819 if (!map->pin_path)
5820 continue;
5821
5822 bpf_map__unpin(map, NULL);
5823 }
5824
5825 return err;
5826 }
5827
5828 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
5829 {
5830 struct bpf_map *map;
5831 int err;
5832
5833 if (!obj)
5834 return -ENOENT;
5835
5836 bpf_object__for_each_map(map, obj) {
5837 char *pin_path = NULL;
5838 char buf[PATH_MAX];
5839
5840 if (path) {
5841 int len;
5842
5843 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5844 bpf_map__name(map));
5845 if (len < 0)
5846 return -EINVAL;
5847 else if (len >= PATH_MAX)
5848 return -ENAMETOOLONG;
5849 pin_path = buf;
5850 } else if (!map->pin_path) {
5851 continue;
5852 }
5853
5854 err = bpf_map__unpin(map, pin_path);
5855 if (err)
5856 return err;
5857 }
5858
5859 return 0;
5860 }
5861
5862 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
5863 {
5864 struct bpf_program *prog;
5865 int err;
5866
5867 if (!obj)
5868 return -ENOENT;
5869
5870 if (!obj->loaded) {
5871 pr_warn("object not yet loaded; load it first\n");
5872 return -ENOENT;
5873 }
5874
5875 bpf_object__for_each_program(prog, obj) {
5876 char buf[PATH_MAX];
5877 int len;
5878
5879 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5880 prog->pin_name);
5881 if (len < 0) {
5882 err = -EINVAL;
5883 goto err_unpin_programs;
5884 } else if (len >= PATH_MAX) {
5885 err = -ENAMETOOLONG;
5886 goto err_unpin_programs;
5887 }
5888
5889 err = bpf_program__pin(prog, buf);
5890 if (err)
5891 goto err_unpin_programs;
5892 }
5893
5894 return 0;
5895
5896 err_unpin_programs:
5897 while ((prog = bpf_program__prev(prog, obj))) {
5898 char buf[PATH_MAX];
5899 int len;
5900
5901 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5902 prog->pin_name);
5903 if (len < 0)
5904 continue;
5905 else if (len >= PATH_MAX)
5906 continue;
5907
5908 bpf_program__unpin(prog, buf);
5909 }
5910
5911 return err;
5912 }
5913
5914 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
5915 {
5916 struct bpf_program *prog;
5917 int err;
5918
5919 if (!obj)
5920 return -ENOENT;
5921
5922 bpf_object__for_each_program(prog, obj) {
5923 char buf[PATH_MAX];
5924 int len;
5925
5926 len = snprintf(buf, PATH_MAX, "%s/%s", path,
5927 prog->pin_name);
5928 if (len < 0)
5929 return -EINVAL;
5930 else if (len >= PATH_MAX)
5931 return -ENAMETOOLONG;
5932
5933 err = bpf_program__unpin(prog, buf);
5934 if (err)
5935 return err;
5936 }
5937
5938 return 0;
5939 }
5940
5941 int bpf_object__pin(struct bpf_object *obj, const char *path)
5942 {
5943 int err;
5944
5945 err = bpf_object__pin_maps(obj, path);
5946 if (err)
5947 return err;
5948
5949 err = bpf_object__pin_programs(obj, path);
5950 if (err) {
5951 bpf_object__unpin_maps(obj, path);
5952 return err;
5953 }
5954
5955 return 0;
5956 }
5957
5958 void bpf_object__close(struct bpf_object *obj)
5959 {
5960 size_t i;
5961
5962 if (!obj)
5963 return;
5964
5965 if (obj->clear_priv)
5966 obj->clear_priv(obj, obj->priv);
5967
5968 bpf_object__elf_finish(obj);
5969 bpf_object__unload(obj);
5970 btf__free(obj->btf);
5971 btf_ext__free(obj->btf_ext);
5972
5973 for (i = 0; i < obj->nr_maps; i++) {
5974 struct bpf_map *map = &obj->maps[i];
5975
5976 if (map->clear_priv)
5977 map->clear_priv(map, map->priv);
5978 map->priv = NULL;
5979 map->clear_priv = NULL;
5980
5981 if (map->mmaped) {
5982 munmap(map->mmaped, bpf_map_mmap_sz(map));
5983 map->mmaped = NULL;
5984 }
5985
5986 if (map->st_ops) {
5987 zfree(&map->st_ops->data);
5988 zfree(&map->st_ops->progs);
5989 zfree(&map->st_ops->kern_func_off);
5990 zfree(&map->st_ops);
5991 }
5992
5993 zfree(&map->name);
5994 zfree(&map->pin_path);
5995 }
5996
5997 zfree(&obj->kconfig);
5998 zfree(&obj->externs);
5999 obj->nr_extern = 0;
6000
6001 zfree(&obj->maps);
6002 obj->nr_maps = 0;
6003
6004 if (obj->programs && obj->nr_programs) {
6005 for (i = 0; i < obj->nr_programs; i++)
6006 bpf_program__exit(&obj->programs[i]);
6007 }
6008 zfree(&obj->programs);
6009
6010 list_del(&obj->list);
6011 free(obj);
6012 }
6013
6014 struct bpf_object *
6015 bpf_object__next(struct bpf_object *prev)
6016 {
6017 struct bpf_object *next;
6018
6019 if (!prev)
6020 next = list_first_entry(&bpf_objects_list,
6021 struct bpf_object,
6022 list);
6023 else
6024 next = list_next_entry(prev, list);
6025
6026 /* Empty list is noticed here so don't need checking on entry. */
6027 if (&next->list == &bpf_objects_list)
6028 return NULL;
6029
6030 return next;
6031 }
6032
6033 const char *bpf_object__name(const struct bpf_object *obj)
6034 {
6035 return obj ? obj->name : ERR_PTR(-EINVAL);
6036 }
6037
6038 unsigned int bpf_object__kversion(const struct bpf_object *obj)
6039 {
6040 return obj ? obj->kern_version : 0;
6041 }
6042
6043 struct btf *bpf_object__btf(const struct bpf_object *obj)
6044 {
6045 return obj ? obj->btf : NULL;
6046 }
6047
6048 int bpf_object__btf_fd(const struct bpf_object *obj)
6049 {
6050 return obj->btf ? btf__fd(obj->btf) : -1;
6051 }
6052
6053 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
6054 bpf_object_clear_priv_t clear_priv)
6055 {
6056 if (obj->priv && obj->clear_priv)
6057 obj->clear_priv(obj, obj->priv);
6058
6059 obj->priv = priv;
6060 obj->clear_priv = clear_priv;
6061 return 0;
6062 }
6063
6064 void *bpf_object__priv(const struct bpf_object *obj)
6065 {
6066 return obj ? obj->priv : ERR_PTR(-EINVAL);
6067 }
6068
6069 static struct bpf_program *
6070 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
6071 bool forward)
6072 {
6073 size_t nr_programs = obj->nr_programs;
6074 ssize_t idx;
6075
6076 if (!nr_programs)
6077 return NULL;
6078
6079 if (!p)
6080 /* Iter from the beginning */
6081 return forward ? &obj->programs[0] :
6082 &obj->programs[nr_programs - 1];
6083
6084 if (p->obj != obj) {
6085 pr_warn("error: program handler doesn't match object\n");
6086 return NULL;
6087 }
6088
6089 idx = (p - obj->programs) + (forward ? 1 : -1);
6090 if (idx >= obj->nr_programs || idx < 0)
6091 return NULL;
6092 return &obj->programs[idx];
6093 }
6094
6095 struct bpf_program *
6096 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
6097 {
6098 struct bpf_program *prog = prev;
6099
6100 do {
6101 prog = __bpf_program__iter(prog, obj, true);
6102 } while (prog && bpf_program__is_function_storage(prog, obj));
6103
6104 return prog;
6105 }
6106
6107 struct bpf_program *
6108 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
6109 {
6110 struct bpf_program *prog = next;
6111
6112 do {
6113 prog = __bpf_program__iter(prog, obj, false);
6114 } while (prog && bpf_program__is_function_storage(prog, obj));
6115
6116 return prog;
6117 }
6118
6119 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
6120 bpf_program_clear_priv_t clear_priv)
6121 {
6122 if (prog->priv && prog->clear_priv)
6123 prog->clear_priv(prog, prog->priv);
6124
6125 prog->priv = priv;
6126 prog->clear_priv = clear_priv;
6127 return 0;
6128 }
6129
6130 void *bpf_program__priv(const struct bpf_program *prog)
6131 {
6132 return prog ? prog->priv : ERR_PTR(-EINVAL);
6133 }
6134
6135 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
6136 {
6137 prog->prog_ifindex = ifindex;
6138 }
6139
6140 const char *bpf_program__name(const struct bpf_program *prog)
6141 {
6142 return prog->name;
6143 }
6144
6145 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
6146 {
6147 const char *title;
6148
6149 title = prog->section_name;
6150 if (needs_copy) {
6151 title = strdup(title);
6152 if (!title) {
6153 pr_warn("failed to strdup program title\n");
6154 return ERR_PTR(-ENOMEM);
6155 }
6156 }
6157
6158 return title;
6159 }
6160
6161 int bpf_program__fd(const struct bpf_program *prog)
6162 {
6163 return bpf_program__nth_fd(prog, 0);
6164 }
6165
6166 size_t bpf_program__size(const struct bpf_program *prog)
6167 {
6168 return prog->insns_cnt * sizeof(struct bpf_insn);
6169 }
6170
6171 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
6172 bpf_program_prep_t prep)
6173 {
6174 int *instances_fds;
6175
6176 if (nr_instances <= 0 || !prep)
6177 return -EINVAL;
6178
6179 if (prog->instances.nr > 0 || prog->instances.fds) {
6180 pr_warn("Can't set pre-processor after loading\n");
6181 return -EINVAL;
6182 }
6183
6184 instances_fds = malloc(sizeof(int) * nr_instances);
6185 if (!instances_fds) {
6186 pr_warn("alloc memory failed for fds\n");
6187 return -ENOMEM;
6188 }
6189
6190 /* fill all fd with -1 */
6191 memset(instances_fds, -1, sizeof(int) * nr_instances);
6192
6193 prog->instances.nr = nr_instances;
6194 prog->instances.fds = instances_fds;
6195 prog->preprocessor = prep;
6196 return 0;
6197 }
6198
6199 int bpf_program__nth_fd(const struct bpf_program *prog, int n)
6200 {
6201 int fd;
6202
6203 if (!prog)
6204 return -EINVAL;
6205
6206 if (n >= prog->instances.nr || n < 0) {
6207 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
6208 n, prog->section_name, prog->instances.nr);
6209 return -EINVAL;
6210 }
6211
6212 fd = prog->instances.fds[n];
6213 if (fd < 0) {
6214 pr_warn("%dth instance of program '%s' is invalid\n",
6215 n, prog->section_name);
6216 return -ENOENT;
6217 }
6218
6219 return fd;
6220 }
6221
6222 enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
6223 {
6224 return prog->type;
6225 }
6226
6227 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
6228 {
6229 prog->type = type;
6230 }
6231
6232 static bool bpf_program__is_type(const struct bpf_program *prog,
6233 enum bpf_prog_type type)
6234 {
6235 return prog ? (prog->type == type) : false;
6236 }
6237
6238 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \
6239 int bpf_program__set_##NAME(struct bpf_program *prog) \
6240 { \
6241 if (!prog) \
6242 return -EINVAL; \
6243 bpf_program__set_type(prog, TYPE); \
6244 return 0; \
6245 } \
6246 \
6247 bool bpf_program__is_##NAME(const struct bpf_program *prog) \
6248 { \
6249 return bpf_program__is_type(prog, TYPE); \
6250 } \
6251
6252 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
6253 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
6254 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
6255 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
6256 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
6257 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
6258 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
6259 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
6260 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
6261 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
6262 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
6263 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
6264
6265 enum bpf_attach_type
6266 bpf_program__get_expected_attach_type(struct bpf_program *prog)
6267 {
6268 return prog->expected_attach_type;
6269 }
6270
6271 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
6272 enum bpf_attach_type type)
6273 {
6274 prog->expected_attach_type = type;
6275 }
6276
6277 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \
6278 attachable, attach_btf) \
6279 { \
6280 .sec = string, \
6281 .len = sizeof(string) - 1, \
6282 .prog_type = ptype, \
6283 .expected_attach_type = eatype, \
6284 .is_exp_attach_type_optional = eatype_optional, \
6285 .is_attachable = attachable, \
6286 .is_attach_btf = attach_btf, \
6287 }
6288
6289 /* Programs that can NOT be attached. */
6290 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
6291
6292 /* Programs that can be attached. */
6293 #define BPF_APROG_SEC(string, ptype, atype) \
6294 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0)
6295
6296 /* Programs that must specify expected attach type at load time. */
6297 #define BPF_EAPROG_SEC(string, ptype, eatype) \
6298 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0)
6299
6300 /* Programs that use BTF to identify attach point */
6301 #define BPF_PROG_BTF(string, ptype, eatype) \
6302 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1)
6303
6304 /* Programs that can be attached but attach type can't be identified by section
6305 * name. Kept for backward compatibility.
6306 */
6307 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
6308
6309 #define SEC_DEF(sec_pfx, ptype, ...) { \
6310 .sec = sec_pfx, \
6311 .len = sizeof(sec_pfx) - 1, \
6312 .prog_type = BPF_PROG_TYPE_##ptype, \
6313 __VA_ARGS__ \
6314 }
6315
6316 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
6317 struct bpf_program *prog);
6318 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
6319 struct bpf_program *prog);
6320 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
6321 struct bpf_program *prog);
6322 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
6323 struct bpf_program *prog);
6324 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
6325 struct bpf_program *prog);
6326
6327 static const struct bpf_sec_def section_defs[] = {
6328 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
6329 BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT),
6330 SEC_DEF("kprobe/", KPROBE,
6331 .attach_fn = attach_kprobe),
6332 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE),
6333 SEC_DEF("kretprobe/", KPROBE,
6334 .attach_fn = attach_kprobe),
6335 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE),
6336 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
6337 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
6338 SEC_DEF("tracepoint/", TRACEPOINT,
6339 .attach_fn = attach_tp),
6340 SEC_DEF("tp/", TRACEPOINT,
6341 .attach_fn = attach_tp),
6342 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
6343 .attach_fn = attach_raw_tp),
6344 SEC_DEF("raw_tp/", RAW_TRACEPOINT,
6345 .attach_fn = attach_raw_tp),
6346 SEC_DEF("tp_btf/", TRACING,
6347 .expected_attach_type = BPF_TRACE_RAW_TP,
6348 .is_attach_btf = true,
6349 .attach_fn = attach_trace),
6350 SEC_DEF("fentry/", TRACING,
6351 .expected_attach_type = BPF_TRACE_FENTRY,
6352 .is_attach_btf = true,
6353 .attach_fn = attach_trace),
6354 SEC_DEF("fmod_ret/", TRACING,
6355 .expected_attach_type = BPF_MODIFY_RETURN,
6356 .is_attach_btf = true,
6357 .attach_fn = attach_trace),
6358 SEC_DEF("fexit/", TRACING,
6359 .expected_attach_type = BPF_TRACE_FEXIT,
6360 .is_attach_btf = true,
6361 .attach_fn = attach_trace),
6362 SEC_DEF("freplace/", EXT,
6363 .is_attach_btf = true,
6364 .attach_fn = attach_trace),
6365 SEC_DEF("lsm/", LSM,
6366 .is_attach_btf = true,
6367 .expected_attach_type = BPF_LSM_MAC,
6368 .attach_fn = attach_lsm),
6369 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP),
6370 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
6371 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
6372 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
6373 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
6374 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL),
6375 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB,
6376 BPF_CGROUP_INET_INGRESS),
6377 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB,
6378 BPF_CGROUP_INET_EGRESS),
6379 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
6380 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK,
6381 BPF_CGROUP_INET_SOCK_CREATE),
6382 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK,
6383 BPF_CGROUP_INET4_POST_BIND),
6384 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK,
6385 BPF_CGROUP_INET6_POST_BIND),
6386 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE,
6387 BPF_CGROUP_DEVICE),
6388 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS,
6389 BPF_CGROUP_SOCK_OPS),
6390 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB,
6391 BPF_SK_SKB_STREAM_PARSER),
6392 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB,
6393 BPF_SK_SKB_STREAM_VERDICT),
6394 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB),
6395 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG,
6396 BPF_SK_MSG_VERDICT),
6397 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2,
6398 BPF_LIRC_MODE2),
6399 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR,
6400 BPF_FLOW_DISSECTOR),
6401 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6402 BPF_CGROUP_INET4_BIND),
6403 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6404 BPF_CGROUP_INET6_BIND),
6405 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6406 BPF_CGROUP_INET4_CONNECT),
6407 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6408 BPF_CGROUP_INET6_CONNECT),
6409 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6410 BPF_CGROUP_UDP4_SENDMSG),
6411 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6412 BPF_CGROUP_UDP6_SENDMSG),
6413 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6414 BPF_CGROUP_UDP4_RECVMSG),
6415 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6416 BPF_CGROUP_UDP6_RECVMSG),
6417 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL,
6418 BPF_CGROUP_SYSCTL),
6419 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
6420 BPF_CGROUP_GETSOCKOPT),
6421 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
6422 BPF_CGROUP_SETSOCKOPT),
6423 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS),
6424 };
6425
6426 #undef BPF_PROG_SEC_IMPL
6427 #undef BPF_PROG_SEC
6428 #undef BPF_APROG_SEC
6429 #undef BPF_EAPROG_SEC
6430 #undef BPF_APROG_COMPAT
6431 #undef SEC_DEF
6432
6433 #define MAX_TYPE_NAME_SIZE 32
6434
6435 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
6436 {
6437 int i, n = ARRAY_SIZE(section_defs);
6438
6439 for (i = 0; i < n; i++) {
6440 if (strncmp(sec_name,
6441 section_defs[i].sec, section_defs[i].len))
6442 continue;
6443 return &section_defs[i];
6444 }
6445 return NULL;
6446 }
6447
6448 static char *libbpf_get_type_names(bool attach_type)
6449 {
6450 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
6451 char *buf;
6452
6453 buf = malloc(len);
6454 if (!buf)
6455 return NULL;
6456
6457 buf[0] = '\0';
6458 /* Forge string buf with all available names */
6459 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
6460 if (attach_type && !section_defs[i].is_attachable)
6461 continue;
6462
6463 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
6464 free(buf);
6465 return NULL;
6466 }
6467 strcat(buf, " ");
6468 strcat(buf, section_defs[i].sec);
6469 }
6470
6471 return buf;
6472 }
6473
6474 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
6475 enum bpf_attach_type *expected_attach_type)
6476 {
6477 const struct bpf_sec_def *sec_def;
6478 char *type_names;
6479
6480 if (!name)
6481 return -EINVAL;
6482
6483 sec_def = find_sec_def(name);
6484 if (sec_def) {
6485 *prog_type = sec_def->prog_type;
6486 *expected_attach_type = sec_def->expected_attach_type;
6487 return 0;
6488 }
6489
6490 pr_debug("failed to guess program type from ELF section '%s'\n", name);
6491 type_names = libbpf_get_type_names(false);
6492 if (type_names != NULL) {
6493 pr_debug("supported section(type) names are:%s\n", type_names);
6494 free(type_names);
6495 }
6496
6497 return -ESRCH;
6498 }
6499
6500 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
6501 size_t offset)
6502 {
6503 struct bpf_map *map;
6504 size_t i;
6505
6506 for (i = 0; i < obj->nr_maps; i++) {
6507 map = &obj->maps[i];
6508 if (!bpf_map__is_struct_ops(map))
6509 continue;
6510 if (map->sec_offset <= offset &&
6511 offset - map->sec_offset < map->def.value_size)
6512 return map;
6513 }
6514
6515 return NULL;
6516 }
6517
6518 /* Collect the reloc from ELF and populate the st_ops->progs[] */
6519 static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj,
6520 GElf_Shdr *shdr,
6521 Elf_Data *data)
6522 {
6523 const struct btf_member *member;
6524 struct bpf_struct_ops *st_ops;
6525 struct bpf_program *prog;
6526 unsigned int shdr_idx;
6527 const struct btf *btf;
6528 struct bpf_map *map;
6529 Elf_Data *symbols;
6530 unsigned int moff;
6531 const char *name;
6532 __u32 member_idx;
6533 GElf_Sym sym;
6534 GElf_Rel rel;
6535 int i, nrels;
6536
6537 symbols = obj->efile.symbols;
6538 btf = obj->btf;
6539 nrels = shdr->sh_size / shdr->sh_entsize;
6540 for (i = 0; i < nrels; i++) {
6541 if (!gelf_getrel(data, i, &rel)) {
6542 pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
6543 return -LIBBPF_ERRNO__FORMAT;
6544 }
6545
6546 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
6547 pr_warn("struct_ops reloc: symbol %zx not found\n",
6548 (size_t)GELF_R_SYM(rel.r_info));
6549 return -LIBBPF_ERRNO__FORMAT;
6550 }
6551
6552 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
6553 sym.st_name) ? : "<?>";
6554 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
6555 if (!map) {
6556 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
6557 (size_t)rel.r_offset);
6558 return -EINVAL;
6559 }
6560
6561 moff = rel.r_offset - map->sec_offset;
6562 shdr_idx = sym.st_shndx;
6563 st_ops = map->st_ops;
6564 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
6565 map->name,
6566 (long long)(rel.r_info >> 32),
6567 (long long)sym.st_value,
6568 shdr_idx, (size_t)rel.r_offset,
6569 map->sec_offset, sym.st_name, name);
6570
6571 if (shdr_idx >= SHN_LORESERVE) {
6572 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
6573 map->name, (size_t)rel.r_offset, shdr_idx);
6574 return -LIBBPF_ERRNO__RELOC;
6575 }
6576
6577 member = find_member_by_offset(st_ops->type, moff * 8);
6578 if (!member) {
6579 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
6580 map->name, moff);
6581 return -EINVAL;
6582 }
6583 member_idx = member - btf_members(st_ops->type);
6584 name = btf__name_by_offset(btf, member->name_off);
6585
6586 if (!resolve_func_ptr(btf, member->type, NULL)) {
6587 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
6588 map->name, name);
6589 return -EINVAL;
6590 }
6591
6592 prog = bpf_object__find_prog_by_idx(obj, shdr_idx);
6593 if (!prog) {
6594 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
6595 map->name, shdr_idx, name);
6596 return -EINVAL;
6597 }
6598
6599 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6600 const struct bpf_sec_def *sec_def;
6601
6602 sec_def = find_sec_def(prog->section_name);
6603 if (sec_def &&
6604 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
6605 /* for pr_warn */
6606 prog->type = sec_def->prog_type;
6607 goto invalid_prog;
6608 }
6609
6610 prog->type = BPF_PROG_TYPE_STRUCT_OPS;
6611 prog->attach_btf_id = st_ops->type_id;
6612 prog->expected_attach_type = member_idx;
6613 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
6614 prog->attach_btf_id != st_ops->type_id ||
6615 prog->expected_attach_type != member_idx) {
6616 goto invalid_prog;
6617 }
6618 st_ops->progs[member_idx] = prog;
6619 }
6620
6621 return 0;
6622
6623 invalid_prog:
6624 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
6625 map->name, prog->name, prog->section_name, prog->type,
6626 prog->attach_btf_id, prog->expected_attach_type, name);
6627 return -EINVAL;
6628 }
6629
6630 #define BTF_TRACE_PREFIX "btf_trace_"
6631 #define BTF_LSM_PREFIX "bpf_lsm_"
6632 #define BTF_MAX_NAME_SIZE 128
6633
6634 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
6635 const char *name, __u32 kind)
6636 {
6637 char btf_type_name[BTF_MAX_NAME_SIZE];
6638 int ret;
6639
6640 ret = snprintf(btf_type_name, sizeof(btf_type_name),
6641 "%s%s", prefix, name);
6642 /* snprintf returns the number of characters written excluding the
6643 * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
6644 * indicates truncation.
6645 */
6646 if (ret < 0 || ret >= sizeof(btf_type_name))
6647 return -ENAMETOOLONG;
6648 return btf__find_by_name_kind(btf, btf_type_name, kind);
6649 }
6650
6651 static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name,
6652 enum bpf_attach_type attach_type)
6653 {
6654 int err;
6655
6656 if (attach_type == BPF_TRACE_RAW_TP)
6657 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name,
6658 BTF_KIND_TYPEDEF);
6659 else if (attach_type == BPF_LSM_MAC)
6660 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name,
6661 BTF_KIND_FUNC);
6662 else
6663 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
6664
6665 if (err <= 0)
6666 pr_warn("%s is not found in vmlinux BTF\n", name);
6667
6668 return err;
6669 }
6670
6671 int libbpf_find_vmlinux_btf_id(const char *name,
6672 enum bpf_attach_type attach_type)
6673 {
6674 struct btf *btf;
6675
6676 btf = libbpf_find_kernel_btf();
6677 if (IS_ERR(btf)) {
6678 pr_warn("vmlinux BTF is not found\n");
6679 return -EINVAL;
6680 }
6681
6682 return __find_vmlinux_btf_id(btf, name, attach_type);
6683 }
6684
6685 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
6686 {
6687 struct bpf_prog_info_linear *info_linear;
6688 struct bpf_prog_info *info;
6689 struct btf *btf = NULL;
6690 int err = -EINVAL;
6691
6692 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
6693 if (IS_ERR_OR_NULL(info_linear)) {
6694 pr_warn("failed get_prog_info_linear for FD %d\n",
6695 attach_prog_fd);
6696 return -EINVAL;
6697 }
6698 info = &info_linear->info;
6699 if (!info->btf_id) {
6700 pr_warn("The target program doesn't have BTF\n");
6701 goto out;
6702 }
6703 if (btf__get_from_id(info->btf_id, &btf)) {
6704 pr_warn("Failed to get BTF of the program\n");
6705 goto out;
6706 }
6707 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
6708 btf__free(btf);
6709 if (err <= 0) {
6710 pr_warn("%s is not found in prog's BTF\n", name);
6711 goto out;
6712 }
6713 out:
6714 free(info_linear);
6715 return err;
6716 }
6717
6718 static int libbpf_find_attach_btf_id(struct bpf_program *prog)
6719 {
6720 enum bpf_attach_type attach_type = prog->expected_attach_type;
6721 __u32 attach_prog_fd = prog->attach_prog_fd;
6722 const char *name = prog->section_name;
6723 int i, err;
6724
6725 if (!name)
6726 return -EINVAL;
6727
6728 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
6729 if (!section_defs[i].is_attach_btf)
6730 continue;
6731 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
6732 continue;
6733 if (attach_prog_fd)
6734 err = libbpf_find_prog_btf_id(name + section_defs[i].len,
6735 attach_prog_fd);
6736 else
6737 err = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
6738 name + section_defs[i].len,
6739 attach_type);
6740 return err;
6741 }
6742 pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name);
6743 return -ESRCH;
6744 }
6745
6746 int libbpf_attach_type_by_name(const char *name,
6747 enum bpf_attach_type *attach_type)
6748 {
6749 char *type_names;
6750 int i;
6751
6752 if (!name)
6753 return -EINVAL;
6754
6755 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
6756 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
6757 continue;
6758 if (!section_defs[i].is_attachable)
6759 return -EINVAL;
6760 *attach_type = section_defs[i].expected_attach_type;
6761 return 0;
6762 }
6763 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
6764 type_names = libbpf_get_type_names(true);
6765 if (type_names != NULL) {
6766 pr_debug("attachable section(type) names are:%s\n", type_names);
6767 free(type_names);
6768 }
6769
6770 return -EINVAL;
6771 }
6772
6773 int bpf_map__fd(const struct bpf_map *map)
6774 {
6775 return map ? map->fd : -EINVAL;
6776 }
6777
6778 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
6779 {
6780 return map ? &map->def : ERR_PTR(-EINVAL);
6781 }
6782
6783 const char *bpf_map__name(const struct bpf_map *map)
6784 {
6785 return map ? map->name : NULL;
6786 }
6787
6788 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
6789 {
6790 return map ? map->btf_key_type_id : 0;
6791 }
6792
6793 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
6794 {
6795 return map ? map->btf_value_type_id : 0;
6796 }
6797
6798 int bpf_map__set_priv(struct bpf_map *map, void *priv,
6799 bpf_map_clear_priv_t clear_priv)
6800 {
6801 if (!map)
6802 return -EINVAL;
6803
6804 if (map->priv) {
6805 if (map->clear_priv)
6806 map->clear_priv(map, map->priv);
6807 }
6808
6809 map->priv = priv;
6810 map->clear_priv = clear_priv;
6811 return 0;
6812 }
6813
6814 void *bpf_map__priv(const struct bpf_map *map)
6815 {
6816 return map ? map->priv : ERR_PTR(-EINVAL);
6817 }
6818
6819 int bpf_map__set_initial_value(struct bpf_map *map,
6820 const void *data, size_t size)
6821 {
6822 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
6823 size != map->def.value_size || map->fd >= 0)
6824 return -EINVAL;
6825
6826 memcpy(map->mmaped, data, size);
6827 return 0;
6828 }
6829
6830 bool bpf_map__is_offload_neutral(const struct bpf_map *map)
6831 {
6832 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
6833 }
6834
6835 bool bpf_map__is_internal(const struct bpf_map *map)
6836 {
6837 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
6838 }
6839
6840 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
6841 {
6842 map->map_ifindex = ifindex;
6843 }
6844
6845 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
6846 {
6847 if (!bpf_map_type__is_map_in_map(map->def.type)) {
6848 pr_warn("error: unsupported map type\n");
6849 return -EINVAL;
6850 }
6851 if (map->inner_map_fd != -1) {
6852 pr_warn("error: inner_map_fd already specified\n");
6853 return -EINVAL;
6854 }
6855 map->inner_map_fd = fd;
6856 return 0;
6857 }
6858
6859 static struct bpf_map *
6860 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
6861 {
6862 ssize_t idx;
6863 struct bpf_map *s, *e;
6864
6865 if (!obj || !obj->maps)
6866 return NULL;
6867
6868 s = obj->maps;
6869 e = obj->maps + obj->nr_maps;
6870
6871 if ((m < s) || (m >= e)) {
6872 pr_warn("error in %s: map handler doesn't belong to object\n",
6873 __func__);
6874 return NULL;
6875 }
6876
6877 idx = (m - obj->maps) + i;
6878 if (idx >= obj->nr_maps || idx < 0)
6879 return NULL;
6880 return &obj->maps[idx];
6881 }
6882
6883 struct bpf_map *
6884 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
6885 {
6886 if (prev == NULL)
6887 return obj->maps;
6888
6889 return __bpf_map__iter(prev, obj, 1);
6890 }
6891
6892 struct bpf_map *
6893 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
6894 {
6895 if (next == NULL) {
6896 if (!obj->nr_maps)
6897 return NULL;
6898 return obj->maps + obj->nr_maps - 1;
6899 }
6900
6901 return __bpf_map__iter(next, obj, -1);
6902 }
6903
6904 struct bpf_map *
6905 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
6906 {
6907 struct bpf_map *pos;
6908
6909 bpf_object__for_each_map(pos, obj) {
6910 if (pos->name && !strcmp(pos->name, name))
6911 return pos;
6912 }
6913 return NULL;
6914 }
6915
6916 int
6917 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
6918 {
6919 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
6920 }
6921
6922 struct bpf_map *
6923 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
6924 {
6925 return ERR_PTR(-ENOTSUP);
6926 }
6927
6928 long libbpf_get_error(const void *ptr)
6929 {
6930 return PTR_ERR_OR_ZERO(ptr);
6931 }
6932
6933 int bpf_prog_load(const char *file, enum bpf_prog_type type,
6934 struct bpf_object **pobj, int *prog_fd)
6935 {
6936 struct bpf_prog_load_attr attr;
6937
6938 memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
6939 attr.file = file;
6940 attr.prog_type = type;
6941 attr.expected_attach_type = 0;
6942
6943 return bpf_prog_load_xattr(&attr, pobj, prog_fd);
6944 }
6945
6946 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
6947 struct bpf_object **pobj, int *prog_fd)
6948 {
6949 struct bpf_object_open_attr open_attr = {};
6950 struct bpf_program *prog, *first_prog = NULL;
6951 struct bpf_object *obj;
6952 struct bpf_map *map;
6953 int err;
6954
6955 if (!attr)
6956 return -EINVAL;
6957 if (!attr->file)
6958 return -EINVAL;
6959
6960 open_attr.file = attr->file;
6961 open_attr.prog_type = attr->prog_type;
6962
6963 obj = bpf_object__open_xattr(&open_attr);
6964 if (IS_ERR_OR_NULL(obj))
6965 return -ENOENT;
6966
6967 bpf_object__for_each_program(prog, obj) {
6968 enum bpf_attach_type attach_type = attr->expected_attach_type;
6969 /*
6970 * to preserve backwards compatibility, bpf_prog_load treats
6971 * attr->prog_type, if specified, as an override to whatever
6972 * bpf_object__open guessed
6973 */
6974 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
6975 bpf_program__set_type(prog, attr->prog_type);
6976 bpf_program__set_expected_attach_type(prog,
6977 attach_type);
6978 }
6979 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
6980 /*
6981 * we haven't guessed from section name and user
6982 * didn't provide a fallback type, too bad...
6983 */
6984 bpf_object__close(obj);
6985 return -EINVAL;
6986 }
6987
6988 prog->prog_ifindex = attr->ifindex;
6989 prog->log_level = attr->log_level;
6990 prog->prog_flags = attr->prog_flags;
6991 if (!first_prog)
6992 first_prog = prog;
6993 }
6994
6995 bpf_object__for_each_map(map, obj) {
6996 if (!bpf_map__is_offload_neutral(map))
6997 map->map_ifindex = attr->ifindex;
6998 }
6999
7000 if (!first_prog) {
7001 pr_warn("object file doesn't contain bpf program\n");
7002 bpf_object__close(obj);
7003 return -ENOENT;
7004 }
7005
7006 err = bpf_object__load(obj);
7007 if (err) {
7008 bpf_object__close(obj);
7009 return -EINVAL;
7010 }
7011
7012 *pobj = obj;
7013 *prog_fd = bpf_program__fd(first_prog);
7014 return 0;
7015 }
7016
7017 struct bpf_link {
7018 int (*detach)(struct bpf_link *link);
7019 int (*destroy)(struct bpf_link *link);
7020 char *pin_path; /* NULL, if not pinned */
7021 int fd; /* hook FD, -1 if not applicable */
7022 bool disconnected;
7023 };
7024
7025 /* Replace link's underlying BPF program with the new one */
7026 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
7027 {
7028 return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
7029 }
7030
7031 /* Release "ownership" of underlying BPF resource (typically, BPF program
7032 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
7033 * link, when destructed through bpf_link__destroy() call won't attempt to
7034 * detach/unregisted that BPF resource. This is useful in situations where,
7035 * say, attached BPF program has to outlive userspace program that attached it
7036 * in the system. Depending on type of BPF program, though, there might be
7037 * additional steps (like pinning BPF program in BPF FS) necessary to ensure
7038 * exit of userspace program doesn't trigger automatic detachment and clean up
7039 * inside the kernel.
7040 */
7041 void bpf_link__disconnect(struct bpf_link *link)
7042 {
7043 link->disconnected = true;
7044 }
7045
7046 int bpf_link__destroy(struct bpf_link *link)
7047 {
7048 int err = 0;
7049
7050 if (!link)
7051 return 0;
7052
7053 if (!link->disconnected && link->detach)
7054 err = link->detach(link);
7055 if (link->destroy)
7056 link->destroy(link);
7057 if (link->pin_path)
7058 free(link->pin_path);
7059 free(link);
7060
7061 return err;
7062 }
7063
7064 int bpf_link__fd(const struct bpf_link *link)
7065 {
7066 return link->fd;
7067 }
7068
7069 const char *bpf_link__pin_path(const struct bpf_link *link)
7070 {
7071 return link->pin_path;
7072 }
7073
7074 static int bpf_link__detach_fd(struct bpf_link *link)
7075 {
7076 return close(link->fd);
7077 }
7078
7079 struct bpf_link *bpf_link__open(const char *path)
7080 {
7081 struct bpf_link *link;
7082 int fd;
7083
7084 fd = bpf_obj_get(path);
7085 if (fd < 0) {
7086 fd = -errno;
7087 pr_warn("failed to open link at %s: %d\n", path, fd);
7088 return ERR_PTR(fd);
7089 }
7090
7091 link = calloc(1, sizeof(*link));
7092 if (!link) {
7093 close(fd);
7094 return ERR_PTR(-ENOMEM);
7095 }
7096 link->detach = &bpf_link__detach_fd;
7097 link->fd = fd;
7098
7099 link->pin_path = strdup(path);
7100 if (!link->pin_path) {
7101 bpf_link__destroy(link);
7102 return ERR_PTR(-ENOMEM);
7103 }
7104
7105 return link;
7106 }
7107
7108 int bpf_link__pin(struct bpf_link *link, const char *path)
7109 {
7110 int err;
7111
7112 if (link->pin_path)
7113 return -EBUSY;
7114 err = make_parent_dir(path);
7115 if (err)
7116 return err;
7117 err = check_path(path);
7118 if (err)
7119 return err;
7120
7121 link->pin_path = strdup(path);
7122 if (!link->pin_path)
7123 return -ENOMEM;
7124
7125 if (bpf_obj_pin(link->fd, link->pin_path)) {
7126 err = -errno;
7127 zfree(&link->pin_path);
7128 return err;
7129 }
7130
7131 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
7132 return 0;
7133 }
7134
7135 int bpf_link__unpin(struct bpf_link *link)
7136 {
7137 int err;
7138
7139 if (!link->pin_path)
7140 return -EINVAL;
7141
7142 err = unlink(link->pin_path);
7143 if (err != 0)
7144 return -errno;
7145
7146 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
7147 zfree(&link->pin_path);
7148 return 0;
7149 }
7150
7151 static int bpf_link__detach_perf_event(struct bpf_link *link)
7152 {
7153 int err;
7154
7155 err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0);
7156 if (err)
7157 err = -errno;
7158
7159 close(link->fd);
7160 return err;
7161 }
7162
7163 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
7164 int pfd)
7165 {
7166 char errmsg[STRERR_BUFSIZE];
7167 struct bpf_link *link;
7168 int prog_fd, err;
7169
7170 if (pfd < 0) {
7171 pr_warn("program '%s': invalid perf event FD %d\n",
7172 bpf_program__title(prog, false), pfd);
7173 return ERR_PTR(-EINVAL);
7174 }
7175 prog_fd = bpf_program__fd(prog);
7176 if (prog_fd < 0) {
7177 pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n",
7178 bpf_program__title(prog, false));
7179 return ERR_PTR(-EINVAL);
7180 }
7181
7182 link = calloc(1, sizeof(*link));
7183 if (!link)
7184 return ERR_PTR(-ENOMEM);
7185 link->detach = &bpf_link__detach_perf_event;
7186 link->fd = pfd;
7187
7188 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
7189 err = -errno;
7190 free(link);
7191 pr_warn("program '%s': failed to attach to pfd %d: %s\n",
7192 bpf_program__title(prog, false), pfd,
7193 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7194 return ERR_PTR(err);
7195 }
7196 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
7197 err = -errno;
7198 free(link);
7199 pr_warn("program '%s': failed to enable pfd %d: %s\n",
7200 bpf_program__title(prog, false), pfd,
7201 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7202 return ERR_PTR(err);
7203 }
7204 return link;
7205 }
7206
7207 /*
7208 * this function is expected to parse integer in the range of [0, 2^31-1] from
7209 * given file using scanf format string fmt. If actual parsed value is
7210 * negative, the result might be indistinguishable from error
7211 */
7212 static int parse_uint_from_file(const char *file, const char *fmt)
7213 {
7214 char buf[STRERR_BUFSIZE];
7215 int err, ret;
7216 FILE *f;
7217
7218 f = fopen(file, "r");
7219 if (!f) {
7220 err = -errno;
7221 pr_debug("failed to open '%s': %s\n", file,
7222 libbpf_strerror_r(err, buf, sizeof(buf)));
7223 return err;
7224 }
7225 err = fscanf(f, fmt, &ret);
7226 if (err != 1) {
7227 err = err == EOF ? -EIO : -errno;
7228 pr_debug("failed to parse '%s': %s\n", file,
7229 libbpf_strerror_r(err, buf, sizeof(buf)));
7230 fclose(f);
7231 return err;
7232 }
7233 fclose(f);
7234 return ret;
7235 }
7236
7237 static int determine_kprobe_perf_type(void)
7238 {
7239 const char *file = "/sys/bus/event_source/devices/kprobe/type";
7240
7241 return parse_uint_from_file(file, "%d\n");
7242 }
7243
7244 static int determine_uprobe_perf_type(void)
7245 {
7246 const char *file = "/sys/bus/event_source/devices/uprobe/type";
7247
7248 return parse_uint_from_file(file, "%d\n");
7249 }
7250
7251 static int determine_kprobe_retprobe_bit(void)
7252 {
7253 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
7254
7255 return parse_uint_from_file(file, "config:%d\n");
7256 }
7257
7258 static int determine_uprobe_retprobe_bit(void)
7259 {
7260 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
7261
7262 return parse_uint_from_file(file, "config:%d\n");
7263 }
7264
7265 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
7266 uint64_t offset, int pid)
7267 {
7268 struct perf_event_attr attr = {};
7269 char errmsg[STRERR_BUFSIZE];
7270 int type, pfd, err;
7271
7272 type = uprobe ? determine_uprobe_perf_type()
7273 : determine_kprobe_perf_type();
7274 if (type < 0) {
7275 pr_warn("failed to determine %s perf type: %s\n",
7276 uprobe ? "uprobe" : "kprobe",
7277 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
7278 return type;
7279 }
7280 if (retprobe) {
7281 int bit = uprobe ? determine_uprobe_retprobe_bit()
7282 : determine_kprobe_retprobe_bit();
7283
7284 if (bit < 0) {
7285 pr_warn("failed to determine %s retprobe bit: %s\n",
7286 uprobe ? "uprobe" : "kprobe",
7287 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
7288 return bit;
7289 }
7290 attr.config |= 1 << bit;
7291 }
7292 attr.size = sizeof(attr);
7293 attr.type = type;
7294 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
7295 attr.config2 = offset; /* kprobe_addr or probe_offset */
7296
7297 /* pid filter is meaningful only for uprobes */
7298 pfd = syscall(__NR_perf_event_open, &attr,
7299 pid < 0 ? -1 : pid /* pid */,
7300 pid == -1 ? 0 : -1 /* cpu */,
7301 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
7302 if (pfd < 0) {
7303 err = -errno;
7304 pr_warn("%s perf_event_open() failed: %s\n",
7305 uprobe ? "uprobe" : "kprobe",
7306 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7307 return err;
7308 }
7309 return pfd;
7310 }
7311
7312 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
7313 bool retprobe,
7314 const char *func_name)
7315 {
7316 char errmsg[STRERR_BUFSIZE];
7317 struct bpf_link *link;
7318 int pfd, err;
7319
7320 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,
7321 0 /* offset */, -1 /* pid */);
7322 if (pfd < 0) {
7323 pr_warn("program '%s': failed to create %s '%s' perf event: %s\n",
7324 bpf_program__title(prog, false),
7325 retprobe ? "kretprobe" : "kprobe", func_name,
7326 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7327 return ERR_PTR(pfd);
7328 }
7329 link = bpf_program__attach_perf_event(prog, pfd);
7330 if (IS_ERR(link)) {
7331 close(pfd);
7332 err = PTR_ERR(link);
7333 pr_warn("program '%s': failed to attach to %s '%s': %s\n",
7334 bpf_program__title(prog, false),
7335 retprobe ? "kretprobe" : "kprobe", func_name,
7336 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7337 return link;
7338 }
7339 return link;
7340 }
7341
7342 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
7343 struct bpf_program *prog)
7344 {
7345 const char *func_name;
7346 bool retprobe;
7347
7348 func_name = bpf_program__title(prog, false) + sec->len;
7349 retprobe = strcmp(sec->sec, "kretprobe/") == 0;
7350
7351 return bpf_program__attach_kprobe(prog, retprobe, func_name);
7352 }
7353
7354 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
7355 bool retprobe, pid_t pid,
7356 const char *binary_path,
7357 size_t func_offset)
7358 {
7359 char errmsg[STRERR_BUFSIZE];
7360 struct bpf_link *link;
7361 int pfd, err;
7362
7363 pfd = perf_event_open_probe(true /* uprobe */, retprobe,
7364 binary_path, func_offset, pid);
7365 if (pfd < 0) {
7366 pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
7367 bpf_program__title(prog, false),
7368 retprobe ? "uretprobe" : "uprobe",
7369 binary_path, func_offset,
7370 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7371 return ERR_PTR(pfd);
7372 }
7373 link = bpf_program__attach_perf_event(prog, pfd);
7374 if (IS_ERR(link)) {
7375 close(pfd);
7376 err = PTR_ERR(link);
7377 pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n",
7378 bpf_program__title(prog, false),
7379 retprobe ? "uretprobe" : "uprobe",
7380 binary_path, func_offset,
7381 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7382 return link;
7383 }
7384 return link;
7385 }
7386
7387 static int determine_tracepoint_id(const char *tp_category,
7388 const char *tp_name)
7389 {
7390 char file[PATH_MAX];
7391 int ret;
7392
7393 ret = snprintf(file, sizeof(file),
7394 "/sys/kernel/debug/tracing/events/%s/%s/id",
7395 tp_category, tp_name);
7396 if (ret < 0)
7397 return -errno;
7398 if (ret >= sizeof(file)) {
7399 pr_debug("tracepoint %s/%s path is too long\n",
7400 tp_category, tp_name);
7401 return -E2BIG;
7402 }
7403 return parse_uint_from_file(file, "%d\n");
7404 }
7405
7406 static int perf_event_open_tracepoint(const char *tp_category,
7407 const char *tp_name)
7408 {
7409 struct perf_event_attr attr = {};
7410 char errmsg[STRERR_BUFSIZE];
7411 int tp_id, pfd, err;
7412
7413 tp_id = determine_tracepoint_id(tp_category, tp_name);
7414 if (tp_id < 0) {
7415 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
7416 tp_category, tp_name,
7417 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
7418 return tp_id;
7419 }
7420
7421 attr.type = PERF_TYPE_TRACEPOINT;
7422 attr.size = sizeof(attr);
7423 attr.config = tp_id;
7424
7425 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
7426 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
7427 if (pfd < 0) {
7428 err = -errno;
7429 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
7430 tp_category, tp_name,
7431 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7432 return err;
7433 }
7434 return pfd;
7435 }
7436
7437 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
7438 const char *tp_category,
7439 const char *tp_name)
7440 {
7441 char errmsg[STRERR_BUFSIZE];
7442 struct bpf_link *link;
7443 int pfd, err;
7444
7445 pfd = perf_event_open_tracepoint(tp_category, tp_name);
7446 if (pfd < 0) {
7447 pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
7448 bpf_program__title(prog, false),
7449 tp_category, tp_name,
7450 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7451 return ERR_PTR(pfd);
7452 }
7453 link = bpf_program__attach_perf_event(prog, pfd);
7454 if (IS_ERR(link)) {
7455 close(pfd);
7456 err = PTR_ERR(link);
7457 pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n",
7458 bpf_program__title(prog, false),
7459 tp_category, tp_name,
7460 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7461 return link;
7462 }
7463 return link;
7464 }
7465
7466 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
7467 struct bpf_program *prog)
7468 {
7469 char *sec_name, *tp_cat, *tp_name;
7470 struct bpf_link *link;
7471
7472 sec_name = strdup(bpf_program__title(prog, false));
7473 if (!sec_name)
7474 return ERR_PTR(-ENOMEM);
7475
7476 /* extract "tp/<category>/<name>" */
7477 tp_cat = sec_name + sec->len;
7478 tp_name = strchr(tp_cat, '/');
7479 if (!tp_name) {
7480 link = ERR_PTR(-EINVAL);
7481 goto out;
7482 }
7483 *tp_name = '\0';
7484 tp_name++;
7485
7486 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
7487 out:
7488 free(sec_name);
7489 return link;
7490 }
7491
7492 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
7493 const char *tp_name)
7494 {
7495 char errmsg[STRERR_BUFSIZE];
7496 struct bpf_link *link;
7497 int prog_fd, pfd;
7498
7499 prog_fd = bpf_program__fd(prog);
7500 if (prog_fd < 0) {
7501 pr_warn("program '%s': can't attach before loaded\n",
7502 bpf_program__title(prog, false));
7503 return ERR_PTR(-EINVAL);
7504 }
7505
7506 link = calloc(1, sizeof(*link));
7507 if (!link)
7508 return ERR_PTR(-ENOMEM);
7509 link->detach = &bpf_link__detach_fd;
7510
7511 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
7512 if (pfd < 0) {
7513 pfd = -errno;
7514 free(link);
7515 pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n",
7516 bpf_program__title(prog, false), tp_name,
7517 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7518 return ERR_PTR(pfd);
7519 }
7520 link->fd = pfd;
7521 return link;
7522 }
7523
7524 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
7525 struct bpf_program *prog)
7526 {
7527 const char *tp_name = bpf_program__title(prog, false) + sec->len;
7528
7529 return bpf_program__attach_raw_tracepoint(prog, tp_name);
7530 }
7531
7532 /* Common logic for all BPF program types that attach to a btf_id */
7533 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
7534 {
7535 char errmsg[STRERR_BUFSIZE];
7536 struct bpf_link *link;
7537 int prog_fd, pfd;
7538
7539 prog_fd = bpf_program__fd(prog);
7540 if (prog_fd < 0) {
7541 pr_warn("program '%s': can't attach before loaded\n",
7542 bpf_program__title(prog, false));
7543 return ERR_PTR(-EINVAL);
7544 }
7545
7546 link = calloc(1, sizeof(*link));
7547 if (!link)
7548 return ERR_PTR(-ENOMEM);
7549 link->detach = &bpf_link__detach_fd;
7550
7551 pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
7552 if (pfd < 0) {
7553 pfd = -errno;
7554 free(link);
7555 pr_warn("program '%s': failed to attach: %s\n",
7556 bpf_program__title(prog, false),
7557 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7558 return ERR_PTR(pfd);
7559 }
7560 link->fd = pfd;
7561 return (struct bpf_link *)link;
7562 }
7563
7564 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
7565 {
7566 return bpf_program__attach_btf_id(prog);
7567 }
7568
7569 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
7570 {
7571 return bpf_program__attach_btf_id(prog);
7572 }
7573
7574 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
7575 struct bpf_program *prog)
7576 {
7577 return bpf_program__attach_trace(prog);
7578 }
7579
7580 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
7581 struct bpf_program *prog)
7582 {
7583 return bpf_program__attach_lsm(prog);
7584 }
7585
7586 struct bpf_link *
7587 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
7588 {
7589 enum bpf_attach_type attach_type;
7590 char errmsg[STRERR_BUFSIZE];
7591 struct bpf_link *link;
7592 int prog_fd, link_fd;
7593
7594 prog_fd = bpf_program__fd(prog);
7595 if (prog_fd < 0) {
7596 pr_warn("program '%s': can't attach before loaded\n",
7597 bpf_program__title(prog, false));
7598 return ERR_PTR(-EINVAL);
7599 }
7600
7601 link = calloc(1, sizeof(*link));
7602 if (!link)
7603 return ERR_PTR(-ENOMEM);
7604 link->detach = &bpf_link__detach_fd;
7605
7606 attach_type = bpf_program__get_expected_attach_type(prog);
7607 link_fd = bpf_link_create(prog_fd, cgroup_fd, attach_type, NULL);
7608 if (link_fd < 0) {
7609 link_fd = -errno;
7610 free(link);
7611 pr_warn("program '%s': failed to attach to cgroup: %s\n",
7612 bpf_program__title(prog, false),
7613 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
7614 return ERR_PTR(link_fd);
7615 }
7616 link->fd = link_fd;
7617 return link;
7618 }
7619
7620 struct bpf_link *bpf_program__attach(struct bpf_program *prog)
7621 {
7622 const struct bpf_sec_def *sec_def;
7623
7624 sec_def = find_sec_def(bpf_program__title(prog, false));
7625 if (!sec_def || !sec_def->attach_fn)
7626 return ERR_PTR(-ESRCH);
7627
7628 return sec_def->attach_fn(sec_def, prog);
7629 }
7630
7631 static int bpf_link__detach_struct_ops(struct bpf_link *link)
7632 {
7633 __u32 zero = 0;
7634
7635 if (bpf_map_delete_elem(link->fd, &zero))
7636 return -errno;
7637
7638 return 0;
7639 }
7640
7641 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
7642 {
7643 struct bpf_struct_ops *st_ops;
7644 struct bpf_link *link;
7645 __u32 i, zero = 0;
7646 int err;
7647
7648 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
7649 return ERR_PTR(-EINVAL);
7650
7651 link = calloc(1, sizeof(*link));
7652 if (!link)
7653 return ERR_PTR(-EINVAL);
7654
7655 st_ops = map->st_ops;
7656 for (i = 0; i < btf_vlen(st_ops->type); i++) {
7657 struct bpf_program *prog = st_ops->progs[i];
7658 void *kern_data;
7659 int prog_fd;
7660
7661 if (!prog)
7662 continue;
7663
7664 prog_fd = bpf_program__fd(prog);
7665 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
7666 *(unsigned long *)kern_data = prog_fd;
7667 }
7668
7669 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
7670 if (err) {
7671 err = -errno;
7672 free(link);
7673 return ERR_PTR(err);
7674 }
7675
7676 link->detach = bpf_link__detach_struct_ops;
7677 link->fd = map->fd;
7678
7679 return link;
7680 }
7681
7682 enum bpf_perf_event_ret
7683 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
7684 void **copy_mem, size_t *copy_size,
7685 bpf_perf_event_print_t fn, void *private_data)
7686 {
7687 struct perf_event_mmap_page *header = mmap_mem;
7688 __u64 data_head = ring_buffer_read_head(header);
7689 __u64 data_tail = header->data_tail;
7690 void *base = ((__u8 *)header) + page_size;
7691 int ret = LIBBPF_PERF_EVENT_CONT;
7692 struct perf_event_header *ehdr;
7693 size_t ehdr_size;
7694
7695 while (data_head != data_tail) {
7696 ehdr = base + (data_tail & (mmap_size - 1));
7697 ehdr_size = ehdr->size;
7698
7699 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
7700 void *copy_start = ehdr;
7701 size_t len_first = base + mmap_size - copy_start;
7702 size_t len_secnd = ehdr_size - len_first;
7703
7704 if (*copy_size < ehdr_size) {
7705 free(*copy_mem);
7706 *copy_mem = malloc(ehdr_size);
7707 if (!*copy_mem) {
7708 *copy_size = 0;
7709 ret = LIBBPF_PERF_EVENT_ERROR;
7710 break;
7711 }
7712 *copy_size = ehdr_size;
7713 }
7714
7715 memcpy(*copy_mem, copy_start, len_first);
7716 memcpy(*copy_mem + len_first, base, len_secnd);
7717 ehdr = *copy_mem;
7718 }
7719
7720 ret = fn(ehdr, private_data);
7721 data_tail += ehdr_size;
7722 if (ret != LIBBPF_PERF_EVENT_CONT)
7723 break;
7724 }
7725
7726 ring_buffer_write_tail(header, data_tail);
7727 return ret;
7728 }
7729
7730 struct perf_buffer;
7731
7732 struct perf_buffer_params {
7733 struct perf_event_attr *attr;
7734 /* if event_cb is specified, it takes precendence */
7735 perf_buffer_event_fn event_cb;
7736 /* sample_cb and lost_cb are higher-level common-case callbacks */
7737 perf_buffer_sample_fn sample_cb;
7738 perf_buffer_lost_fn lost_cb;
7739 void *ctx;
7740 int cpu_cnt;
7741 int *cpus;
7742 int *map_keys;
7743 };
7744
7745 struct perf_cpu_buf {
7746 struct perf_buffer *pb;
7747 void *base; /* mmap()'ed memory */
7748 void *buf; /* for reconstructing segmented data */
7749 size_t buf_size;
7750 int fd;
7751 int cpu;
7752 int map_key;
7753 };
7754
7755 struct perf_buffer {
7756 perf_buffer_event_fn event_cb;
7757 perf_buffer_sample_fn sample_cb;
7758 perf_buffer_lost_fn lost_cb;
7759 void *ctx; /* passed into callbacks */
7760
7761 size_t page_size;
7762 size_t mmap_size;
7763 struct perf_cpu_buf **cpu_bufs;
7764 struct epoll_event *events;
7765 int cpu_cnt; /* number of allocated CPU buffers */
7766 int epoll_fd; /* perf event FD */
7767 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
7768 };
7769
7770 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
7771 struct perf_cpu_buf *cpu_buf)
7772 {
7773 if (!cpu_buf)
7774 return;
7775 if (cpu_buf->base &&
7776 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
7777 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
7778 if (cpu_buf->fd >= 0) {
7779 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
7780 close(cpu_buf->fd);
7781 }
7782 free(cpu_buf->buf);
7783 free(cpu_buf);
7784 }
7785
7786 void perf_buffer__free(struct perf_buffer *pb)
7787 {
7788 int i;
7789
7790 if (!pb)
7791 return;
7792 if (pb->cpu_bufs) {
7793 for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) {
7794 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
7795
7796 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
7797 perf_buffer__free_cpu_buf(pb, cpu_buf);
7798 }
7799 free(pb->cpu_bufs);
7800 }
7801 if (pb->epoll_fd >= 0)
7802 close(pb->epoll_fd);
7803 free(pb->events);
7804 free(pb);
7805 }
7806
7807 static struct perf_cpu_buf *
7808 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
7809 int cpu, int map_key)
7810 {
7811 struct perf_cpu_buf *cpu_buf;
7812 char msg[STRERR_BUFSIZE];
7813 int err;
7814
7815 cpu_buf = calloc(1, sizeof(*cpu_buf));
7816 if (!cpu_buf)
7817 return ERR_PTR(-ENOMEM);
7818
7819 cpu_buf->pb = pb;
7820 cpu_buf->cpu = cpu;
7821 cpu_buf->map_key = map_key;
7822
7823 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
7824 -1, PERF_FLAG_FD_CLOEXEC);
7825 if (cpu_buf->fd < 0) {
7826 err = -errno;
7827 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
7828 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
7829 goto error;
7830 }
7831
7832 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
7833 PROT_READ | PROT_WRITE, MAP_SHARED,
7834 cpu_buf->fd, 0);
7835 if (cpu_buf->base == MAP_FAILED) {
7836 cpu_buf->base = NULL;
7837 err = -errno;
7838 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
7839 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
7840 goto error;
7841 }
7842
7843 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
7844 err = -errno;
7845 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
7846 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
7847 goto error;
7848 }
7849
7850 return cpu_buf;
7851
7852 error:
7853 perf_buffer__free_cpu_buf(pb, cpu_buf);
7854 return (struct perf_cpu_buf *)ERR_PTR(err);
7855 }
7856
7857 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
7858 struct perf_buffer_params *p);
7859
7860 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
7861 const struct perf_buffer_opts *opts)
7862 {
7863 struct perf_buffer_params p = {};
7864 struct perf_event_attr attr = { 0, };
7865
7866 attr.config = PERF_COUNT_SW_BPF_OUTPUT,
7867 attr.type = PERF_TYPE_SOFTWARE;
7868 attr.sample_type = PERF_SAMPLE_RAW;
7869 attr.sample_period = 1;
7870 attr.wakeup_events = 1;
7871
7872 p.attr = &attr;
7873 p.sample_cb = opts ? opts->sample_cb : NULL;
7874 p.lost_cb = opts ? opts->lost_cb : NULL;
7875 p.ctx = opts ? opts->ctx : NULL;
7876
7877 return __perf_buffer__new(map_fd, page_cnt, &p);
7878 }
7879
7880 struct perf_buffer *
7881 perf_buffer__new_raw(int map_fd, size_t page_cnt,
7882 const struct perf_buffer_raw_opts *opts)
7883 {
7884 struct perf_buffer_params p = {};
7885
7886 p.attr = opts->attr;
7887 p.event_cb = opts->event_cb;
7888 p.ctx = opts->ctx;
7889 p.cpu_cnt = opts->cpu_cnt;
7890 p.cpus = opts->cpus;
7891 p.map_keys = opts->map_keys;
7892
7893 return __perf_buffer__new(map_fd, page_cnt, &p);
7894 }
7895
7896 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
7897 struct perf_buffer_params *p)
7898 {
7899 const char *online_cpus_file = "/sys/devices/system/cpu/online";
7900 struct bpf_map_info map = {};
7901 char msg[STRERR_BUFSIZE];
7902 struct perf_buffer *pb;
7903 bool *online = NULL;
7904 __u32 map_info_len;
7905 int err, i, j, n;
7906
7907 if (page_cnt & (page_cnt - 1)) {
7908 pr_warn("page count should be power of two, but is %zu\n",
7909 page_cnt);
7910 return ERR_PTR(-EINVAL);
7911 }
7912
7913 map_info_len = sizeof(map);
7914 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
7915 if (err) {
7916 err = -errno;
7917 pr_warn("failed to get map info for map FD %d: %s\n",
7918 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
7919 return ERR_PTR(err);
7920 }
7921
7922 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
7923 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
7924 map.name);
7925 return ERR_PTR(-EINVAL);
7926 }
7927
7928 pb = calloc(1, sizeof(*pb));
7929 if (!pb)
7930 return ERR_PTR(-ENOMEM);
7931
7932 pb->event_cb = p->event_cb;
7933 pb->sample_cb = p->sample_cb;
7934 pb->lost_cb = p->lost_cb;
7935 pb->ctx = p->ctx;
7936
7937 pb->page_size = getpagesize();
7938 pb->mmap_size = pb->page_size * page_cnt;
7939 pb->map_fd = map_fd;
7940
7941 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
7942 if (pb->epoll_fd < 0) {
7943 err = -errno;
7944 pr_warn("failed to create epoll instance: %s\n",
7945 libbpf_strerror_r(err, msg, sizeof(msg)));
7946 goto error;
7947 }
7948
7949 if (p->cpu_cnt > 0) {
7950 pb->cpu_cnt = p->cpu_cnt;
7951 } else {
7952 pb->cpu_cnt = libbpf_num_possible_cpus();
7953 if (pb->cpu_cnt < 0) {
7954 err = pb->cpu_cnt;
7955 goto error;
7956 }
7957 if (map.max_entries < pb->cpu_cnt)
7958 pb->cpu_cnt = map.max_entries;
7959 }
7960
7961 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
7962 if (!pb->events) {
7963 err = -ENOMEM;
7964 pr_warn("failed to allocate events: out of memory\n");
7965 goto error;
7966 }
7967 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
7968 if (!pb->cpu_bufs) {
7969 err = -ENOMEM;
7970 pr_warn("failed to allocate buffers: out of memory\n");
7971 goto error;
7972 }
7973
7974 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
7975 if (err) {
7976 pr_warn("failed to get online CPU mask: %d\n", err);
7977 goto error;
7978 }
7979
7980 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
7981 struct perf_cpu_buf *cpu_buf;
7982 int cpu, map_key;
7983
7984 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
7985 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
7986
7987 /* in case user didn't explicitly requested particular CPUs to
7988 * be attached to, skip offline/not present CPUs
7989 */
7990 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
7991 continue;
7992
7993 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
7994 if (IS_ERR(cpu_buf)) {
7995 err = PTR_ERR(cpu_buf);
7996 goto error;
7997 }
7998
7999 pb->cpu_bufs[j] = cpu_buf;
8000
8001 err = bpf_map_update_elem(pb->map_fd, &map_key,
8002 &cpu_buf->fd, 0);
8003 if (err) {
8004 err = -errno;
8005 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
8006 cpu, map_key, cpu_buf->fd,
8007 libbpf_strerror_r(err, msg, sizeof(msg)));
8008 goto error;
8009 }
8010
8011 pb->events[j].events = EPOLLIN;
8012 pb->events[j].data.ptr = cpu_buf;
8013 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
8014 &pb->events[j]) < 0) {
8015 err = -errno;
8016 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
8017 cpu, cpu_buf->fd,
8018 libbpf_strerror_r(err, msg, sizeof(msg)));
8019 goto error;
8020 }
8021 j++;
8022 }
8023 pb->cpu_cnt = j;
8024 free(online);
8025
8026 return pb;
8027
8028 error:
8029 free(online);
8030 if (pb)
8031 perf_buffer__free(pb);
8032 return ERR_PTR(err);
8033 }
8034
8035 struct perf_sample_raw {
8036 struct perf_event_header header;
8037 uint32_t size;
8038 char data[0];
8039 };
8040
8041 struct perf_sample_lost {
8042 struct perf_event_header header;
8043 uint64_t id;
8044 uint64_t lost;
8045 uint64_t sample_id;
8046 };
8047
8048 static enum bpf_perf_event_ret
8049 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
8050 {
8051 struct perf_cpu_buf *cpu_buf = ctx;
8052 struct perf_buffer *pb = cpu_buf->pb;
8053 void *data = e;
8054
8055 /* user wants full control over parsing perf event */
8056 if (pb->event_cb)
8057 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
8058
8059 switch (e->type) {
8060 case PERF_RECORD_SAMPLE: {
8061 struct perf_sample_raw *s = data;
8062
8063 if (pb->sample_cb)
8064 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
8065 break;
8066 }
8067 case PERF_RECORD_LOST: {
8068 struct perf_sample_lost *s = data;
8069
8070 if (pb->lost_cb)
8071 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
8072 break;
8073 }
8074 default:
8075 pr_warn("unknown perf sample type %d\n", e->type);
8076 return LIBBPF_PERF_EVENT_ERROR;
8077 }
8078 return LIBBPF_PERF_EVENT_CONT;
8079 }
8080
8081 static int perf_buffer__process_records(struct perf_buffer *pb,
8082 struct perf_cpu_buf *cpu_buf)
8083 {
8084 enum bpf_perf_event_ret ret;
8085
8086 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
8087 pb->page_size, &cpu_buf->buf,
8088 &cpu_buf->buf_size,
8089 perf_buffer__process_record, cpu_buf);
8090 if (ret != LIBBPF_PERF_EVENT_CONT)
8091 return ret;
8092 return 0;
8093 }
8094
8095 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
8096 {
8097 int i, cnt, err;
8098
8099 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
8100 for (i = 0; i < cnt; i++) {
8101 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
8102
8103 err = perf_buffer__process_records(pb, cpu_buf);
8104 if (err) {
8105 pr_warn("error while processing records: %d\n", err);
8106 return err;
8107 }
8108 }
8109 return cnt < 0 ? -errno : cnt;
8110 }
8111
8112 struct bpf_prog_info_array_desc {
8113 int array_offset; /* e.g. offset of jited_prog_insns */
8114 int count_offset; /* e.g. offset of jited_prog_len */
8115 int size_offset; /* > 0: offset of rec size,
8116 * < 0: fix size of -size_offset
8117 */
8118 };
8119
8120 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
8121 [BPF_PROG_INFO_JITED_INSNS] = {
8122 offsetof(struct bpf_prog_info, jited_prog_insns),
8123 offsetof(struct bpf_prog_info, jited_prog_len),
8124 -1,
8125 },
8126 [BPF_PROG_INFO_XLATED_INSNS] = {
8127 offsetof(struct bpf_prog_info, xlated_prog_insns),
8128 offsetof(struct bpf_prog_info, xlated_prog_len),
8129 -1,
8130 },
8131 [BPF_PROG_INFO_MAP_IDS] = {
8132 offsetof(struct bpf_prog_info, map_ids),
8133 offsetof(struct bpf_prog_info, nr_map_ids),
8134 -(int)sizeof(__u32),
8135 },
8136 [BPF_PROG_INFO_JITED_KSYMS] = {
8137 offsetof(struct bpf_prog_info, jited_ksyms),
8138 offsetof(struct bpf_prog_info, nr_jited_ksyms),
8139 -(int)sizeof(__u64),
8140 },
8141 [BPF_PROG_INFO_JITED_FUNC_LENS] = {
8142 offsetof(struct bpf_prog_info, jited_func_lens),
8143 offsetof(struct bpf_prog_info, nr_jited_func_lens),
8144 -(int)sizeof(__u32),
8145 },
8146 [BPF_PROG_INFO_FUNC_INFO] = {
8147 offsetof(struct bpf_prog_info, func_info),
8148 offsetof(struct bpf_prog_info, nr_func_info),
8149 offsetof(struct bpf_prog_info, func_info_rec_size),
8150 },
8151 [BPF_PROG_INFO_LINE_INFO] = {
8152 offsetof(struct bpf_prog_info, line_info),
8153 offsetof(struct bpf_prog_info, nr_line_info),
8154 offsetof(struct bpf_prog_info, line_info_rec_size),
8155 },
8156 [BPF_PROG_INFO_JITED_LINE_INFO] = {
8157 offsetof(struct bpf_prog_info, jited_line_info),
8158 offsetof(struct bpf_prog_info, nr_jited_line_info),
8159 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
8160 },
8161 [BPF_PROG_INFO_PROG_TAGS] = {
8162 offsetof(struct bpf_prog_info, prog_tags),
8163 offsetof(struct bpf_prog_info, nr_prog_tags),
8164 -(int)sizeof(__u8) * BPF_TAG_SIZE,
8165 },
8166
8167 };
8168
8169 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
8170 int offset)
8171 {
8172 __u32 *array = (__u32 *)info;
8173
8174 if (offset >= 0)
8175 return array[offset / sizeof(__u32)];
8176 return -(int)offset;
8177 }
8178
8179 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
8180 int offset)
8181 {
8182 __u64 *array = (__u64 *)info;
8183
8184 if (offset >= 0)
8185 return array[offset / sizeof(__u64)];
8186 return -(int)offset;
8187 }
8188
8189 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
8190 __u32 val)
8191 {
8192 __u32 *array = (__u32 *)info;
8193
8194 if (offset >= 0)
8195 array[offset / sizeof(__u32)] = val;
8196 }
8197
8198 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
8199 __u64 val)
8200 {
8201 __u64 *array = (__u64 *)info;
8202
8203 if (offset >= 0)
8204 array[offset / sizeof(__u64)] = val;
8205 }
8206
8207 struct bpf_prog_info_linear *
8208 bpf_program__get_prog_info_linear(int fd, __u64 arrays)
8209 {
8210 struct bpf_prog_info_linear *info_linear;
8211 struct bpf_prog_info info = {};
8212 __u32 info_len = sizeof(info);
8213 __u32 data_len = 0;
8214 int i, err;
8215 void *ptr;
8216
8217 if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
8218 return ERR_PTR(-EINVAL);
8219
8220 /* step 1: get array dimensions */
8221 err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
8222 if (err) {
8223 pr_debug("can't get prog info: %s", strerror(errno));
8224 return ERR_PTR(-EFAULT);
8225 }
8226
8227 /* step 2: calculate total size of all arrays */
8228 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8229 bool include_array = (arrays & (1UL << i)) > 0;
8230 struct bpf_prog_info_array_desc *desc;
8231 __u32 count, size;
8232
8233 desc = bpf_prog_info_array_desc + i;
8234
8235 /* kernel is too old to support this field */
8236 if (info_len < desc->array_offset + sizeof(__u32) ||
8237 info_len < desc->count_offset + sizeof(__u32) ||
8238 (desc->size_offset > 0 && info_len < desc->size_offset))
8239 include_array = false;
8240
8241 if (!include_array) {
8242 arrays &= ~(1UL << i); /* clear the bit */
8243 continue;
8244 }
8245
8246 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
8247 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
8248
8249 data_len += count * size;
8250 }
8251
8252 /* step 3: allocate continuous memory */
8253 data_len = roundup(data_len, sizeof(__u64));
8254 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
8255 if (!info_linear)
8256 return ERR_PTR(-ENOMEM);
8257
8258 /* step 4: fill data to info_linear->info */
8259 info_linear->arrays = arrays;
8260 memset(&info_linear->info, 0, sizeof(info));
8261 ptr = info_linear->data;
8262
8263 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8264 struct bpf_prog_info_array_desc *desc;
8265 __u32 count, size;
8266
8267 if ((arrays & (1UL << i)) == 0)
8268 continue;
8269
8270 desc = bpf_prog_info_array_desc + i;
8271 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
8272 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
8273 bpf_prog_info_set_offset_u32(&info_linear->info,
8274 desc->count_offset, count);
8275 bpf_prog_info_set_offset_u32(&info_linear->info,
8276 desc->size_offset, size);
8277 bpf_prog_info_set_offset_u64(&info_linear->info,
8278 desc->array_offset,
8279 ptr_to_u64(ptr));
8280 ptr += count * size;
8281 }
8282
8283 /* step 5: call syscall again to get required arrays */
8284 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
8285 if (err) {
8286 pr_debug("can't get prog info: %s", strerror(errno));
8287 free(info_linear);
8288 return ERR_PTR(-EFAULT);
8289 }
8290
8291 /* step 6: verify the data */
8292 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8293 struct bpf_prog_info_array_desc *desc;
8294 __u32 v1, v2;
8295
8296 if ((arrays & (1UL << i)) == 0)
8297 continue;
8298
8299 desc = bpf_prog_info_array_desc + i;
8300 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
8301 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
8302 desc->count_offset);
8303 if (v1 != v2)
8304 pr_warn("%s: mismatch in element count\n", __func__);
8305
8306 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
8307 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
8308 desc->size_offset);
8309 if (v1 != v2)
8310 pr_warn("%s: mismatch in rec size\n", __func__);
8311 }
8312
8313 /* step 7: update info_len and data_len */
8314 info_linear->info_len = sizeof(struct bpf_prog_info);
8315 info_linear->data_len = data_len;
8316
8317 return info_linear;
8318 }
8319
8320 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
8321 {
8322 int i;
8323
8324 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8325 struct bpf_prog_info_array_desc *desc;
8326 __u64 addr, offs;
8327
8328 if ((info_linear->arrays & (1UL << i)) == 0)
8329 continue;
8330
8331 desc = bpf_prog_info_array_desc + i;
8332 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
8333 desc->array_offset);
8334 offs = addr - ptr_to_u64(info_linear->data);
8335 bpf_prog_info_set_offset_u64(&info_linear->info,
8336 desc->array_offset, offs);
8337 }
8338 }
8339
8340 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
8341 {
8342 int i;
8343
8344 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8345 struct bpf_prog_info_array_desc *desc;
8346 __u64 addr, offs;
8347
8348 if ((info_linear->arrays & (1UL << i)) == 0)
8349 continue;
8350
8351 desc = bpf_prog_info_array_desc + i;
8352 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
8353 desc->array_offset);
8354 addr = offs + ptr_to_u64(info_linear->data);
8355 bpf_prog_info_set_offset_u64(&info_linear->info,
8356 desc->array_offset, addr);
8357 }
8358 }
8359
8360 int bpf_program__set_attach_target(struct bpf_program *prog,
8361 int attach_prog_fd,
8362 const char *attach_func_name)
8363 {
8364 int btf_id;
8365
8366 if (!prog || attach_prog_fd < 0 || !attach_func_name)
8367 return -EINVAL;
8368
8369 if (attach_prog_fd)
8370 btf_id = libbpf_find_prog_btf_id(attach_func_name,
8371 attach_prog_fd);
8372 else
8373 btf_id = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
8374 attach_func_name,
8375 prog->expected_attach_type);
8376
8377 if (btf_id < 0)
8378 return btf_id;
8379
8380 prog->attach_btf_id = btf_id;
8381 prog->attach_prog_fd = attach_prog_fd;
8382 return 0;
8383 }
8384
8385 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
8386 {
8387 int err = 0, n, len, start, end = -1;
8388 bool *tmp;
8389
8390 *mask = NULL;
8391 *mask_sz = 0;
8392
8393 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
8394 while (*s) {
8395 if (*s == ',' || *s == '\n') {
8396 s++;
8397 continue;
8398 }
8399 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
8400 if (n <= 0 || n > 2) {
8401 pr_warn("Failed to get CPU range %s: %d\n", s, n);
8402 err = -EINVAL;
8403 goto cleanup;
8404 } else if (n == 1) {
8405 end = start;
8406 }
8407 if (start < 0 || start > end) {
8408 pr_warn("Invalid CPU range [%d,%d] in %s\n",
8409 start, end, s);
8410 err = -EINVAL;
8411 goto cleanup;
8412 }
8413 tmp = realloc(*mask, end + 1);
8414 if (!tmp) {
8415 err = -ENOMEM;
8416 goto cleanup;
8417 }
8418 *mask = tmp;
8419 memset(tmp + *mask_sz, 0, start - *mask_sz);
8420 memset(tmp + start, 1, end - start + 1);
8421 *mask_sz = end + 1;
8422 s += len;
8423 }
8424 if (!*mask_sz) {
8425 pr_warn("Empty CPU range\n");
8426 return -EINVAL;
8427 }
8428 return 0;
8429 cleanup:
8430 free(*mask);
8431 *mask = NULL;
8432 return err;
8433 }
8434
8435 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
8436 {
8437 int fd, err = 0, len;
8438 char buf[128];
8439
8440 fd = open(fcpu, O_RDONLY);
8441 if (fd < 0) {
8442 err = -errno;
8443 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
8444 return err;
8445 }
8446 len = read(fd, buf, sizeof(buf));
8447 close(fd);
8448 if (len <= 0) {
8449 err = len ? -errno : -EINVAL;
8450 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
8451 return err;
8452 }
8453 if (len >= sizeof(buf)) {
8454 pr_warn("CPU mask is too big in file %s\n", fcpu);
8455 return -E2BIG;
8456 }
8457 buf[len] = '\0';
8458
8459 return parse_cpu_mask_str(buf, mask, mask_sz);
8460 }
8461
8462 int libbpf_num_possible_cpus(void)
8463 {
8464 static const char *fcpu = "/sys/devices/system/cpu/possible";
8465 static int cpus;
8466 int err, n, i, tmp_cpus;
8467 bool *mask;
8468
8469 tmp_cpus = READ_ONCE(cpus);
8470 if (tmp_cpus > 0)
8471 return tmp_cpus;
8472
8473 err = parse_cpu_mask_file(fcpu, &mask, &n);
8474 if (err)
8475 return err;
8476
8477 tmp_cpus = 0;
8478 for (i = 0; i < n; i++) {
8479 if (mask[i])
8480 tmp_cpus++;
8481 }
8482 free(mask);
8483
8484 WRITE_ONCE(cpus, tmp_cpus);
8485 return tmp_cpus;
8486 }
8487
8488 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
8489 const struct bpf_object_open_opts *opts)
8490 {
8491 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
8492 .object_name = s->name,
8493 );
8494 struct bpf_object *obj;
8495 int i;
8496
8497 /* Attempt to preserve opts->object_name, unless overriden by user
8498 * explicitly. Overwriting object name for skeletons is discouraged,
8499 * as it breaks global data maps, because they contain object name
8500 * prefix as their own map name prefix. When skeleton is generated,
8501 * bpftool is making an assumption that this name will stay the same.
8502 */
8503 if (opts) {
8504 memcpy(&skel_opts, opts, sizeof(*opts));
8505 if (!opts->object_name)
8506 skel_opts.object_name = s->name;
8507 }
8508
8509 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
8510 if (IS_ERR(obj)) {
8511 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n",
8512 s->name, PTR_ERR(obj));
8513 return PTR_ERR(obj);
8514 }
8515
8516 *s->obj = obj;
8517
8518 for (i = 0; i < s->map_cnt; i++) {
8519 struct bpf_map **map = s->maps[i].map;
8520 const char *name = s->maps[i].name;
8521 void **mmaped = s->maps[i].mmaped;
8522
8523 *map = bpf_object__find_map_by_name(obj, name);
8524 if (!*map) {
8525 pr_warn("failed to find skeleton map '%s'\n", name);
8526 return -ESRCH;
8527 }
8528
8529 /* externs shouldn't be pre-setup from user code */
8530 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
8531 *mmaped = (*map)->mmaped;
8532 }
8533
8534 for (i = 0; i < s->prog_cnt; i++) {
8535 struct bpf_program **prog = s->progs[i].prog;
8536 const char *name = s->progs[i].name;
8537
8538 *prog = bpf_object__find_program_by_name(obj, name);
8539 if (!*prog) {
8540 pr_warn("failed to find skeleton program '%s'\n", name);
8541 return -ESRCH;
8542 }
8543 }
8544
8545 return 0;
8546 }
8547
8548 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
8549 {
8550 int i, err;
8551
8552 err = bpf_object__load(*s->obj);
8553 if (err) {
8554 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
8555 return err;
8556 }
8557
8558 for (i = 0; i < s->map_cnt; i++) {
8559 struct bpf_map *map = *s->maps[i].map;
8560 size_t mmap_sz = bpf_map_mmap_sz(map);
8561 int prot, map_fd = bpf_map__fd(map);
8562 void **mmaped = s->maps[i].mmaped;
8563
8564 if (!mmaped)
8565 continue;
8566
8567 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
8568 *mmaped = NULL;
8569 continue;
8570 }
8571
8572 if (map->def.map_flags & BPF_F_RDONLY_PROG)
8573 prot = PROT_READ;
8574 else
8575 prot = PROT_READ | PROT_WRITE;
8576
8577 /* Remap anonymous mmap()-ed "map initialization image" as
8578 * a BPF map-backed mmap()-ed memory, but preserving the same
8579 * memory address. This will cause kernel to change process'
8580 * page table to point to a different piece of kernel memory,
8581 * but from userspace point of view memory address (and its
8582 * contents, being identical at this point) will stay the
8583 * same. This mapping will be released by bpf_object__close()
8584 * as per normal clean up procedure, so we don't need to worry
8585 * about it from skeleton's clean up perspective.
8586 */
8587 *mmaped = mmap(map->mmaped, mmap_sz, prot,
8588 MAP_SHARED | MAP_FIXED, map_fd, 0);
8589 if (*mmaped == MAP_FAILED) {
8590 err = -errno;
8591 *mmaped = NULL;
8592 pr_warn("failed to re-mmap() map '%s': %d\n",
8593 bpf_map__name(map), err);
8594 return err;
8595 }
8596 }
8597
8598 return 0;
8599 }
8600
8601 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
8602 {
8603 int i;
8604
8605 for (i = 0; i < s->prog_cnt; i++) {
8606 struct bpf_program *prog = *s->progs[i].prog;
8607 struct bpf_link **link = s->progs[i].link;
8608 const struct bpf_sec_def *sec_def;
8609 const char *sec_name = bpf_program__title(prog, false);
8610
8611 sec_def = find_sec_def(sec_name);
8612 if (!sec_def || !sec_def->attach_fn)
8613 continue;
8614
8615 *link = sec_def->attach_fn(sec_def, prog);
8616 if (IS_ERR(*link)) {
8617 pr_warn("failed to auto-attach program '%s': %ld\n",
8618 bpf_program__name(prog), PTR_ERR(*link));
8619 return PTR_ERR(*link);
8620 }
8621 }
8622
8623 return 0;
8624 }
8625
8626 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
8627 {
8628 int i;
8629
8630 for (i = 0; i < s->prog_cnt; i++) {
8631 struct bpf_link **link = s->progs[i].link;
8632
8633 if (!IS_ERR_OR_NULL(*link))
8634 bpf_link__destroy(*link);
8635 *link = NULL;
8636 }
8637 }
8638
8639 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
8640 {
8641 if (s->progs)
8642 bpf_object__detach_skeleton(s);
8643 if (s->obj)
8644 bpf_object__close(*s->obj);
8645 free(s->maps);
8646 free(s->progs);
8647 free(s);
8648 }
A mirror of Linus' kernel repository