1 // SPDX-License-Identifier: GPL-2.0
6 #include <bpf/libbpf.h>
9 #include <linux/string.h>
10 #include <internal/lib.h>
11 #include <symbol/kallsyms.h>
12 #include "bpf-event.h"
13 #include "bpf-utils.h"
23 #include "util/synthetic-events.h"
25 #ifndef HAVE_LIBBPF_BTF__LOAD_FROM_KERNEL_BY_ID
26 struct btf
*btf__load_from_kernel_by_id(__u32 id
)
29 #pragma GCC diagnostic push
30 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
31 int err
= btf__get_from_id(id
, &btf
);
32 #pragma GCC diagnostic pop
34 return err
? ERR_PTR(err
) : btf
;
38 #ifndef HAVE_LIBBPF_BPF_PROG_LOAD
39 int bpf_prog_load(enum bpf_prog_type prog_type
,
40 const char *prog_name __maybe_unused
,
42 const struct bpf_insn
*insns
, size_t insn_cnt
,
43 const struct bpf_prog_load_opts
*opts
)
45 #pragma GCC diagnostic push
46 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
47 return bpf_load_program(prog_type
, insns
, insn_cnt
, license
,
48 opts
->kern_version
, opts
->log_buf
, opts
->log_size
);
49 #pragma GCC diagnostic pop
53 #ifndef HAVE_LIBBPF_BPF_OBJECT__NEXT_PROGRAM
55 bpf_object__next_program(const struct bpf_object
*obj
, struct bpf_program
*prev
)
57 #pragma GCC diagnostic push
58 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
59 return bpf_program__next(prev
, obj
);
60 #pragma GCC diagnostic pop
64 struct bpf_map
* __weak
65 bpf_object__next_map(const struct bpf_object
*obj
, const struct bpf_map
*prev
)
67 #pragma GCC diagnostic push
68 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
69 return bpf_map__next(prev
, obj
);
70 #pragma GCC diagnostic pop
74 btf__raw_data(const struct btf
*btf_ro
, __u32
*size
)
76 #pragma GCC diagnostic push
77 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
78 return btf__get_raw_data(btf_ro
, size
);
79 #pragma GCC diagnostic pop
82 static int snprintf_hex(char *buf
, size_t size
, unsigned char *data
, size_t len
)
87 for (i
= 0; i
< len
; i
++)
88 ret
+= snprintf(buf
+ ret
, size
- ret
, "%02x", data
[i
]);
92 static int machine__process_bpf_event_load(struct machine
*machine
,
93 union perf_event
*event
,
94 struct perf_sample
*sample __maybe_unused
)
96 struct bpf_prog_info_node
*info_node
;
97 struct perf_env
*env
= machine
->env
;
98 struct perf_bpil
*info_linear
;
99 int id
= event
->bpf
.id
;
102 /* perf-record, no need to handle bpf-event */
106 info_node
= perf_env__find_bpf_prog_info(env
, id
);
109 info_linear
= info_node
->info_linear
;
111 for (i
= 0; i
< info_linear
->info
.nr_jited_ksyms
; i
++) {
112 u64
*addrs
= (u64
*)(uintptr_t)(info_linear
->info
.jited_ksyms
);
114 struct map
*map
= maps__find(machine__kernel_maps(machine
), addr
);
117 map
->dso
->binary_type
= DSO_BINARY_TYPE__BPF_PROG_INFO
;
118 map
->dso
->bpf_prog
.id
= id
;
119 map
->dso
->bpf_prog
.sub_id
= i
;
120 map
->dso
->bpf_prog
.env
= env
;
126 int machine__process_bpf(struct machine
*machine
, union perf_event
*event
,
127 struct perf_sample
*sample
)
130 perf_event__fprintf_bpf(event
, stdout
);
132 switch (event
->bpf
.type
) {
133 case PERF_BPF_EVENT_PROG_LOAD
:
134 return machine__process_bpf_event_load(machine
, event
, sample
);
136 case PERF_BPF_EVENT_PROG_UNLOAD
:
138 * Do not free bpf_prog_info and btf of the program here,
139 * as annotation still need them. They will be freed at
140 * the end of the session.
144 pr_debug("unexpected bpf event type of %d\n", event
->bpf
.type
);
150 static int perf_env__fetch_btf(struct perf_env
*env
,
154 struct btf_node
*node
;
158 data
= btf__raw_data(btf
, &data_size
);
160 node
= malloc(data_size
+ sizeof(struct btf_node
));
165 node
->data_size
= data_size
;
166 memcpy(node
->data
, data
, data_size
);
168 if (!perf_env__insert_btf(env
, node
)) {
169 /* Insertion failed because of a duplicate. */
176 static int synthesize_bpf_prog_name(char *buf
, int size
,
177 struct bpf_prog_info
*info
,
181 u8 (*prog_tags
)[BPF_TAG_SIZE
] = (void *)(uintptr_t)(info
->prog_tags
);
182 void *func_infos
= (void *)(uintptr_t)(info
->func_info
);
183 u32 sub_prog_cnt
= info
->nr_jited_ksyms
;
184 const struct bpf_func_info
*finfo
;
185 const char *short_name
= NULL
;
186 const struct btf_type
*t
;
189 name_len
= snprintf(buf
, size
, "bpf_prog_");
190 name_len
+= snprintf_hex(buf
+ name_len
, size
- name_len
,
191 prog_tags
[sub_id
], BPF_TAG_SIZE
);
193 finfo
= func_infos
+ sub_id
* info
->func_info_rec_size
;
194 t
= btf__type_by_id(btf
, finfo
->type_id
);
195 short_name
= btf__name_by_offset(btf
, t
->name_off
);
196 } else if (sub_id
== 0 && sub_prog_cnt
== 1) {
199 short_name
= info
->name
;
203 name_len
+= snprintf(buf
+ name_len
, size
- name_len
,
209 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf
210 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And
211 * one PERF_RECORD_KSYMBOL is generated for each sub program.
216 * -2 for lack of kernel support.
218 static int perf_event__synthesize_one_bpf_prog(struct perf_session
*session
,
219 perf_event__handler_t process
,
220 struct machine
*machine
,
222 union perf_event
*event
,
223 struct record_opts
*opts
)
225 struct perf_record_ksymbol
*ksymbol_event
= &event
->ksymbol
;
226 struct perf_record_bpf_event
*bpf_event
= &event
->bpf
;
227 struct perf_tool
*tool
= session
->tool
;
228 struct bpf_prog_info_node
*info_node
;
229 struct perf_bpil
*info_linear
;
230 struct bpf_prog_info
*info
;
231 struct btf
*btf
= NULL
;
232 struct perf_env
*env
;
238 * for perf-record and perf-report use header.env;
239 * otherwise, use global perf_env.
241 env
= session
->data
? &session
->header
.env
: &perf_env
;
243 arrays
= 1UL << PERF_BPIL_JITED_KSYMS
;
244 arrays
|= 1UL << PERF_BPIL_JITED_FUNC_LENS
;
245 arrays
|= 1UL << PERF_BPIL_FUNC_INFO
;
246 arrays
|= 1UL << PERF_BPIL_PROG_TAGS
;
247 arrays
|= 1UL << PERF_BPIL_JITED_INSNS
;
248 arrays
|= 1UL << PERF_BPIL_LINE_INFO
;
249 arrays
|= 1UL << PERF_BPIL_JITED_LINE_INFO
;
251 info_linear
= get_bpf_prog_info_linear(fd
, arrays
);
252 if (IS_ERR_OR_NULL(info_linear
)) {
254 pr_debug("%s: failed to get BPF program info. aborting\n", __func__
);
258 if (info_linear
->info_len
< offsetof(struct bpf_prog_info
, prog_tags
)) {
260 pr_debug("%s: the kernel is too old, aborting\n", __func__
);
264 info
= &info_linear
->info
;
265 if (!info
->jited_ksyms
) {
270 /* number of ksyms, func_lengths, and tags should match */
271 sub_prog_cnt
= info
->nr_jited_ksyms
;
272 if (sub_prog_cnt
!= info
->nr_prog_tags
||
273 sub_prog_cnt
!= info
->nr_jited_func_lens
) {
278 /* check BTF func info support */
279 if (info
->btf_id
&& info
->nr_func_info
&& info
->func_info_rec_size
) {
280 /* btf func info number should be same as sub_prog_cnt */
281 if (sub_prog_cnt
!= info
->nr_func_info
) {
282 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__
);
286 btf
= btf__load_from_kernel_by_id(info
->btf_id
);
287 if (libbpf_get_error(btf
)) {
288 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__
, info
->btf_id
);
292 perf_env__fetch_btf(env
, info
->btf_id
, btf
);
295 /* Synthesize PERF_RECORD_KSYMBOL */
296 for (i
= 0; i
< sub_prog_cnt
; i
++) {
297 __u32
*prog_lens
= (__u32
*)(uintptr_t)(info
->jited_func_lens
);
298 __u64
*prog_addrs
= (__u64
*)(uintptr_t)(info
->jited_ksyms
);
301 *ksymbol_event
= (struct perf_record_ksymbol
) {
303 .type
= PERF_RECORD_KSYMBOL
,
304 .size
= offsetof(struct perf_record_ksymbol
, name
),
306 .addr
= prog_addrs
[i
],
308 .ksym_type
= PERF_RECORD_KSYMBOL_TYPE_BPF
,
312 name_len
= synthesize_bpf_prog_name(ksymbol_event
->name
,
313 KSYM_NAME_LEN
, info
, btf
, i
);
314 ksymbol_event
->header
.size
+= PERF_ALIGN(name_len
+ 1,
317 memset((void *)event
+ event
->header
.size
, 0, machine
->id_hdr_size
);
318 event
->header
.size
+= machine
->id_hdr_size
;
319 err
= perf_tool__process_synth_event(tool
, event
,
323 if (!opts
->no_bpf_event
) {
324 /* Synthesize PERF_RECORD_BPF_EVENT */
325 *bpf_event
= (struct perf_record_bpf_event
) {
327 .type
= PERF_RECORD_BPF_EVENT
,
328 .size
= sizeof(struct perf_record_bpf_event
),
330 .type
= PERF_BPF_EVENT_PROG_LOAD
,
334 memcpy(bpf_event
->tag
, info
->tag
, BPF_TAG_SIZE
);
335 memset((void *)event
+ event
->header
.size
, 0, machine
->id_hdr_size
);
336 event
->header
.size
+= machine
->id_hdr_size
;
338 /* save bpf_prog_info to env */
339 info_node
= malloc(sizeof(struct bpf_prog_info_node
));
345 info_node
->info_linear
= info_linear
;
346 perf_env__insert_bpf_prog_info(env
, info_node
);
350 * process after saving bpf_prog_info to env, so that
351 * required information is ready for look up
353 err
= perf_tool__process_synth_event(tool
, event
,
363 struct kallsyms_parse
{
364 union perf_event
*event
;
365 perf_event__handler_t process
;
366 struct machine
*machine
;
367 struct perf_tool
*tool
;
371 process_bpf_image(char *name
, u64 addr
, struct kallsyms_parse
*data
)
373 struct machine
*machine
= data
->machine
;
374 union perf_event
*event
= data
->event
;
375 struct perf_record_ksymbol
*ksymbol
;
378 ksymbol
= &event
->ksymbol
;
380 *ksymbol
= (struct perf_record_ksymbol
) {
382 .type
= PERF_RECORD_KSYMBOL
,
383 .size
= offsetof(struct perf_record_ksymbol
, name
),
387 .ksym_type
= PERF_RECORD_KSYMBOL_TYPE_BPF
,
391 len
= scnprintf(ksymbol
->name
, KSYM_NAME_LEN
, "%s", name
);
392 ksymbol
->header
.size
+= PERF_ALIGN(len
+ 1, sizeof(u64
));
393 memset((void *) event
+ event
->header
.size
, 0, machine
->id_hdr_size
);
394 event
->header
.size
+= machine
->id_hdr_size
;
396 return perf_tool__process_synth_event(data
->tool
, event
, machine
,
401 kallsyms_process_symbol(void *data
, const char *_name
,
402 char type __maybe_unused
, u64 start
)
404 char disp
[KSYM_NAME_LEN
];
409 module
= strchr(_name
, '\t');
413 /* We are going after [bpf] module ... */
414 if (strcmp(module
+ 1, "[bpf]"))
417 name
= memdup(_name
, (module
- _name
) + 1);
421 name
[module
- _name
] = 0;
423 /* .. and only for trampolines and dispatchers */
424 if ((sscanf(name
, "bpf_trampoline_%lu", &id
) == 1) ||
425 (sscanf(name
, "bpf_dispatcher_%s", disp
) == 1))
426 err
= process_bpf_image(name
, start
, data
);
432 int perf_event__synthesize_bpf_events(struct perf_session
*session
,
433 perf_event__handler_t process
,
434 struct machine
*machine
,
435 struct record_opts
*opts
)
437 const char *kallsyms_filename
= "/proc/kallsyms";
438 struct kallsyms_parse arg
;
439 union perf_event
*event
;
444 event
= malloc(sizeof(event
->bpf
) + KSYM_NAME_LEN
+ machine
->id_hdr_size
);
448 /* Synthesize all the bpf programs in system. */
450 err
= bpf_prog_get_next_id(id
, &id
);
452 if (errno
== ENOENT
) {
456 pr_debug("%s: can't get next program: %s%s\n",
457 __func__
, strerror(errno
),
458 errno
== EINVAL
? " -- kernel too old?" : "");
459 /* don't report error on old kernel or EPERM */
460 err
= (errno
== EINVAL
|| errno
== EPERM
) ? 0 : -1;
463 fd
= bpf_prog_get_fd_by_id(id
);
465 pr_debug("%s: failed to get fd for prog_id %u\n",
470 err
= perf_event__synthesize_one_bpf_prog(session
, process
,
475 /* do not return error for old kernel */
482 /* Synthesize all the bpf images - trampolines/dispatchers. */
483 if (symbol_conf
.kallsyms_name
!= NULL
)
484 kallsyms_filename
= symbol_conf
.kallsyms_name
;
486 arg
= (struct kallsyms_parse
) {
490 .tool
= session
->tool
,
493 if (kallsyms__parse(kallsyms_filename
, &arg
, kallsyms_process_symbol
)) {
494 pr_err("%s: failed to synthesize bpf images: %s\n",
495 __func__
, strerror(errno
));
502 static void perf_env__add_bpf_info(struct perf_env
*env
, u32 id
)
504 struct bpf_prog_info_node
*info_node
;
505 struct perf_bpil
*info_linear
;
506 struct btf
*btf
= NULL
;
511 fd
= bpf_prog_get_fd_by_id(id
);
515 arrays
= 1UL << PERF_BPIL_JITED_KSYMS
;
516 arrays
|= 1UL << PERF_BPIL_JITED_FUNC_LENS
;
517 arrays
|= 1UL << PERF_BPIL_FUNC_INFO
;
518 arrays
|= 1UL << PERF_BPIL_PROG_TAGS
;
519 arrays
|= 1UL << PERF_BPIL_JITED_INSNS
;
520 arrays
|= 1UL << PERF_BPIL_LINE_INFO
;
521 arrays
|= 1UL << PERF_BPIL_JITED_LINE_INFO
;
523 info_linear
= get_bpf_prog_info_linear(fd
, arrays
);
524 if (IS_ERR_OR_NULL(info_linear
)) {
525 pr_debug("%s: failed to get BPF program info. aborting\n", __func__
);
529 btf_id
= info_linear
->info
.btf_id
;
531 info_node
= malloc(sizeof(struct bpf_prog_info_node
));
533 info_node
->info_linear
= info_linear
;
534 perf_env__insert_bpf_prog_info(env
, info_node
);
541 btf
= btf__load_from_kernel_by_id(btf_id
);
542 if (libbpf_get_error(btf
)) {
543 pr_debug("%s: failed to get BTF of id %u, aborting\n",
547 perf_env__fetch_btf(env
, btf_id
, btf
);
554 static int bpf_event__sb_cb(union perf_event
*event
, void *data
)
556 struct perf_env
*env
= data
;
558 if (event
->header
.type
!= PERF_RECORD_BPF_EVENT
)
561 switch (event
->bpf
.type
) {
562 case PERF_BPF_EVENT_PROG_LOAD
:
563 perf_env__add_bpf_info(env
, event
->bpf
.id
);
565 case PERF_BPF_EVENT_PROG_UNLOAD
:
567 * Do not free bpf_prog_info and btf of the program here,
568 * as annotation still need them. They will be freed at
569 * the end of the session.
573 pr_debug("unexpected bpf event type of %d\n", event
->bpf
.type
);
580 int evlist__add_bpf_sb_event(struct evlist
*evlist
, struct perf_env
*env
)
582 struct perf_event_attr attr
= {
583 .type
= PERF_TYPE_SOFTWARE
,
584 .config
= PERF_COUNT_SW_DUMMY
,
588 .size
= sizeof(attr
), /* to capture ABI version */
592 * Older gcc versions don't support designated initializers, like above,
593 * for unnamed union members, such as the following:
595 attr
.wakeup_watermark
= 1;
597 return evlist__add_sb_event(evlist
, &attr
, bpf_event__sb_cb
, env
);
600 void bpf_event__print_bpf_prog_info(struct bpf_prog_info
*info
,
601 struct perf_env
*env
,
604 __u32
*prog_lens
= (__u32
*)(uintptr_t)(info
->jited_func_lens
);
605 __u64
*prog_addrs
= (__u64
*)(uintptr_t)(info
->jited_ksyms
);
606 char name
[KSYM_NAME_LEN
];
607 struct btf
*btf
= NULL
;
610 sub_prog_cnt
= info
->nr_jited_ksyms
;
611 if (sub_prog_cnt
!= info
->nr_prog_tags
||
612 sub_prog_cnt
!= info
->nr_jited_func_lens
)
616 struct btf_node
*node
;
618 node
= perf_env__find_btf(env
, info
->btf_id
);
620 btf
= btf__new((__u8
*)(node
->data
),
624 if (sub_prog_cnt
== 1) {
625 synthesize_bpf_prog_name(name
, KSYM_NAME_LEN
, info
, btf
, 0);
626 fprintf(fp
, "# bpf_prog_info %u: %s addr 0x%llx size %u\n",
627 info
->id
, name
, prog_addrs
[0], prog_lens
[0]);
631 fprintf(fp
, "# bpf_prog_info %u:\n", info
->id
);
632 for (i
= 0; i
< sub_prog_cnt
; i
++) {
633 synthesize_bpf_prog_name(name
, KSYM_NAME_LEN
, info
, btf
, i
);
635 fprintf(fp
, "# \tsub_prog %u: %s addr 0x%llx size %u\n",
636 i
, name
, prog_addrs
[i
], prog_lens
[i
]);