1 // SPDX-License-Identifier: GPL-2.0
16 #include "map_symbol.h"
18 #include "mem-events.h"
28 #include <sys/types.h>
32 #include "linux/hash.h"
34 #include "bpf-event.h"
35 #include <internal/lib.h> // page_size
38 #include <linux/ctype.h>
39 #include <symbol/kallsyms.h>
40 #include <linux/mman.h>
41 #include <linux/string.h>
42 #include <linux/zalloc.h>
44 static void __machine__remove_thread(struct machine
*machine
, struct thread
*th
, bool lock
);
46 static struct dso
*machine__kernel_dso(struct machine
*machine
)
48 return machine
->vmlinux_map
->dso
;
51 static void dsos__init(struct dsos
*dsos
)
53 INIT_LIST_HEAD(&dsos
->head
);
55 init_rwsem(&dsos
->lock
);
58 static void machine__threads_init(struct machine
*machine
)
62 for (i
= 0; i
< THREADS__TABLE_SIZE
; i
++) {
63 struct threads
*threads
= &machine
->threads
[i
];
64 threads
->entries
= RB_ROOT_CACHED
;
65 init_rwsem(&threads
->lock
);
67 INIT_LIST_HEAD(&threads
->dead
);
68 threads
->last_match
= NULL
;
72 static int machine__set_mmap_name(struct machine
*machine
)
74 if (machine__is_host(machine
))
75 machine
->mmap_name
= strdup("[kernel.kallsyms]");
76 else if (machine__is_default_guest(machine
))
77 machine
->mmap_name
= strdup("[guest.kernel.kallsyms]");
78 else if (asprintf(&machine
->mmap_name
, "[guest.kernel.kallsyms.%d]",
80 machine
->mmap_name
= NULL
;
82 return machine
->mmap_name
? 0 : -ENOMEM
;
85 int machine__init(struct machine
*machine
, const char *root_dir
, pid_t pid
)
89 memset(machine
, 0, sizeof(*machine
));
90 maps__init(&machine
->kmaps
, machine
);
91 RB_CLEAR_NODE(&machine
->rb_node
);
92 dsos__init(&machine
->dsos
);
94 machine__threads_init(machine
);
96 machine
->vdso_info
= NULL
;
101 machine
->id_hdr_size
= 0;
102 machine
->kptr_restrict_warned
= false;
103 machine
->comm_exec
= false;
104 machine
->kernel_start
= 0;
105 machine
->vmlinux_map
= NULL
;
107 machine
->root_dir
= strdup(root_dir
);
108 if (machine
->root_dir
== NULL
)
111 if (machine__set_mmap_name(machine
))
114 if (pid
!= HOST_KERNEL_ID
) {
115 struct thread
*thread
= machine__findnew_thread(machine
, -1,
122 snprintf(comm
, sizeof(comm
), "[guest/%d]", pid
);
123 thread__set_comm(thread
, comm
, 0);
127 machine
->current_tid
= NULL
;
132 zfree(&machine
->root_dir
);
133 zfree(&machine
->mmap_name
);
138 struct machine
*machine__new_host(void)
140 struct machine
*machine
= malloc(sizeof(*machine
));
142 if (machine
!= NULL
) {
143 machine__init(machine
, "", HOST_KERNEL_ID
);
145 if (machine__create_kernel_maps(machine
) < 0)
155 struct machine
*machine__new_kallsyms(void)
157 struct machine
*machine
= machine__new_host();
160 * 1) We should switch to machine__load_kallsyms(), i.e. not explicitly
161 * ask for not using the kcore parsing code, once this one is fixed
162 * to create a map per module.
164 if (machine
&& machine__load_kallsyms(machine
, "/proc/kallsyms") <= 0) {
165 machine__delete(machine
);
172 static void dsos__purge(struct dsos
*dsos
)
176 down_write(&dsos
->lock
);
178 list_for_each_entry_safe(pos
, n
, &dsos
->head
, node
) {
179 RB_CLEAR_NODE(&pos
->rb_node
);
181 list_del_init(&pos
->node
);
185 up_write(&dsos
->lock
);
188 static void dsos__exit(struct dsos
*dsos
)
191 exit_rwsem(&dsos
->lock
);
194 void machine__delete_threads(struct machine
*machine
)
199 for (i
= 0; i
< THREADS__TABLE_SIZE
; i
++) {
200 struct threads
*threads
= &machine
->threads
[i
];
201 down_write(&threads
->lock
);
202 nd
= rb_first_cached(&threads
->entries
);
204 struct thread
*t
= rb_entry(nd
, struct thread
, rb_node
);
207 __machine__remove_thread(machine
, t
, false);
209 up_write(&threads
->lock
);
213 void machine__exit(struct machine
*machine
)
220 machine__destroy_kernel_maps(machine
);
221 maps__exit(&machine
->kmaps
);
222 dsos__exit(&machine
->dsos
);
223 machine__exit_vdso(machine
);
224 zfree(&machine
->root_dir
);
225 zfree(&machine
->mmap_name
);
226 zfree(&machine
->current_tid
);
228 for (i
= 0; i
< THREADS__TABLE_SIZE
; i
++) {
229 struct threads
*threads
= &machine
->threads
[i
];
230 struct thread
*thread
, *n
;
232 * Forget about the dead, at this point whatever threads were
233 * left in the dead lists better have a reference count taken
234 * by who is using them, and then, when they drop those references
235 * and it finally hits zero, thread__put() will check and see that
236 * its not in the dead threads list and will not try to remove it
237 * from there, just calling thread__delete() straight away.
239 list_for_each_entry_safe(thread
, n
, &threads
->dead
, node
)
240 list_del_init(&thread
->node
);
242 exit_rwsem(&threads
->lock
);
246 void machine__delete(struct machine
*machine
)
249 machine__exit(machine
);
254 void machines__init(struct machines
*machines
)
256 machine__init(&machines
->host
, "", HOST_KERNEL_ID
);
257 machines
->guests
= RB_ROOT_CACHED
;
260 void machines__exit(struct machines
*machines
)
262 machine__exit(&machines
->host
);
266 struct machine
*machines__add(struct machines
*machines
, pid_t pid
,
267 const char *root_dir
)
269 struct rb_node
**p
= &machines
->guests
.rb_root
.rb_node
;
270 struct rb_node
*parent
= NULL
;
271 struct machine
*pos
, *machine
= malloc(sizeof(*machine
));
272 bool leftmost
= true;
277 if (machine__init(machine
, root_dir
, pid
) != 0) {
284 pos
= rb_entry(parent
, struct machine
, rb_node
);
293 rb_link_node(&machine
->rb_node
, parent
, p
);
294 rb_insert_color_cached(&machine
->rb_node
, &machines
->guests
, leftmost
);
299 void machines__set_comm_exec(struct machines
*machines
, bool comm_exec
)
303 machines
->host
.comm_exec
= comm_exec
;
305 for (nd
= rb_first_cached(&machines
->guests
); nd
; nd
= rb_next(nd
)) {
306 struct machine
*machine
= rb_entry(nd
, struct machine
, rb_node
);
308 machine
->comm_exec
= comm_exec
;
312 struct machine
*machines__find(struct machines
*machines
, pid_t pid
)
314 struct rb_node
**p
= &machines
->guests
.rb_root
.rb_node
;
315 struct rb_node
*parent
= NULL
;
316 struct machine
*machine
;
317 struct machine
*default_machine
= NULL
;
319 if (pid
== HOST_KERNEL_ID
)
320 return &machines
->host
;
324 machine
= rb_entry(parent
, struct machine
, rb_node
);
325 if (pid
< machine
->pid
)
327 else if (pid
> machine
->pid
)
332 default_machine
= machine
;
335 return default_machine
;
338 struct machine
*machines__findnew(struct machines
*machines
, pid_t pid
)
341 const char *root_dir
= "";
342 struct machine
*machine
= machines__find(machines
, pid
);
344 if (machine
&& (machine
->pid
== pid
))
347 if ((pid
!= HOST_KERNEL_ID
) &&
348 (pid
!= DEFAULT_GUEST_KERNEL_ID
) &&
349 (symbol_conf
.guestmount
)) {
350 sprintf(path
, "%s/%d", symbol_conf
.guestmount
, pid
);
351 if (access(path
, R_OK
)) {
352 static struct strlist
*seen
;
355 seen
= strlist__new(NULL
, NULL
);
357 if (!strlist__has_entry(seen
, path
)) {
358 pr_err("Can't access file %s\n", path
);
359 strlist__add(seen
, path
);
367 machine
= machines__add(machines
, pid
, root_dir
);
372 void machines__process_guests(struct machines
*machines
,
373 machine__process_t process
, void *data
)
377 for (nd
= rb_first_cached(&machines
->guests
); nd
; nd
= rb_next(nd
)) {
378 struct machine
*pos
= rb_entry(nd
, struct machine
, rb_node
);
383 void machines__set_id_hdr_size(struct machines
*machines
, u16 id_hdr_size
)
385 struct rb_node
*node
;
386 struct machine
*machine
;
388 machines
->host
.id_hdr_size
= id_hdr_size
;
390 for (node
= rb_first_cached(&machines
->guests
); node
;
391 node
= rb_next(node
)) {
392 machine
= rb_entry(node
, struct machine
, rb_node
);
393 machine
->id_hdr_size
= id_hdr_size
;
399 static void machine__update_thread_pid(struct machine
*machine
,
400 struct thread
*th
, pid_t pid
)
402 struct thread
*leader
;
404 if (pid
== th
->pid_
|| pid
== -1 || th
->pid_
!= -1)
409 if (th
->pid_
== th
->tid
)
412 leader
= __machine__findnew_thread(machine
, th
->pid_
, th
->pid_
);
417 leader
->maps
= maps__new(machine
);
422 if (th
->maps
== leader
->maps
)
427 * Maps are created from MMAP events which provide the pid and
428 * tid. Consequently there never should be any maps on a thread
429 * with an unknown pid. Just print an error if there are.
431 if (!maps__empty(th
->maps
))
432 pr_err("Discarding thread maps for %d:%d\n",
437 th
->maps
= maps__get(leader
->maps
);
442 pr_err("Failed to join map groups for %d:%d\n", th
->pid_
, th
->tid
);
447 * Front-end cache - TID lookups come in blocks,
448 * so most of the time we dont have to look up
451 static struct thread
*
452 __threads__get_last_match(struct threads
*threads
, struct machine
*machine
,
457 th
= threads
->last_match
;
459 if (th
->tid
== tid
) {
460 machine__update_thread_pid(machine
, th
, pid
);
461 return thread__get(th
);
464 threads
->last_match
= NULL
;
470 static struct thread
*
471 threads__get_last_match(struct threads
*threads
, struct machine
*machine
,
474 struct thread
*th
= NULL
;
476 if (perf_singlethreaded
)
477 th
= __threads__get_last_match(threads
, machine
, pid
, tid
);
483 __threads__set_last_match(struct threads
*threads
, struct thread
*th
)
485 threads
->last_match
= th
;
489 threads__set_last_match(struct threads
*threads
, struct thread
*th
)
491 if (perf_singlethreaded
)
492 __threads__set_last_match(threads
, th
);
496 * Caller must eventually drop thread->refcnt returned with a successful
497 * lookup/new thread inserted.
499 static struct thread
*____machine__findnew_thread(struct machine
*machine
,
500 struct threads
*threads
,
501 pid_t pid
, pid_t tid
,
504 struct rb_node
**p
= &threads
->entries
.rb_root
.rb_node
;
505 struct rb_node
*parent
= NULL
;
507 bool leftmost
= true;
509 th
= threads__get_last_match(threads
, machine
, pid
, tid
);
515 th
= rb_entry(parent
, struct thread
, rb_node
);
517 if (th
->tid
== tid
) {
518 threads__set_last_match(threads
, th
);
519 machine__update_thread_pid(machine
, th
, pid
);
520 return thread__get(th
);
534 th
= thread__new(pid
, tid
);
536 rb_link_node(&th
->rb_node
, parent
, p
);
537 rb_insert_color_cached(&th
->rb_node
, &threads
->entries
, leftmost
);
540 * We have to initialize maps separately after rb tree is updated.
542 * The reason is that we call machine__findnew_thread
543 * within thread__init_maps to find the thread
544 * leader and that would screwed the rb tree.
546 if (thread__init_maps(th
, machine
)) {
547 rb_erase_cached(&th
->rb_node
, &threads
->entries
);
548 RB_CLEAR_NODE(&th
->rb_node
);
553 * It is now in the rbtree, get a ref
556 threads__set_last_match(threads
, th
);
563 struct thread
*__machine__findnew_thread(struct machine
*machine
, pid_t pid
, pid_t tid
)
565 return ____machine__findnew_thread(machine
, machine__threads(machine
, tid
), pid
, tid
, true);
568 struct thread
*machine__findnew_thread(struct machine
*machine
, pid_t pid
,
571 struct threads
*threads
= machine__threads(machine
, tid
);
574 down_write(&threads
->lock
);
575 th
= __machine__findnew_thread(machine
, pid
, tid
);
576 up_write(&threads
->lock
);
580 struct thread
*machine__find_thread(struct machine
*machine
, pid_t pid
,
583 struct threads
*threads
= machine__threads(machine
, tid
);
586 down_read(&threads
->lock
);
587 th
= ____machine__findnew_thread(machine
, threads
, pid
, tid
, false);
588 up_read(&threads
->lock
);
592 struct comm
*machine__thread_exec_comm(struct machine
*machine
,
593 struct thread
*thread
)
595 if (machine
->comm_exec
)
596 return thread__exec_comm(thread
);
598 return thread__comm(thread
);
601 int machine__process_comm_event(struct machine
*machine
, union perf_event
*event
,
602 struct perf_sample
*sample
)
604 struct thread
*thread
= machine__findnew_thread(machine
,
607 bool exec
= event
->header
.misc
& PERF_RECORD_MISC_COMM_EXEC
;
611 machine
->comm_exec
= true;
614 perf_event__fprintf_comm(event
, stdout
);
616 if (thread
== NULL
||
617 __thread__set_comm(thread
, event
->comm
.comm
, sample
->time
, exec
)) {
618 dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n");
627 int machine__process_namespaces_event(struct machine
*machine __maybe_unused
,
628 union perf_event
*event
,
629 struct perf_sample
*sample __maybe_unused
)
631 struct thread
*thread
= machine__findnew_thread(machine
,
632 event
->namespaces
.pid
,
633 event
->namespaces
.tid
);
636 WARN_ONCE(event
->namespaces
.nr_namespaces
> NR_NAMESPACES
,
637 "\nWARNING: kernel seems to support more namespaces than perf"
638 " tool.\nTry updating the perf tool..\n\n");
640 WARN_ONCE(event
->namespaces
.nr_namespaces
< NR_NAMESPACES
,
641 "\nWARNING: perf tool seems to support more namespaces than"
642 " the kernel.\nTry updating the kernel..\n\n");
645 perf_event__fprintf_namespaces(event
, stdout
);
647 if (thread
== NULL
||
648 thread__set_namespaces(thread
, sample
->time
, &event
->namespaces
)) {
649 dump_printf("problem processing PERF_RECORD_NAMESPACES, skipping event.\n");
658 int machine__process_cgroup_event(struct machine
*machine
,
659 union perf_event
*event
,
660 struct perf_sample
*sample __maybe_unused
)
665 perf_event__fprintf_cgroup(event
, stdout
);
667 cgrp
= cgroup__findnew(machine
->env
, event
->cgroup
.id
, event
->cgroup
.path
);
674 int machine__process_lost_event(struct machine
*machine __maybe_unused
,
675 union perf_event
*event
, struct perf_sample
*sample __maybe_unused
)
677 dump_printf(": id:%" PRI_lu64
": lost:%" PRI_lu64
"\n",
678 event
->lost
.id
, event
->lost
.lost
);
682 int machine__process_lost_samples_event(struct machine
*machine __maybe_unused
,
683 union perf_event
*event
, struct perf_sample
*sample
)
685 dump_printf(": id:%" PRIu64
": lost samples :%" PRI_lu64
"\n",
686 sample
->id
, event
->lost_samples
.lost
);
690 static struct dso
*machine__findnew_module_dso(struct machine
*machine
,
692 const char *filename
)
696 down_write(&machine
->dsos
.lock
);
698 dso
= __dsos__find(&machine
->dsos
, m
->name
, true);
700 dso
= __dsos__addnew(&machine
->dsos
, m
->name
);
704 dso__set_module_info(dso
, m
, machine
);
705 dso__set_long_name(dso
, strdup(filename
), true);
706 dso
->kernel
= DSO_TYPE_KERNEL
;
711 up_write(&machine
->dsos
.lock
);
715 int machine__process_aux_event(struct machine
*machine __maybe_unused
,
716 union perf_event
*event
)
719 perf_event__fprintf_aux(event
, stdout
);
723 int machine__process_itrace_start_event(struct machine
*machine __maybe_unused
,
724 union perf_event
*event
)
727 perf_event__fprintf_itrace_start(event
, stdout
);
731 int machine__process_switch_event(struct machine
*machine __maybe_unused
,
732 union perf_event
*event
)
735 perf_event__fprintf_switch(event
, stdout
);
739 static int is_bpf_image(const char *name
)
741 return strncmp(name
, "bpf_trampoline_", sizeof("bpf_trampoline_") - 1) ||
742 strncmp(name
, "bpf_dispatcher_", sizeof("bpf_dispatcher_") - 1);
745 static int machine__process_ksymbol_register(struct machine
*machine
,
746 union perf_event
*event
,
747 struct perf_sample
*sample __maybe_unused
)
750 struct map
*map
= maps__find(&machine
->kmaps
, event
->ksymbol
.addr
);
753 struct dso
*dso
= dso__new(event
->ksymbol
.name
);
756 dso
->kernel
= DSO_TYPE_KERNEL
;
757 map
= map__new2(0, dso
);
765 map
->start
= event
->ksymbol
.addr
;
766 map
->end
= map
->start
+ event
->ksymbol
.len
;
767 maps__insert(&machine
->kmaps
, map
);
768 dso__set_loaded(dso
);
770 if (is_bpf_image(event
->ksymbol
.name
)) {
771 dso
->binary_type
= DSO_BINARY_TYPE__BPF_IMAGE
;
772 dso__set_long_name(dso
, "", false);
776 sym
= symbol__new(map
->map_ip(map
, map
->start
),
778 0, 0, event
->ksymbol
.name
);
781 dso__insert_symbol(map
->dso
, sym
);
785 static int machine__process_ksymbol_unregister(struct machine
*machine
,
786 union perf_event
*event
,
787 struct perf_sample
*sample __maybe_unused
)
791 map
= maps__find(&machine
->kmaps
, event
->ksymbol
.addr
);
793 maps__remove(&machine
->kmaps
, map
);
798 int machine__process_ksymbol(struct machine
*machine __maybe_unused
,
799 union perf_event
*event
,
800 struct perf_sample
*sample
)
803 perf_event__fprintf_ksymbol(event
, stdout
);
805 if (event
->ksymbol
.flags
& PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER
)
806 return machine__process_ksymbol_unregister(machine
, event
,
808 return machine__process_ksymbol_register(machine
, event
, sample
);
811 static struct map
*machine__addnew_module_map(struct machine
*machine
, u64 start
,
812 const char *filename
)
814 struct map
*map
= NULL
;
818 if (kmod_path__parse_name(&m
, filename
))
821 dso
= machine__findnew_module_dso(machine
, &m
, filename
);
825 map
= map__new2(start
, dso
);
829 maps__insert(&machine
->kmaps
, map
);
831 /* Put the map here because maps__insert alread got it */
834 /* put the dso here, corresponding to machine__findnew_module_dso */
840 size_t machines__fprintf_dsos(struct machines
*machines
, FILE *fp
)
843 size_t ret
= __dsos__fprintf(&machines
->host
.dsos
.head
, fp
);
845 for (nd
= rb_first_cached(&machines
->guests
); nd
; nd
= rb_next(nd
)) {
846 struct machine
*pos
= rb_entry(nd
, struct machine
, rb_node
);
847 ret
+= __dsos__fprintf(&pos
->dsos
.head
, fp
);
853 size_t machine__fprintf_dsos_buildid(struct machine
*m
, FILE *fp
,
854 bool (skip
)(struct dso
*dso
, int parm
), int parm
)
856 return __dsos__fprintf_buildid(&m
->dsos
.head
, fp
, skip
, parm
);
859 size_t machines__fprintf_dsos_buildid(struct machines
*machines
, FILE *fp
,
860 bool (skip
)(struct dso
*dso
, int parm
), int parm
)
863 size_t ret
= machine__fprintf_dsos_buildid(&machines
->host
, fp
, skip
, parm
);
865 for (nd
= rb_first_cached(&machines
->guests
); nd
; nd
= rb_next(nd
)) {
866 struct machine
*pos
= rb_entry(nd
, struct machine
, rb_node
);
867 ret
+= machine__fprintf_dsos_buildid(pos
, fp
, skip
, parm
);
872 size_t machine__fprintf_vmlinux_path(struct machine
*machine
, FILE *fp
)
876 struct dso
*kdso
= machine__kernel_dso(machine
);
878 if (kdso
->has_build_id
) {
879 char filename
[PATH_MAX
];
880 if (dso__build_id_filename(kdso
, filename
, sizeof(filename
),
882 printed
+= fprintf(fp
, "[0] %s\n", filename
);
885 for (i
= 0; i
< vmlinux_path__nr_entries
; ++i
)
886 printed
+= fprintf(fp
, "[%d] %s\n",
887 i
+ kdso
->has_build_id
, vmlinux_path
[i
]);
892 size_t machine__fprintf(struct machine
*machine
, FILE *fp
)
898 for (i
= 0; i
< THREADS__TABLE_SIZE
; i
++) {
899 struct threads
*threads
= &machine
->threads
[i
];
901 down_read(&threads
->lock
);
903 ret
= fprintf(fp
, "Threads: %u\n", threads
->nr
);
905 for (nd
= rb_first_cached(&threads
->entries
); nd
;
907 struct thread
*pos
= rb_entry(nd
, struct thread
, rb_node
);
909 ret
+= thread__fprintf(pos
, fp
);
912 up_read(&threads
->lock
);
917 static struct dso
*machine__get_kernel(struct machine
*machine
)
919 const char *vmlinux_name
= machine
->mmap_name
;
922 if (machine__is_host(machine
)) {
923 if (symbol_conf
.vmlinux_name
)
924 vmlinux_name
= symbol_conf
.vmlinux_name
;
926 kernel
= machine__findnew_kernel(machine
, vmlinux_name
,
927 "[kernel]", DSO_TYPE_KERNEL
);
929 if (symbol_conf
.default_guest_vmlinux_name
)
930 vmlinux_name
= symbol_conf
.default_guest_vmlinux_name
;
932 kernel
= machine__findnew_kernel(machine
, vmlinux_name
,
934 DSO_TYPE_GUEST_KERNEL
);
937 if (kernel
!= NULL
&& (!kernel
->has_build_id
))
938 dso__read_running_kernel_build_id(kernel
, machine
);
943 struct process_args
{
947 void machine__get_kallsyms_filename(struct machine
*machine
, char *buf
,
950 if (machine__is_default_guest(machine
))
951 scnprintf(buf
, bufsz
, "%s", symbol_conf
.default_guest_kallsyms
);
953 scnprintf(buf
, bufsz
, "%s/proc/kallsyms", machine
->root_dir
);
956 const char *ref_reloc_sym_names
[] = {"_text", "_stext", NULL
};
958 /* Figure out the start address of kernel map from /proc/kallsyms.
959 * Returns the name of the start symbol in *symbol_name. Pass in NULL as
960 * symbol_name if it's not that important.
962 static int machine__get_running_kernel_start(struct machine
*machine
,
963 const char **symbol_name
,
964 u64
*start
, u64
*end
)
966 char filename
[PATH_MAX
];
971 machine__get_kallsyms_filename(machine
, filename
, PATH_MAX
);
973 if (symbol__restricted_filename(filename
, "/proc/kallsyms"))
976 for (i
= 0; (name
= ref_reloc_sym_names
[i
]) != NULL
; i
++) {
977 err
= kallsyms__get_function_start(filename
, name
, &addr
);
990 err
= kallsyms__get_function_start(filename
, "_etext", &addr
);
997 int machine__create_extra_kernel_map(struct machine
*machine
,
999 struct extra_kernel_map
*xm
)
1004 map
= map__new2(xm
->start
, kernel
);
1009 map
->pgoff
= xm
->pgoff
;
1011 kmap
= map__kmap(map
);
1013 strlcpy(kmap
->name
, xm
->name
, KMAP_NAME_LEN
);
1015 maps__insert(&machine
->kmaps
, map
);
1017 pr_debug2("Added extra kernel map %s %" PRIx64
"-%" PRIx64
"\n",
1018 kmap
->name
, map
->start
, map
->end
);
1025 static u64
find_entry_trampoline(struct dso
*dso
)
1027 /* Duplicates are removed so lookup all aliases */
1028 const char *syms
[] = {
1029 "_entry_trampoline",
1030 "__entry_trampoline_start",
1031 "entry_SYSCALL_64_trampoline",
1033 struct symbol
*sym
= dso__first_symbol(dso
);
1036 for (; sym
; sym
= dso__next_symbol(sym
)) {
1037 if (sym
->binding
!= STB_GLOBAL
)
1039 for (i
= 0; i
< ARRAY_SIZE(syms
); i
++) {
1040 if (!strcmp(sym
->name
, syms
[i
]))
1049 * These values can be used for kernels that do not have symbols for the entry
1050 * trampolines in kallsyms.
1052 #define X86_64_CPU_ENTRY_AREA_PER_CPU 0xfffffe0000000000ULL
1053 #define X86_64_CPU_ENTRY_AREA_SIZE 0x2c000
1054 #define X86_64_ENTRY_TRAMPOLINE 0x6000
1056 /* Map x86_64 PTI entry trampolines */
1057 int machine__map_x86_64_entry_trampolines(struct machine
*machine
,
1060 struct maps
*kmaps
= &machine
->kmaps
;
1061 int nr_cpus_avail
, cpu
;
1067 * In the vmlinux case, pgoff is a virtual address which must now be
1068 * mapped to a vmlinux offset.
1070 maps__for_each_entry(kmaps
, map
) {
1071 struct kmap
*kmap
= __map__kmap(map
);
1072 struct map
*dest_map
;
1074 if (!kmap
|| !is_entry_trampoline(kmap
->name
))
1077 dest_map
= maps__find(kmaps
, map
->pgoff
);
1078 if (dest_map
!= map
)
1079 map
->pgoff
= dest_map
->map_ip(dest_map
, map
->pgoff
);
1082 if (found
|| machine
->trampolines_mapped
)
1085 pgoff
= find_entry_trampoline(kernel
);
1089 nr_cpus_avail
= machine__nr_cpus_avail(machine
);
1091 /* Add a 1 page map for each CPU's entry trampoline */
1092 for (cpu
= 0; cpu
< nr_cpus_avail
; cpu
++) {
1093 u64 va
= X86_64_CPU_ENTRY_AREA_PER_CPU
+
1094 cpu
* X86_64_CPU_ENTRY_AREA_SIZE
+
1095 X86_64_ENTRY_TRAMPOLINE
;
1096 struct extra_kernel_map xm
= {
1098 .end
= va
+ page_size
,
1102 strlcpy(xm
.name
, ENTRY_TRAMPOLINE_NAME
, KMAP_NAME_LEN
);
1104 if (machine__create_extra_kernel_map(machine
, kernel
, &xm
) < 0)
1108 machine
->trampolines_mapped
= nr_cpus_avail
;
1113 int __weak
machine__create_extra_kernel_maps(struct machine
*machine __maybe_unused
,
1114 struct dso
*kernel __maybe_unused
)
1120 __machine__create_kernel_maps(struct machine
*machine
, struct dso
*kernel
)
1122 /* In case of renewal the kernel map, destroy previous one */
1123 machine__destroy_kernel_maps(machine
);
1125 machine
->vmlinux_map
= map__new2(0, kernel
);
1126 if (machine
->vmlinux_map
== NULL
)
1129 machine
->vmlinux_map
->map_ip
= machine
->vmlinux_map
->unmap_ip
= identity__map_ip
;
1130 maps__insert(&machine
->kmaps
, machine
->vmlinux_map
);
1134 void machine__destroy_kernel_maps(struct machine
*machine
)
1137 struct map
*map
= machine__kernel_map(machine
);
1142 kmap
= map__kmap(map
);
1143 maps__remove(&machine
->kmaps
, map
);
1144 if (kmap
&& kmap
->ref_reloc_sym
) {
1145 zfree((char **)&kmap
->ref_reloc_sym
->name
);
1146 zfree(&kmap
->ref_reloc_sym
);
1149 map__zput(machine
->vmlinux_map
);
1152 int machines__create_guest_kernel_maps(struct machines
*machines
)
1155 struct dirent
**namelist
= NULL
;
1157 char path
[PATH_MAX
];
1161 if (symbol_conf
.default_guest_vmlinux_name
||
1162 symbol_conf
.default_guest_modules
||
1163 symbol_conf
.default_guest_kallsyms
) {
1164 machines__create_kernel_maps(machines
, DEFAULT_GUEST_KERNEL_ID
);
1167 if (symbol_conf
.guestmount
) {
1168 items
= scandir(symbol_conf
.guestmount
, &namelist
, NULL
, NULL
);
1171 for (i
= 0; i
< items
; i
++) {
1172 if (!isdigit(namelist
[i
]->d_name
[0])) {
1173 /* Filter out . and .. */
1176 pid
= (pid_t
)strtol(namelist
[i
]->d_name
, &endp
, 10);
1177 if ((*endp
!= '\0') ||
1178 (endp
== namelist
[i
]->d_name
) ||
1179 (errno
== ERANGE
)) {
1180 pr_debug("invalid directory (%s). Skipping.\n",
1181 namelist
[i
]->d_name
);
1184 sprintf(path
, "%s/%s/proc/kallsyms",
1185 symbol_conf
.guestmount
,
1186 namelist
[i
]->d_name
);
1187 ret
= access(path
, R_OK
);
1189 pr_debug("Can't access file %s\n", path
);
1192 machines__create_kernel_maps(machines
, pid
);
1201 void machines__destroy_kernel_maps(struct machines
*machines
)
1203 struct rb_node
*next
= rb_first_cached(&machines
->guests
);
1205 machine__destroy_kernel_maps(&machines
->host
);
1208 struct machine
*pos
= rb_entry(next
, struct machine
, rb_node
);
1210 next
= rb_next(&pos
->rb_node
);
1211 rb_erase_cached(&pos
->rb_node
, &machines
->guests
);
1212 machine__delete(pos
);
1216 int machines__create_kernel_maps(struct machines
*machines
, pid_t pid
)
1218 struct machine
*machine
= machines__findnew(machines
, pid
);
1220 if (machine
== NULL
)
1223 return machine__create_kernel_maps(machine
);
1226 int machine__load_kallsyms(struct machine
*machine
, const char *filename
)
1228 struct map
*map
= machine__kernel_map(machine
);
1229 int ret
= __dso__load_kallsyms(map
->dso
, filename
, map
, true);
1232 dso__set_loaded(map
->dso
);
1234 * Since /proc/kallsyms will have multiple sessions for the
1235 * kernel, with modules between them, fixup the end of all
1238 maps__fixup_end(&machine
->kmaps
);
1244 int machine__load_vmlinux_path(struct machine
*machine
)
1246 struct map
*map
= machine__kernel_map(machine
);
1247 int ret
= dso__load_vmlinux_path(map
->dso
, map
);
1250 dso__set_loaded(map
->dso
);
1255 static char *get_kernel_version(const char *root_dir
)
1257 char version
[PATH_MAX
];
1260 const char *prefix
= "Linux version ";
1262 sprintf(version
, "%s/proc/version", root_dir
);
1263 file
= fopen(version
, "r");
1267 tmp
= fgets(version
, sizeof(version
), file
);
1272 name
= strstr(version
, prefix
);
1275 name
+= strlen(prefix
);
1276 tmp
= strchr(name
, ' ');
1280 return strdup(name
);
1283 static bool is_kmod_dso(struct dso
*dso
)
1285 return dso
->symtab_type
== DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE
||
1286 dso
->symtab_type
== DSO_BINARY_TYPE__GUEST_KMODULE
;
1289 static int maps__set_module_path(struct maps
*maps
, const char *path
, struct kmod_path
*m
)
1292 struct map
*map
= maps__find_by_name(maps
, m
->name
);
1297 long_name
= strdup(path
);
1298 if (long_name
== NULL
)
1301 dso__set_long_name(map
->dso
, long_name
, true);
1302 dso__kernel_module_get_build_id(map
->dso
, "");
1305 * Full name could reveal us kmod compression, so
1306 * we need to update the symtab_type if needed.
1308 if (m
->comp
&& is_kmod_dso(map
->dso
)) {
1309 map
->dso
->symtab_type
++;
1310 map
->dso
->comp
= m
->comp
;
1316 static int maps__set_modules_path_dir(struct maps
*maps
, const char *dir_name
, int depth
)
1318 struct dirent
*dent
;
1319 DIR *dir
= opendir(dir_name
);
1323 pr_debug("%s: cannot open %s dir\n", __func__
, dir_name
);
1327 while ((dent
= readdir(dir
)) != NULL
) {
1328 char path
[PATH_MAX
];
1331 /*sshfs might return bad dent->d_type, so we have to stat*/
1332 snprintf(path
, sizeof(path
), "%s/%s", dir_name
, dent
->d_name
);
1333 if (stat(path
, &st
))
1336 if (S_ISDIR(st
.st_mode
)) {
1337 if (!strcmp(dent
->d_name
, ".") ||
1338 !strcmp(dent
->d_name
, ".."))
1341 /* Do not follow top-level source and build symlinks */
1343 if (!strcmp(dent
->d_name
, "source") ||
1344 !strcmp(dent
->d_name
, "build"))
1348 ret
= maps__set_modules_path_dir(maps
, path
, depth
+ 1);
1354 ret
= kmod_path__parse_name(&m
, dent
->d_name
);
1359 ret
= maps__set_module_path(maps
, path
, &m
);
1373 static int machine__set_modules_path(struct machine
*machine
)
1376 char modules_path
[PATH_MAX
];
1378 version
= get_kernel_version(machine
->root_dir
);
1382 snprintf(modules_path
, sizeof(modules_path
), "%s/lib/modules/%s",
1383 machine
->root_dir
, version
);
1386 return maps__set_modules_path_dir(&machine
->kmaps
, modules_path
, 0);
1388 int __weak
arch__fix_module_text_start(u64
*start __maybe_unused
,
1389 u64
*size __maybe_unused
,
1390 const char *name __maybe_unused
)
1395 static int machine__create_module(void *arg
, const char *name
, u64 start
,
1398 struct machine
*machine
= arg
;
1401 if (arch__fix_module_text_start(&start
, &size
, name
) < 0)
1404 map
= machine__addnew_module_map(machine
, start
, name
);
1407 map
->end
= start
+ size
;
1409 dso__kernel_module_get_build_id(map
->dso
, machine
->root_dir
);
1414 static int machine__create_modules(struct machine
*machine
)
1416 const char *modules
;
1417 char path
[PATH_MAX
];
1419 if (machine__is_default_guest(machine
)) {
1420 modules
= symbol_conf
.default_guest_modules
;
1422 snprintf(path
, PATH_MAX
, "%s/proc/modules", machine
->root_dir
);
1426 if (symbol__restricted_filename(modules
, "/proc/modules"))
1429 if (modules__parse(modules
, machine
, machine__create_module
))
1432 if (!machine__set_modules_path(machine
))
1435 pr_debug("Problems setting modules path maps, continuing anyway...\n");
1440 static void machine__set_kernel_mmap(struct machine
*machine
,
1443 machine
->vmlinux_map
->start
= start
;
1444 machine
->vmlinux_map
->end
= end
;
1446 * Be a bit paranoid here, some perf.data file came with
1447 * a zero sized synthesized MMAP event for the kernel.
1449 if (start
== 0 && end
== 0)
1450 machine
->vmlinux_map
->end
= ~0ULL;
1453 static void machine__update_kernel_mmap(struct machine
*machine
,
1456 struct map
*map
= machine__kernel_map(machine
);
1459 maps__remove(&machine
->kmaps
, map
);
1461 machine__set_kernel_mmap(machine
, start
, end
);
1463 maps__insert(&machine
->kmaps
, map
);
1467 int machine__create_kernel_maps(struct machine
*machine
)
1469 struct dso
*kernel
= machine__get_kernel(machine
);
1470 const char *name
= NULL
;
1472 u64 start
= 0, end
= ~0ULL;
1478 ret
= __machine__create_kernel_maps(machine
, kernel
);
1482 if (symbol_conf
.use_modules
&& machine__create_modules(machine
) < 0) {
1483 if (machine__is_host(machine
))
1484 pr_debug("Problems creating module maps, "
1485 "continuing anyway...\n");
1487 pr_debug("Problems creating module maps for guest %d, "
1488 "continuing anyway...\n", machine
->pid
);
1491 if (!machine__get_running_kernel_start(machine
, &name
, &start
, &end
)) {
1493 map__set_kallsyms_ref_reloc_sym(machine
->vmlinux_map
, name
, start
)) {
1494 machine__destroy_kernel_maps(machine
);
1500 * we have a real start address now, so re-order the kmaps
1501 * assume it's the last in the kmaps
1503 machine__update_kernel_mmap(machine
, start
, end
);
1506 if (machine__create_extra_kernel_maps(machine
, kernel
))
1507 pr_debug("Problems creating extra kernel maps, continuing anyway...\n");
1510 /* update end address of the kernel map using adjacent module address */
1511 map
= map__next(machine__kernel_map(machine
));
1513 machine__set_kernel_mmap(machine
, start
, map
->start
);
1521 static bool machine__uses_kcore(struct machine
*machine
)
1525 list_for_each_entry(dso
, &machine
->dsos
.head
, node
) {
1526 if (dso__is_kcore(dso
))
1533 static bool perf_event__is_extra_kernel_mmap(struct machine
*machine
,
1534 union perf_event
*event
)
1536 return machine__is(machine
, "x86_64") &&
1537 is_entry_trampoline(event
->mmap
.filename
);
1540 static int machine__process_extra_kernel_map(struct machine
*machine
,
1541 union perf_event
*event
)
1543 struct dso
*kernel
= machine__kernel_dso(machine
);
1544 struct extra_kernel_map xm
= {
1545 .start
= event
->mmap
.start
,
1546 .end
= event
->mmap
.start
+ event
->mmap
.len
,
1547 .pgoff
= event
->mmap
.pgoff
,
1553 strlcpy(xm
.name
, event
->mmap
.filename
, KMAP_NAME_LEN
);
1555 return machine__create_extra_kernel_map(machine
, kernel
, &xm
);
1558 static int machine__process_kernel_mmap_event(struct machine
*machine
,
1559 union perf_event
*event
)
1562 enum dso_kernel_type kernel_type
;
1563 bool is_kernel_mmap
;
1565 /* If we have maps from kcore then we do not need or want any others */
1566 if (machine__uses_kcore(machine
))
1569 if (machine__is_host(machine
))
1570 kernel_type
= DSO_TYPE_KERNEL
;
1572 kernel_type
= DSO_TYPE_GUEST_KERNEL
;
1574 is_kernel_mmap
= memcmp(event
->mmap
.filename
,
1576 strlen(machine
->mmap_name
) - 1) == 0;
1577 if (event
->mmap
.filename
[0] == '/' ||
1578 (!is_kernel_mmap
&& event
->mmap
.filename
[0] == '[')) {
1579 map
= machine__addnew_module_map(machine
, event
->mmap
.start
,
1580 event
->mmap
.filename
);
1584 map
->end
= map
->start
+ event
->mmap
.len
;
1585 } else if (is_kernel_mmap
) {
1586 const char *symbol_name
= (event
->mmap
.filename
+
1587 strlen(machine
->mmap_name
));
1589 * Should be there already, from the build-id table in
1592 struct dso
*kernel
= NULL
;
1595 down_read(&machine
->dsos
.lock
);
1597 list_for_each_entry(dso
, &machine
->dsos
.head
, node
) {
1600 * The cpumode passed to is_kernel_module is not the
1601 * cpumode of *this* event. If we insist on passing
1602 * correct cpumode to is_kernel_module, we should
1603 * record the cpumode when we adding this dso to the
1606 * However we don't really need passing correct
1607 * cpumode. We know the correct cpumode must be kernel
1608 * mode (if not, we should not link it onto kernel_dsos
1611 * Therefore, we pass PERF_RECORD_MISC_CPUMODE_UNKNOWN.
1612 * is_kernel_module() treats it as a kernel cpumode.
1616 is_kernel_module(dso
->long_name
,
1617 PERF_RECORD_MISC_CPUMODE_UNKNOWN
))
1625 up_read(&machine
->dsos
.lock
);
1628 kernel
= machine__findnew_dso(machine
, machine
->mmap_name
);
1632 kernel
->kernel
= kernel_type
;
1633 if (__machine__create_kernel_maps(machine
, kernel
) < 0) {
1638 if (strstr(kernel
->long_name
, "vmlinux"))
1639 dso__set_short_name(kernel
, "[kernel.vmlinux]", false);
1641 machine__update_kernel_mmap(machine
, event
->mmap
.start
,
1642 event
->mmap
.start
+ event
->mmap
.len
);
1645 * Avoid using a zero address (kptr_restrict) for the ref reloc
1646 * symbol. Effectively having zero here means that at record
1647 * time /proc/sys/kernel/kptr_restrict was non zero.
1649 if (event
->mmap
.pgoff
!= 0) {
1650 map__set_kallsyms_ref_reloc_sym(machine
->vmlinux_map
,
1655 if (machine__is_default_guest(machine
)) {
1657 * preload dso of guest kernel and modules
1659 dso__load(kernel
, machine__kernel_map(machine
));
1661 } else if (perf_event__is_extra_kernel_mmap(machine
, event
)) {
1662 return machine__process_extra_kernel_map(machine
, event
);
1669 int machine__process_mmap2_event(struct machine
*machine
,
1670 union perf_event
*event
,
1671 struct perf_sample
*sample
)
1673 struct thread
*thread
;
1675 struct dso_id dso_id
= {
1676 .maj
= event
->mmap2
.maj
,
1677 .min
= event
->mmap2
.min
,
1678 .ino
= event
->mmap2
.ino
,
1679 .ino_generation
= event
->mmap2
.ino_generation
,
1684 perf_event__fprintf_mmap2(event
, stdout
);
1686 if (sample
->cpumode
== PERF_RECORD_MISC_GUEST_KERNEL
||
1687 sample
->cpumode
== PERF_RECORD_MISC_KERNEL
) {
1688 ret
= machine__process_kernel_mmap_event(machine
, event
);
1694 thread
= machine__findnew_thread(machine
, event
->mmap2
.pid
,
1699 map
= map__new(machine
, event
->mmap2
.start
,
1700 event
->mmap2
.len
, event
->mmap2
.pgoff
,
1701 &dso_id
, event
->mmap2
.prot
,
1703 event
->mmap2
.filename
, thread
);
1706 goto out_problem_map
;
1708 ret
= thread__insert_map(thread
, map
);
1710 goto out_problem_insert
;
1712 thread__put(thread
);
1719 thread__put(thread
);
1721 dump_printf("problem processing PERF_RECORD_MMAP2, skipping event.\n");
1725 int machine__process_mmap_event(struct machine
*machine
, union perf_event
*event
,
1726 struct perf_sample
*sample
)
1728 struct thread
*thread
;
1734 perf_event__fprintf_mmap(event
, stdout
);
1736 if (sample
->cpumode
== PERF_RECORD_MISC_GUEST_KERNEL
||
1737 sample
->cpumode
== PERF_RECORD_MISC_KERNEL
) {
1738 ret
= machine__process_kernel_mmap_event(machine
, event
);
1744 thread
= machine__findnew_thread(machine
, event
->mmap
.pid
,
1749 if (!(event
->header
.misc
& PERF_RECORD_MISC_MMAP_DATA
))
1752 map
= map__new(machine
, event
->mmap
.start
,
1753 event
->mmap
.len
, event
->mmap
.pgoff
,
1754 NULL
, prot
, 0, event
->mmap
.filename
, thread
);
1757 goto out_problem_map
;
1759 ret
= thread__insert_map(thread
, map
);
1761 goto out_problem_insert
;
1763 thread__put(thread
);
1770 thread__put(thread
);
1772 dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n");
1776 static void __machine__remove_thread(struct machine
*machine
, struct thread
*th
, bool lock
)
1778 struct threads
*threads
= machine__threads(machine
, th
->tid
);
1780 if (threads
->last_match
== th
)
1781 threads__set_last_match(threads
, NULL
);
1784 down_write(&threads
->lock
);
1786 BUG_ON(refcount_read(&th
->refcnt
) == 0);
1788 rb_erase_cached(&th
->rb_node
, &threads
->entries
);
1789 RB_CLEAR_NODE(&th
->rb_node
);
1792 * Move it first to the dead_threads list, then drop the reference,
1793 * if this is the last reference, then the thread__delete destructor
1794 * will be called and we will remove it from the dead_threads list.
1796 list_add_tail(&th
->node
, &threads
->dead
);
1799 * We need to do the put here because if this is the last refcount,
1800 * then we will be touching the threads->dead head when removing the
1806 up_write(&threads
->lock
);
1809 void machine__remove_thread(struct machine
*machine
, struct thread
*th
)
1811 return __machine__remove_thread(machine
, th
, true);
1814 int machine__process_fork_event(struct machine
*machine
, union perf_event
*event
,
1815 struct perf_sample
*sample
)
1817 struct thread
*thread
= machine__find_thread(machine
,
1820 struct thread
*parent
= machine__findnew_thread(machine
,
1823 bool do_maps_clone
= true;
1827 perf_event__fprintf_task(event
, stdout
);
1830 * There may be an existing thread that is not actually the parent,
1831 * either because we are processing events out of order, or because the
1832 * (fork) event that would have removed the thread was lost. Assume the
1833 * latter case and continue on as best we can.
1835 if (parent
->pid_
!= (pid_t
)event
->fork
.ppid
) {
1836 dump_printf("removing erroneous parent thread %d/%d\n",
1837 parent
->pid_
, parent
->tid
);
1838 machine__remove_thread(machine
, parent
);
1839 thread__put(parent
);
1840 parent
= machine__findnew_thread(machine
, event
->fork
.ppid
,
1844 /* if a thread currently exists for the thread id remove it */
1845 if (thread
!= NULL
) {
1846 machine__remove_thread(machine
, thread
);
1847 thread__put(thread
);
1850 thread
= machine__findnew_thread(machine
, event
->fork
.pid
,
1853 * When synthesizing FORK events, we are trying to create thread
1854 * objects for the already running tasks on the machine.
1856 * Normally, for a kernel FORK event, we want to clone the parent's
1857 * maps because that is what the kernel just did.
1859 * But when synthesizing, this should not be done. If we do, we end up
1860 * with overlapping maps as we process the sythesized MMAP2 events that
1861 * get delivered shortly thereafter.
1863 * Use the FORK event misc flags in an internal way to signal this
1864 * situation, so we can elide the map clone when appropriate.
1866 if (event
->fork
.header
.misc
& PERF_RECORD_MISC_FORK_EXEC
)
1867 do_maps_clone
= false;
1869 if (thread
== NULL
|| parent
== NULL
||
1870 thread__fork(thread
, parent
, sample
->time
, do_maps_clone
) < 0) {
1871 dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
1874 thread__put(thread
);
1875 thread__put(parent
);
1880 int machine__process_exit_event(struct machine
*machine
, union perf_event
*event
,
1881 struct perf_sample
*sample __maybe_unused
)
1883 struct thread
*thread
= machine__find_thread(machine
,
1888 perf_event__fprintf_task(event
, stdout
);
1890 if (thread
!= NULL
) {
1891 thread__exited(thread
);
1892 thread__put(thread
);
1898 int machine__process_event(struct machine
*machine
, union perf_event
*event
,
1899 struct perf_sample
*sample
)
1903 switch (event
->header
.type
) {
1904 case PERF_RECORD_COMM
:
1905 ret
= machine__process_comm_event(machine
, event
, sample
); break;
1906 case PERF_RECORD_MMAP
:
1907 ret
= machine__process_mmap_event(machine
, event
, sample
); break;
1908 case PERF_RECORD_NAMESPACES
:
1909 ret
= machine__process_namespaces_event(machine
, event
, sample
); break;
1910 case PERF_RECORD_CGROUP
:
1911 ret
= machine__process_cgroup_event(machine
, event
, sample
); break;
1912 case PERF_RECORD_MMAP2
:
1913 ret
= machine__process_mmap2_event(machine
, event
, sample
); break;
1914 case PERF_RECORD_FORK
:
1915 ret
= machine__process_fork_event(machine
, event
, sample
); break;
1916 case PERF_RECORD_EXIT
:
1917 ret
= machine__process_exit_event(machine
, event
, sample
); break;
1918 case PERF_RECORD_LOST
:
1919 ret
= machine__process_lost_event(machine
, event
, sample
); break;
1920 case PERF_RECORD_AUX
:
1921 ret
= machine__process_aux_event(machine
, event
); break;
1922 case PERF_RECORD_ITRACE_START
:
1923 ret
= machine__process_itrace_start_event(machine
, event
); break;
1924 case PERF_RECORD_LOST_SAMPLES
:
1925 ret
= machine__process_lost_samples_event(machine
, event
, sample
); break;
1926 case PERF_RECORD_SWITCH
:
1927 case PERF_RECORD_SWITCH_CPU_WIDE
:
1928 ret
= machine__process_switch_event(machine
, event
); break;
1929 case PERF_RECORD_KSYMBOL
:
1930 ret
= machine__process_ksymbol(machine
, event
, sample
); break;
1931 case PERF_RECORD_BPF_EVENT
:
1932 ret
= machine__process_bpf(machine
, event
, sample
); break;
1941 static bool symbol__match_regex(struct symbol
*sym
, regex_t
*regex
)
1943 if (!regexec(regex
, sym
->name
, 0, NULL
, 0))
1948 static void ip__resolve_ams(struct thread
*thread
,
1949 struct addr_map_symbol
*ams
,
1952 struct addr_location al
;
1954 memset(&al
, 0, sizeof(al
));
1956 * We cannot use the header.misc hint to determine whether a
1957 * branch stack address is user, kernel, guest, hypervisor.
1958 * Branches may straddle the kernel/user/hypervisor boundaries.
1959 * Thus, we have to try consecutively until we find a match
1960 * or else, the symbol is unknown
1962 thread__find_cpumode_addr_location(thread
, ip
, &al
);
1965 ams
->al_addr
= al
.addr
;
1966 ams
->ms
.maps
= al
.maps
;
1967 ams
->ms
.sym
= al
.sym
;
1968 ams
->ms
.map
= al
.map
;
1972 static void ip__resolve_data(struct thread
*thread
,
1973 u8 m
, struct addr_map_symbol
*ams
,
1974 u64 addr
, u64 phys_addr
)
1976 struct addr_location al
;
1978 memset(&al
, 0, sizeof(al
));
1980 thread__find_symbol(thread
, m
, addr
, &al
);
1983 ams
->al_addr
= al
.addr
;
1984 ams
->ms
.maps
= al
.maps
;
1985 ams
->ms
.sym
= al
.sym
;
1986 ams
->ms
.map
= al
.map
;
1987 ams
->phys_addr
= phys_addr
;
1990 struct mem_info
*sample__resolve_mem(struct perf_sample
*sample
,
1991 struct addr_location
*al
)
1993 struct mem_info
*mi
= mem_info__new();
1998 ip__resolve_ams(al
->thread
, &mi
->iaddr
, sample
->ip
);
1999 ip__resolve_data(al
->thread
, al
->cpumode
, &mi
->daddr
,
2000 sample
->addr
, sample
->phys_addr
);
2001 mi
->data_src
.val
= sample
->data_src
;
2006 static char *callchain_srcline(struct map_symbol
*ms
, u64 ip
)
2008 struct map
*map
= ms
->map
;
2009 char *srcline
= NULL
;
2011 if (!map
|| callchain_param
.key
== CCKEY_FUNCTION
)
2014 srcline
= srcline__tree_find(&map
->dso
->srclines
, ip
);
2016 bool show_sym
= false;
2017 bool show_addr
= callchain_param
.key
== CCKEY_ADDRESS
;
2019 srcline
= get_srcline(map
->dso
, map__rip_2objdump(map
, ip
),
2020 ms
->sym
, show_sym
, show_addr
, ip
);
2021 srcline__tree_insert(&map
->dso
->srclines
, ip
, srcline
);
2032 static int add_callchain_ip(struct thread
*thread
,
2033 struct callchain_cursor
*cursor
,
2034 struct symbol
**parent
,
2035 struct addr_location
*root_al
,
2039 struct branch_flags
*flags
,
2040 struct iterations
*iter
,
2043 struct map_symbol ms
;
2044 struct addr_location al
;
2045 int nr_loop_iter
= 0;
2046 u64 iter_cycles
= 0;
2047 const char *srcline
= NULL
;
2052 thread__find_cpumode_addr_location(thread
, ip
, &al
);
2054 if (ip
>= PERF_CONTEXT_MAX
) {
2056 case PERF_CONTEXT_HV
:
2057 *cpumode
= PERF_RECORD_MISC_HYPERVISOR
;
2059 case PERF_CONTEXT_KERNEL
:
2060 *cpumode
= PERF_RECORD_MISC_KERNEL
;
2062 case PERF_CONTEXT_USER
:
2063 *cpumode
= PERF_RECORD_MISC_USER
;
2066 pr_debug("invalid callchain context: "
2067 "%"PRId64
"\n", (s64
) ip
);
2069 * It seems the callchain is corrupted.
2072 callchain_cursor_reset(cursor
);
2077 thread__find_symbol(thread
, *cpumode
, ip
, &al
);
2080 if (al
.sym
!= NULL
) {
2081 if (perf_hpp_list
.parent
&& !*parent
&&
2082 symbol__match_regex(al
.sym
, &parent_regex
))
2084 else if (have_ignore_callees
&& root_al
&&
2085 symbol__match_regex(al
.sym
, &ignore_callees_regex
)) {
2086 /* Treat this symbol as the root,
2087 forgetting its callees. */
2089 callchain_cursor_reset(cursor
);
2093 if (symbol_conf
.hide_unresolved
&& al
.sym
== NULL
)
2097 nr_loop_iter
= iter
->nr_loop_iter
;
2098 iter_cycles
= iter
->cycles
;
2104 srcline
= callchain_srcline(&ms
, al
.addr
);
2105 return callchain_cursor_append(cursor
, ip
, &ms
,
2106 branch
, flags
, nr_loop_iter
,
2107 iter_cycles
, branch_from
, srcline
);
2110 struct branch_info
*sample__resolve_bstack(struct perf_sample
*sample
,
2111 struct addr_location
*al
)
2114 const struct branch_stack
*bs
= sample
->branch_stack
;
2115 struct branch_entry
*entries
= perf_sample__branch_entries(sample
);
2116 struct branch_info
*bi
= calloc(bs
->nr
, sizeof(struct branch_info
));
2121 for (i
= 0; i
< bs
->nr
; i
++) {
2122 ip__resolve_ams(al
->thread
, &bi
[i
].to
, entries
[i
].to
);
2123 ip__resolve_ams(al
->thread
, &bi
[i
].from
, entries
[i
].from
);
2124 bi
[i
].flags
= entries
[i
].flags
;
2129 static void save_iterations(struct iterations
*iter
,
2130 struct branch_entry
*be
, int nr
)
2134 iter
->nr_loop_iter
++;
2137 for (i
= 0; i
< nr
; i
++)
2138 iter
->cycles
+= be
[i
].flags
.cycles
;
2143 #define NO_ENTRY 0xff
2145 #define PERF_MAX_BRANCH_DEPTH 127
2148 static int remove_loops(struct branch_entry
*l
, int nr
,
2149 struct iterations
*iter
)
2152 unsigned char chash
[CHASHSZ
];
2154 memset(chash
, NO_ENTRY
, sizeof(chash
));
2156 BUG_ON(PERF_MAX_BRANCH_DEPTH
> 255);
2158 for (i
= 0; i
< nr
; i
++) {
2159 int h
= hash_64(l
[i
].from
, CHASHBITS
) % CHASHSZ
;
2161 /* no collision handling for now */
2162 if (chash
[h
] == NO_ENTRY
) {
2164 } else if (l
[chash
[h
]].from
== l
[i
].from
) {
2165 bool is_loop
= true;
2166 /* check if it is a real loop */
2168 for (j
= chash
[h
]; j
< i
&& i
+ off
< nr
; j
++, off
++)
2169 if (l
[j
].from
!= l
[i
+ off
].from
) {
2176 save_iterations(iter
+ i
+ off
,
2179 memmove(iter
+ i
, iter
+ i
+ off
,
2182 memmove(l
+ i
, l
+ i
+ off
,
2193 static int lbr_callchain_add_kernel_ip(struct thread
*thread
,
2194 struct callchain_cursor
*cursor
,
2195 struct perf_sample
*sample
,
2196 struct symbol
**parent
,
2197 struct addr_location
*root_al
,
2199 bool callee
, int end
)
2201 struct ip_callchain
*chain
= sample
->callchain
;
2202 u8 cpumode
= PERF_RECORD_MISC_USER
;
2206 for (i
= 0; i
< end
+ 1; i
++) {
2207 err
= add_callchain_ip(thread
, cursor
, parent
,
2208 root_al
, &cpumode
, chain
->ips
[i
],
2209 false, NULL
, NULL
, branch_from
);
2216 for (i
= end
; i
>= 0; i
--) {
2217 err
= add_callchain_ip(thread
, cursor
, parent
,
2218 root_al
, &cpumode
, chain
->ips
[i
],
2219 false, NULL
, NULL
, branch_from
);
2227 static void save_lbr_cursor_node(struct thread
*thread
,
2228 struct callchain_cursor
*cursor
,
2231 struct lbr_stitch
*lbr_stitch
= thread
->lbr_stitch
;
2236 if (cursor
->pos
== cursor
->nr
) {
2237 lbr_stitch
->prev_lbr_cursor
[idx
].valid
= false;
2242 cursor
->curr
= cursor
->first
;
2244 cursor
->curr
= cursor
->curr
->next
;
2245 memcpy(&lbr_stitch
->prev_lbr_cursor
[idx
], cursor
->curr
,
2246 sizeof(struct callchain_cursor_node
));
2248 lbr_stitch
->prev_lbr_cursor
[idx
].valid
= true;
2252 static int lbr_callchain_add_lbr_ip(struct thread
*thread
,
2253 struct callchain_cursor
*cursor
,
2254 struct perf_sample
*sample
,
2255 struct symbol
**parent
,
2256 struct addr_location
*root_al
,
2260 struct branch_stack
*lbr_stack
= sample
->branch_stack
;
2261 struct branch_entry
*entries
= perf_sample__branch_entries(sample
);
2262 u8 cpumode
= PERF_RECORD_MISC_USER
;
2263 int lbr_nr
= lbr_stack
->nr
;
2264 struct branch_flags
*flags
;
2269 * The curr and pos are not used in writing session. They are cleared
2270 * in callchain_cursor_commit() when the writing session is closed.
2271 * Using curr and pos to track the current cursor node.
2273 if (thread
->lbr_stitch
) {
2274 cursor
->curr
= NULL
;
2275 cursor
->pos
= cursor
->nr
;
2277 cursor
->curr
= cursor
->first
;
2278 for (i
= 0; i
< (int)(cursor
->nr
- 1); i
++)
2279 cursor
->curr
= cursor
->curr
->next
;
2284 /* Add LBR ip from first entries.to */
2286 flags
= &entries
[0].flags
;
2287 *branch_from
= entries
[0].from
;
2288 err
= add_callchain_ip(thread
, cursor
, parent
,
2289 root_al
, &cpumode
, ip
,
2296 * The number of cursor node increases.
2297 * Move the current cursor node.
2298 * But does not need to save current cursor node for entry 0.
2299 * It's impossible to stitch the whole LBRs of previous sample.
2301 if (thread
->lbr_stitch
&& (cursor
->pos
!= cursor
->nr
)) {
2303 cursor
->curr
= cursor
->first
;
2305 cursor
->curr
= cursor
->curr
->next
;
2309 /* Add LBR ip from entries.from one by one. */
2310 for (i
= 0; i
< lbr_nr
; i
++) {
2311 ip
= entries
[i
].from
;
2312 flags
= &entries
[i
].flags
;
2313 err
= add_callchain_ip(thread
, cursor
, parent
,
2314 root_al
, &cpumode
, ip
,
2319 save_lbr_cursor_node(thread
, cursor
, i
);
2324 /* Add LBR ip from entries.from one by one. */
2325 for (i
= lbr_nr
- 1; i
>= 0; i
--) {
2326 ip
= entries
[i
].from
;
2327 flags
= &entries
[i
].flags
;
2328 err
= add_callchain_ip(thread
, cursor
, parent
,
2329 root_al
, &cpumode
, ip
,
2334 save_lbr_cursor_node(thread
, cursor
, i
);
2337 /* Add LBR ip from first entries.to */
2339 flags
= &entries
[0].flags
;
2340 *branch_from
= entries
[0].from
;
2341 err
= add_callchain_ip(thread
, cursor
, parent
,
2342 root_al
, &cpumode
, ip
,
2351 static int lbr_callchain_add_stitched_lbr_ip(struct thread
*thread
,
2352 struct callchain_cursor
*cursor
)
2354 struct lbr_stitch
*lbr_stitch
= thread
->lbr_stitch
;
2355 struct callchain_cursor_node
*cnode
;
2356 struct stitch_list
*stitch_node
;
2359 list_for_each_entry(stitch_node
, &lbr_stitch
->lists
, node
) {
2360 cnode
= &stitch_node
->cursor
;
2362 err
= callchain_cursor_append(cursor
, cnode
->ip
,
2365 &cnode
->branch_flags
,
2366 cnode
->nr_loop_iter
,
2376 static struct stitch_list
*get_stitch_node(struct thread
*thread
)
2378 struct lbr_stitch
*lbr_stitch
= thread
->lbr_stitch
;
2379 struct stitch_list
*stitch_node
;
2381 if (!list_empty(&lbr_stitch
->free_lists
)) {
2382 stitch_node
= list_first_entry(&lbr_stitch
->free_lists
,
2383 struct stitch_list
, node
);
2384 list_del(&stitch_node
->node
);
2389 return malloc(sizeof(struct stitch_list
));
2392 static bool has_stitched_lbr(struct thread
*thread
,
2393 struct perf_sample
*cur
,
2394 struct perf_sample
*prev
,
2395 unsigned int max_lbr
,
2398 struct branch_stack
*cur_stack
= cur
->branch_stack
;
2399 struct branch_entry
*cur_entries
= perf_sample__branch_entries(cur
);
2400 struct branch_stack
*prev_stack
= prev
->branch_stack
;
2401 struct branch_entry
*prev_entries
= perf_sample__branch_entries(prev
);
2402 struct lbr_stitch
*lbr_stitch
= thread
->lbr_stitch
;
2403 int i
, j
, nr_identical_branches
= 0;
2404 struct stitch_list
*stitch_node
;
2405 u64 cur_base
, distance
;
2407 if (!cur_stack
|| !prev_stack
)
2410 /* Find the physical index of the base-of-stack for current sample. */
2411 cur_base
= max_lbr
- cur_stack
->nr
+ cur_stack
->hw_idx
+ 1;
2413 distance
= (prev_stack
->hw_idx
> cur_base
) ? (prev_stack
->hw_idx
- cur_base
) :
2414 (max_lbr
+ prev_stack
->hw_idx
- cur_base
);
2415 /* Previous sample has shorter stack. Nothing can be stitched. */
2416 if (distance
+ 1 > prev_stack
->nr
)
2420 * Check if there are identical LBRs between two samples.
2421 * Identicall LBRs must have same from, to and flags values. Also,
2422 * they have to be saved in the same LBR registers (same physical
2425 * Starts from the base-of-stack of current sample.
2427 for (i
= distance
, j
= cur_stack
->nr
- 1; (i
>= 0) && (j
>= 0); i
--, j
--) {
2428 if ((prev_entries
[i
].from
!= cur_entries
[j
].from
) ||
2429 (prev_entries
[i
].to
!= cur_entries
[j
].to
) ||
2430 (prev_entries
[i
].flags
.value
!= cur_entries
[j
].flags
.value
))
2432 nr_identical_branches
++;
2435 if (!nr_identical_branches
)
2439 * Save the LBRs between the base-of-stack of previous sample
2440 * and the base-of-stack of current sample into lbr_stitch->lists.
2441 * These LBRs will be stitched later.
2443 for (i
= prev_stack
->nr
- 1; i
> (int)distance
; i
--) {
2445 if (!lbr_stitch
->prev_lbr_cursor
[i
].valid
)
2448 stitch_node
= get_stitch_node(thread
);
2452 memcpy(&stitch_node
->cursor
, &lbr_stitch
->prev_lbr_cursor
[i
],
2453 sizeof(struct callchain_cursor_node
));
2456 list_add(&stitch_node
->node
, &lbr_stitch
->lists
);
2458 list_add_tail(&stitch_node
->node
, &lbr_stitch
->lists
);
2464 static bool alloc_lbr_stitch(struct thread
*thread
, unsigned int max_lbr
)
2466 if (thread
->lbr_stitch
)
2469 thread
->lbr_stitch
= zalloc(sizeof(*thread
->lbr_stitch
));
2470 if (!thread
->lbr_stitch
)
2473 thread
->lbr_stitch
->prev_lbr_cursor
= calloc(max_lbr
+ 1, sizeof(struct callchain_cursor_node
));
2474 if (!thread
->lbr_stitch
->prev_lbr_cursor
)
2475 goto free_lbr_stitch
;
2477 INIT_LIST_HEAD(&thread
->lbr_stitch
->lists
);
2478 INIT_LIST_HEAD(&thread
->lbr_stitch
->free_lists
);
2483 zfree(&thread
->lbr_stitch
);
2485 pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n");
2486 thread
->lbr_stitch_enable
= false;
2491 * Recolve LBR callstack chain sample
2493 * 1 on success get LBR callchain information
2494 * 0 no available LBR callchain information, should try fp
2495 * negative error code on other errors.
2497 static int resolve_lbr_callchain_sample(struct thread
*thread
,
2498 struct callchain_cursor
*cursor
,
2499 struct perf_sample
*sample
,
2500 struct symbol
**parent
,
2501 struct addr_location
*root_al
,
2503 unsigned int max_lbr
)
2505 bool callee
= (callchain_param
.order
== ORDER_CALLEE
);
2506 struct ip_callchain
*chain
= sample
->callchain
;
2507 int chain_nr
= min(max_stack
, (int)chain
->nr
), i
;
2508 struct lbr_stitch
*lbr_stitch
;
2509 bool stitched_lbr
= false;
2510 u64 branch_from
= 0;
2513 for (i
= 0; i
< chain_nr
; i
++) {
2514 if (chain
->ips
[i
] == PERF_CONTEXT_USER
)
2518 /* LBR only affects the user callchain */
2522 if (thread
->lbr_stitch_enable
&& !sample
->no_hw_idx
&&
2523 (max_lbr
> 0) && alloc_lbr_stitch(thread
, max_lbr
)) {
2524 lbr_stitch
= thread
->lbr_stitch
;
2526 stitched_lbr
= has_stitched_lbr(thread
, sample
,
2527 &lbr_stitch
->prev_sample
,
2530 if (!stitched_lbr
&& !list_empty(&lbr_stitch
->lists
)) {
2531 list_replace_init(&lbr_stitch
->lists
,
2532 &lbr_stitch
->free_lists
);
2534 memcpy(&lbr_stitch
->prev_sample
, sample
, sizeof(*sample
));
2539 err
= lbr_callchain_add_kernel_ip(thread
, cursor
, sample
,
2540 parent
, root_al
, branch_from
,
2545 err
= lbr_callchain_add_lbr_ip(thread
, cursor
, sample
, parent
,
2546 root_al
, &branch_from
, true);
2551 err
= lbr_callchain_add_stitched_lbr_ip(thread
, cursor
);
2558 err
= lbr_callchain_add_stitched_lbr_ip(thread
, cursor
);
2562 err
= lbr_callchain_add_lbr_ip(thread
, cursor
, sample
, parent
,
2563 root_al
, &branch_from
, false);
2568 err
= lbr_callchain_add_kernel_ip(thread
, cursor
, sample
,
2569 parent
, root_al
, branch_from
,
2577 return (err
< 0) ? err
: 0;
2580 static int find_prev_cpumode(struct ip_callchain
*chain
, struct thread
*thread
,
2581 struct callchain_cursor
*cursor
,
2582 struct symbol
**parent
,
2583 struct addr_location
*root_al
,
2584 u8
*cpumode
, int ent
)
2588 while (--ent
>= 0) {
2589 u64 ip
= chain
->ips
[ent
];
2591 if (ip
>= PERF_CONTEXT_MAX
) {
2592 err
= add_callchain_ip(thread
, cursor
, parent
,
2593 root_al
, cpumode
, ip
,
2594 false, NULL
, NULL
, 0);
2601 static int thread__resolve_callchain_sample(struct thread
*thread
,
2602 struct callchain_cursor
*cursor
,
2603 struct evsel
*evsel
,
2604 struct perf_sample
*sample
,
2605 struct symbol
**parent
,
2606 struct addr_location
*root_al
,
2609 struct branch_stack
*branch
= sample
->branch_stack
;
2610 struct branch_entry
*entries
= perf_sample__branch_entries(sample
);
2611 struct ip_callchain
*chain
= sample
->callchain
;
2613 u8 cpumode
= PERF_RECORD_MISC_USER
;
2614 int i
, j
, err
, nr_entries
;
2619 chain_nr
= chain
->nr
;
2621 if (evsel__has_branch_callstack(evsel
)) {
2622 struct perf_env
*env
= evsel__env(evsel
);
2624 err
= resolve_lbr_callchain_sample(thread
, cursor
, sample
, parent
,
2626 !env
? 0 : env
->max_branches
);
2628 return (err
< 0) ? err
: 0;
2632 * Based on DWARF debug information, some architectures skip
2633 * a callchain entry saved by the kernel.
2635 skip_idx
= arch_skip_callchain_idx(thread
, chain
);
2638 * Add branches to call stack for easier browsing. This gives
2639 * more context for a sample than just the callers.
2641 * This uses individual histograms of paths compared to the
2642 * aggregated histograms the normal LBR mode uses.
2644 * Limitations for now:
2645 * - No extra filters
2646 * - No annotations (should annotate somehow)
2649 if (branch
&& callchain_param
.branch_callstack
) {
2650 int nr
= min(max_stack
, (int)branch
->nr
);
2651 struct branch_entry be
[nr
];
2652 struct iterations iter
[nr
];
2654 if (branch
->nr
> PERF_MAX_BRANCH_DEPTH
) {
2655 pr_warning("corrupted branch chain. skipping...\n");
2659 for (i
= 0; i
< nr
; i
++) {
2660 if (callchain_param
.order
== ORDER_CALLEE
) {
2667 * Check for overlap into the callchain.
2668 * The return address is one off compared to
2669 * the branch entry. To adjust for this
2670 * assume the calling instruction is not longer
2673 if (i
== skip_idx
||
2674 chain
->ips
[first_call
] >= PERF_CONTEXT_MAX
)
2676 else if (be
[i
].from
< chain
->ips
[first_call
] &&
2677 be
[i
].from
>= chain
->ips
[first_call
] - 8)
2680 be
[i
] = entries
[branch
->nr
- i
- 1];
2683 memset(iter
, 0, sizeof(struct iterations
) * nr
);
2684 nr
= remove_loops(be
, nr
, iter
);
2686 for (i
= 0; i
< nr
; i
++) {
2687 err
= add_callchain_ip(thread
, cursor
, parent
,
2694 err
= add_callchain_ip(thread
, cursor
, parent
, root_al
,
2711 if (chain
&& callchain_param
.order
!= ORDER_CALLEE
) {
2712 err
= find_prev_cpumode(chain
, thread
, cursor
, parent
, root_al
,
2713 &cpumode
, chain
->nr
- first_call
);
2715 return (err
< 0) ? err
: 0;
2717 for (i
= first_call
, nr_entries
= 0;
2718 i
< chain_nr
&& nr_entries
< max_stack
; i
++) {
2721 if (callchain_param
.order
== ORDER_CALLEE
)
2724 j
= chain
->nr
- i
- 1;
2726 #ifdef HAVE_SKIP_CALLCHAIN_IDX
2731 if (ip
< PERF_CONTEXT_MAX
)
2733 else if (callchain_param
.order
!= ORDER_CALLEE
) {
2734 err
= find_prev_cpumode(chain
, thread
, cursor
, parent
,
2735 root_al
, &cpumode
, j
);
2737 return (err
< 0) ? err
: 0;
2741 err
= add_callchain_ip(thread
, cursor
, parent
,
2742 root_al
, &cpumode
, ip
,
2743 false, NULL
, NULL
, 0);
2746 return (err
< 0) ? err
: 0;
2752 static int append_inlines(struct callchain_cursor
*cursor
, struct map_symbol
*ms
, u64 ip
)
2754 struct symbol
*sym
= ms
->sym
;
2755 struct map
*map
= ms
->map
;
2756 struct inline_node
*inline_node
;
2757 struct inline_list
*ilist
;
2761 if (!symbol_conf
.inline_name
|| !map
|| !sym
)
2764 addr
= map__map_ip(map
, ip
);
2765 addr
= map__rip_2objdump(map
, addr
);
2767 inline_node
= inlines__tree_find(&map
->dso
->inlined_nodes
, addr
);
2769 inline_node
= dso__parse_addr_inlines(map
->dso
, addr
, sym
);
2772 inlines__tree_insert(&map
->dso
->inlined_nodes
, inline_node
);
2775 list_for_each_entry(ilist
, &inline_node
->val
, list
) {
2776 struct map_symbol ilist_ms
= {
2779 .sym
= ilist
->symbol
,
2781 ret
= callchain_cursor_append(cursor
, ip
, &ilist_ms
, false,
2782 NULL
, 0, 0, 0, ilist
->srcline
);
2791 static int unwind_entry(struct unwind_entry
*entry
, void *arg
)
2793 struct callchain_cursor
*cursor
= arg
;
2794 const char *srcline
= NULL
;
2795 u64 addr
= entry
->ip
;
2797 if (symbol_conf
.hide_unresolved
&& entry
->ms
.sym
== NULL
)
2800 if (append_inlines(cursor
, &entry
->ms
, entry
->ip
) == 0)
2804 * Convert entry->ip from a virtual address to an offset in
2805 * its corresponding binary.
2808 addr
= map__map_ip(entry
->ms
.map
, entry
->ip
);
2810 srcline
= callchain_srcline(&entry
->ms
, addr
);
2811 return callchain_cursor_append(cursor
, entry
->ip
, &entry
->ms
,
2812 false, NULL
, 0, 0, 0, srcline
);
2815 static int thread__resolve_callchain_unwind(struct thread
*thread
,
2816 struct callchain_cursor
*cursor
,
2817 struct evsel
*evsel
,
2818 struct perf_sample
*sample
,
2821 /* Can we do dwarf post unwind? */
2822 if (!((evsel
->core
.attr
.sample_type
& PERF_SAMPLE_REGS_USER
) &&
2823 (evsel
->core
.attr
.sample_type
& PERF_SAMPLE_STACK_USER
)))
2826 /* Bail out if nothing was captured. */
2827 if ((!sample
->user_regs
.regs
) ||
2828 (!sample
->user_stack
.size
))
2831 return unwind__get_entries(unwind_entry
, cursor
,
2832 thread
, sample
, max_stack
);
2835 int thread__resolve_callchain(struct thread
*thread
,
2836 struct callchain_cursor
*cursor
,
2837 struct evsel
*evsel
,
2838 struct perf_sample
*sample
,
2839 struct symbol
**parent
,
2840 struct addr_location
*root_al
,
2845 callchain_cursor_reset(cursor
);
2847 if (callchain_param
.order
== ORDER_CALLEE
) {
2848 ret
= thread__resolve_callchain_sample(thread
, cursor
,
2854 ret
= thread__resolve_callchain_unwind(thread
, cursor
,
2858 ret
= thread__resolve_callchain_unwind(thread
, cursor
,
2863 ret
= thread__resolve_callchain_sample(thread
, cursor
,
2872 int machine__for_each_thread(struct machine
*machine
,
2873 int (*fn
)(struct thread
*thread
, void *p
),
2876 struct threads
*threads
;
2878 struct thread
*thread
;
2882 for (i
= 0; i
< THREADS__TABLE_SIZE
; i
++) {
2883 threads
= &machine
->threads
[i
];
2884 for (nd
= rb_first_cached(&threads
->entries
); nd
;
2886 thread
= rb_entry(nd
, struct thread
, rb_node
);
2887 rc
= fn(thread
, priv
);
2892 list_for_each_entry(thread
, &threads
->dead
, node
) {
2893 rc
= fn(thread
, priv
);
2901 int machines__for_each_thread(struct machines
*machines
,
2902 int (*fn
)(struct thread
*thread
, void *p
),
2908 rc
= machine__for_each_thread(&machines
->host
, fn
, priv
);
2912 for (nd
= rb_first_cached(&machines
->guests
); nd
; nd
= rb_next(nd
)) {
2913 struct machine
*machine
= rb_entry(nd
, struct machine
, rb_node
);
2915 rc
= machine__for_each_thread(machine
, fn
, priv
);
2922 pid_t
machine__get_current_tid(struct machine
*machine
, int cpu
)
2924 int nr_cpus
= min(machine
->env
->nr_cpus_online
, MAX_NR_CPUS
);
2926 if (cpu
< 0 || cpu
>= nr_cpus
|| !machine
->current_tid
)
2929 return machine
->current_tid
[cpu
];
2932 int machine__set_current_tid(struct machine
*machine
, int cpu
, pid_t pid
,
2935 struct thread
*thread
;
2936 int nr_cpus
= min(machine
->env
->nr_cpus_online
, MAX_NR_CPUS
);
2941 if (!machine
->current_tid
) {
2944 machine
->current_tid
= calloc(nr_cpus
, sizeof(pid_t
));
2945 if (!machine
->current_tid
)
2947 for (i
= 0; i
< nr_cpus
; i
++)
2948 machine
->current_tid
[i
] = -1;
2951 if (cpu
>= nr_cpus
) {
2952 pr_err("Requested CPU %d too large. ", cpu
);
2953 pr_err("Consider raising MAX_NR_CPUS\n");
2957 machine
->current_tid
[cpu
] = tid
;
2959 thread
= machine__findnew_thread(machine
, pid
, tid
);
2964 thread__put(thread
);
2970 * Compares the raw arch string. N.B. see instead perf_env__arch() if a
2971 * normalized arch is needed.
2973 bool machine__is(struct machine
*machine
, const char *arch
)
2975 return machine
&& !strcmp(perf_env__raw_arch(machine
->env
), arch
);
2978 int machine__nr_cpus_avail(struct machine
*machine
)
2980 return machine
? perf_env__nr_cpus_avail(machine
->env
) : 0;
2983 int machine__get_kernel_start(struct machine
*machine
)
2985 struct map
*map
= machine__kernel_map(machine
);
2989 * The only addresses above 2^63 are kernel addresses of a 64-bit
2990 * kernel. Note that addresses are unsigned so that on a 32-bit system
2991 * all addresses including kernel addresses are less than 2^32. In
2992 * that case (32-bit system), if the kernel mapping is unknown, all
2993 * addresses will be assumed to be in user space - see
2994 * machine__kernel_ip().
2996 machine
->kernel_start
= 1ULL << 63;
2998 err
= map__load(map
);
3000 * On x86_64, PTI entry trampolines are less than the
3001 * start of kernel text, but still above 2^63. So leave
3002 * kernel_start = 1ULL << 63 for x86_64.
3004 if (!err
&& !machine__is(machine
, "x86_64"))
3005 machine
->kernel_start
= map
->start
;
3010 u8
machine__addr_cpumode(struct machine
*machine
, u8 cpumode
, u64 addr
)
3012 u8 addr_cpumode
= cpumode
;
3015 if (!machine
->single_address_space
)
3018 kernel_ip
= machine__kernel_ip(machine
, addr
);
3020 case PERF_RECORD_MISC_KERNEL
:
3021 case PERF_RECORD_MISC_USER
:
3022 addr_cpumode
= kernel_ip
? PERF_RECORD_MISC_KERNEL
:
3023 PERF_RECORD_MISC_USER
;
3025 case PERF_RECORD_MISC_GUEST_KERNEL
:
3026 case PERF_RECORD_MISC_GUEST_USER
:
3027 addr_cpumode
= kernel_ip
? PERF_RECORD_MISC_GUEST_KERNEL
:
3028 PERF_RECORD_MISC_GUEST_USER
;
3034 return addr_cpumode
;
3037 struct dso
*machine__findnew_dso_id(struct machine
*machine
, const char *filename
, struct dso_id
*id
)
3039 return dsos__findnew_id(&machine
->dsos
, filename
, id
);
3042 struct dso
*machine__findnew_dso(struct machine
*machine
, const char *filename
)
3044 return machine__findnew_dso_id(machine
, filename
, NULL
);
3047 char *machine__resolve_kernel_addr(void *vmachine
, unsigned long long *addrp
, char **modp
)
3049 struct machine
*machine
= vmachine
;
3051 struct symbol
*sym
= machine__find_kernel_symbol(machine
, *addrp
, &map
);
3056 *modp
= __map__is_kmodule(map
) ? (char *)map
->dso
->short_name
: NULL
;
3057 *addrp
= map
->unmap_ip(map
, sym
->start
);