1 // SPDX-License-Identifier: GPL-2.0-only
3 * auxtrace.c: AUX area trace support
4 * Copyright (c) 2013-2015, Intel Corporation.
15 #include <linux/kernel.h>
16 #include <linux/perf_event.h>
17 #include <linux/types.h>
18 #include <linux/bitops.h>
19 #include <linux/log2.h>
20 #include <linux/string.h>
21 #include <linux/time64.h>
23 #include <sys/param.h>
26 #include <linux/list.h>
27 #include <linux/zalloc.h>
34 #include "evsel_config.h"
36 #include "util/perf_api_probe.h"
37 #include "util/synthetic-events.h"
38 #include "thread_map.h"
42 #include <linux/hash.h>
48 #include <subcmd/parse-options.h>
52 #include "intel-bts.h"
54 #include "s390-cpumsf.h"
55 #include "util/mmap.h"
57 #include <linux/ctype.h>
58 #include <linux/kernel.h>
59 #include "symbol/kallsyms.h"
60 #include <internal/lib.h>
63 * Make a group from 'leader' to 'last', requiring that the events were not
64 * already grouped to a different leader.
66 static int perf_evlist__regroup(struct evlist
*evlist
,
73 if (!evsel__is_group_leader(leader
))
77 evlist__for_each_entry(evlist
, evsel
) {
79 if (!(evsel
->leader
== leader
||
80 (evsel
->leader
== evsel
&&
81 evsel
->core
.nr_members
<= 1)))
83 } else if (evsel
== leader
) {
91 evlist__for_each_entry(evlist
, evsel
) {
93 if (evsel
->leader
!= leader
) {
94 evsel
->leader
= leader
;
95 if (leader
->core
.nr_members
< 1)
96 leader
->core
.nr_members
= 1;
97 leader
->core
.nr_members
+= 1;
99 } else if (evsel
== leader
) {
109 static bool auxtrace__dont_decode(struct perf_session
*session
)
111 return !session
->itrace_synth_opts
||
112 session
->itrace_synth_opts
->dont_decode
;
115 int auxtrace_mmap__mmap(struct auxtrace_mmap
*mm
,
116 struct auxtrace_mmap_params
*mp
,
117 void *userpg
, int fd
)
119 struct perf_event_mmap_page
*pc
= userpg
;
121 WARN_ONCE(mm
->base
, "Uninitialized auxtrace_mmap\n");
136 #if BITS_PER_LONG != 64 && !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT)
137 pr_err("Cannot use AUX area tracing mmaps\n");
141 pc
->aux_offset
= mp
->offset
;
142 pc
->aux_size
= mp
->len
;
144 mm
->base
= mmap(NULL
, mp
->len
, mp
->prot
, MAP_SHARED
, fd
, mp
->offset
);
145 if (mm
->base
== MAP_FAILED
) {
146 pr_debug2("failed to mmap AUX area\n");
154 void auxtrace_mmap__munmap(struct auxtrace_mmap
*mm
)
157 munmap(mm
->base
, mm
->len
);
162 void auxtrace_mmap_params__init(struct auxtrace_mmap_params
*mp
,
163 off_t auxtrace_offset
,
164 unsigned int auxtrace_pages
,
165 bool auxtrace_overwrite
)
167 if (auxtrace_pages
) {
168 mp
->offset
= auxtrace_offset
;
169 mp
->len
= auxtrace_pages
* (size_t)page_size
;
170 mp
->mask
= is_power_of_2(mp
->len
) ? mp
->len
- 1 : 0;
171 mp
->prot
= PROT_READ
| (auxtrace_overwrite
? 0 : PROT_WRITE
);
172 pr_debug2("AUX area mmap length %zu\n", mp
->len
);
178 void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params
*mp
,
179 struct evlist
*evlist
, int idx
,
185 mp
->cpu
= evlist
->core
.cpus
->map
[idx
];
186 if (evlist
->core
.threads
)
187 mp
->tid
= perf_thread_map__pid(evlist
->core
.threads
, 0);
192 mp
->tid
= perf_thread_map__pid(evlist
->core
.threads
, idx
);
196 #define AUXTRACE_INIT_NR_QUEUES 32
198 static struct auxtrace_queue
*auxtrace_alloc_queue_array(unsigned int nr_queues
)
200 struct auxtrace_queue
*queue_array
;
201 unsigned int max_nr_queues
, i
;
203 max_nr_queues
= UINT_MAX
/ sizeof(struct auxtrace_queue
);
204 if (nr_queues
> max_nr_queues
)
207 queue_array
= calloc(nr_queues
, sizeof(struct auxtrace_queue
));
211 for (i
= 0; i
< nr_queues
; i
++) {
212 INIT_LIST_HEAD(&queue_array
[i
].head
);
213 queue_array
[i
].priv
= NULL
;
219 int auxtrace_queues__init(struct auxtrace_queues
*queues
)
221 queues
->nr_queues
= AUXTRACE_INIT_NR_QUEUES
;
222 queues
->queue_array
= auxtrace_alloc_queue_array(queues
->nr_queues
);
223 if (!queues
->queue_array
)
228 static int auxtrace_queues__grow(struct auxtrace_queues
*queues
,
229 unsigned int new_nr_queues
)
231 unsigned int nr_queues
= queues
->nr_queues
;
232 struct auxtrace_queue
*queue_array
;
236 nr_queues
= AUXTRACE_INIT_NR_QUEUES
;
238 while (nr_queues
&& nr_queues
< new_nr_queues
)
241 if (nr_queues
< queues
->nr_queues
|| nr_queues
< new_nr_queues
)
244 queue_array
= auxtrace_alloc_queue_array(nr_queues
);
248 for (i
= 0; i
< queues
->nr_queues
; i
++) {
249 list_splice_tail(&queues
->queue_array
[i
].head
,
250 &queue_array
[i
].head
);
251 queue_array
[i
].tid
= queues
->queue_array
[i
].tid
;
252 queue_array
[i
].cpu
= queues
->queue_array
[i
].cpu
;
253 queue_array
[i
].set
= queues
->queue_array
[i
].set
;
254 queue_array
[i
].priv
= queues
->queue_array
[i
].priv
;
257 queues
->nr_queues
= nr_queues
;
258 queues
->queue_array
= queue_array
;
263 static void *auxtrace_copy_data(u64 size
, struct perf_session
*session
)
265 int fd
= perf_data__fd(session
->data
);
269 if (size
> SSIZE_MAX
)
276 ret
= readn(fd
, p
, size
);
277 if (ret
!= (ssize_t
)size
) {
285 static int auxtrace_queues__queue_buffer(struct auxtrace_queues
*queues
,
287 struct auxtrace_buffer
*buffer
)
289 struct auxtrace_queue
*queue
;
292 if (idx
>= queues
->nr_queues
) {
293 err
= auxtrace_queues__grow(queues
, idx
+ 1);
298 queue
= &queues
->queue_array
[idx
];
302 queue
->tid
= buffer
->tid
;
303 queue
->cpu
= buffer
->cpu
;
304 } else if (buffer
->cpu
!= queue
->cpu
|| buffer
->tid
!= queue
->tid
) {
305 pr_err("auxtrace queue conflict: cpu %d, tid %d vs cpu %d, tid %d\n",
306 queue
->cpu
, queue
->tid
, buffer
->cpu
, buffer
->tid
);
310 buffer
->buffer_nr
= queues
->next_buffer_nr
++;
312 list_add_tail(&buffer
->list
, &queue
->head
);
314 queues
->new_data
= true;
315 queues
->populated
= true;
320 /* Limit buffers to 32MiB on 32-bit */
321 #define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024)
323 static int auxtrace_queues__split_buffer(struct auxtrace_queues
*queues
,
325 struct auxtrace_buffer
*buffer
)
327 u64 sz
= buffer
->size
;
328 bool consecutive
= false;
329 struct auxtrace_buffer
*b
;
332 while (sz
> BUFFER_LIMIT_FOR_32_BIT
) {
333 b
= memdup(buffer
, sizeof(struct auxtrace_buffer
));
336 b
->size
= BUFFER_LIMIT_FOR_32_BIT
;
337 b
->consecutive
= consecutive
;
338 err
= auxtrace_queues__queue_buffer(queues
, idx
, b
);
340 auxtrace_buffer__free(b
);
343 buffer
->data_offset
+= BUFFER_LIMIT_FOR_32_BIT
;
344 sz
-= BUFFER_LIMIT_FOR_32_BIT
;
349 buffer
->consecutive
= consecutive
;
354 static bool filter_cpu(struct perf_session
*session
, int cpu
)
356 unsigned long *cpu_bitmap
= session
->itrace_synth_opts
->cpu_bitmap
;
358 return cpu_bitmap
&& cpu
!= -1 && !test_bit(cpu
, cpu_bitmap
);
361 static int auxtrace_queues__add_buffer(struct auxtrace_queues
*queues
,
362 struct perf_session
*session
,
364 struct auxtrace_buffer
*buffer
,
365 struct auxtrace_buffer
**buffer_ptr
)
369 if (filter_cpu(session
, buffer
->cpu
))
372 buffer
= memdup(buffer
, sizeof(*buffer
));
376 if (session
->one_mmap
) {
377 buffer
->data
= buffer
->data_offset
- session
->one_mmap_offset
+
378 session
->one_mmap_addr
;
379 } else if (perf_data__is_pipe(session
->data
)) {
380 buffer
->data
= auxtrace_copy_data(buffer
->size
, session
);
383 buffer
->data_needs_freeing
= true;
384 } else if (BITS_PER_LONG
== 32 &&
385 buffer
->size
> BUFFER_LIMIT_FOR_32_BIT
) {
386 err
= auxtrace_queues__split_buffer(queues
, idx
, buffer
);
391 err
= auxtrace_queues__queue_buffer(queues
, idx
, buffer
);
395 /* FIXME: Doesn't work for split buffer */
397 *buffer_ptr
= buffer
;
402 auxtrace_buffer__free(buffer
);
406 int auxtrace_queues__add_event(struct auxtrace_queues
*queues
,
407 struct perf_session
*session
,
408 union perf_event
*event
, off_t data_offset
,
409 struct auxtrace_buffer
**buffer_ptr
)
411 struct auxtrace_buffer buffer
= {
413 .tid
= event
->auxtrace
.tid
,
414 .cpu
= event
->auxtrace
.cpu
,
415 .data_offset
= data_offset
,
416 .offset
= event
->auxtrace
.offset
,
417 .reference
= event
->auxtrace
.reference
,
418 .size
= event
->auxtrace
.size
,
420 unsigned int idx
= event
->auxtrace
.idx
;
422 return auxtrace_queues__add_buffer(queues
, session
, idx
, &buffer
,
426 static int auxtrace_queues__add_indexed_event(struct auxtrace_queues
*queues
,
427 struct perf_session
*session
,
428 off_t file_offset
, size_t sz
)
430 union perf_event
*event
;
432 char buf
[PERF_SAMPLE_MAX_SIZE
];
434 err
= perf_session__peek_event(session
, file_offset
, buf
,
435 PERF_SAMPLE_MAX_SIZE
, &event
, NULL
);
439 if (event
->header
.type
== PERF_RECORD_AUXTRACE
) {
440 if (event
->header
.size
< sizeof(struct perf_record_auxtrace
) ||
441 event
->header
.size
!= sz
) {
445 file_offset
+= event
->header
.size
;
446 err
= auxtrace_queues__add_event(queues
, session
, event
,
453 void auxtrace_queues__free(struct auxtrace_queues
*queues
)
457 for (i
= 0; i
< queues
->nr_queues
; i
++) {
458 while (!list_empty(&queues
->queue_array
[i
].head
)) {
459 struct auxtrace_buffer
*buffer
;
461 buffer
= list_entry(queues
->queue_array
[i
].head
.next
,
462 struct auxtrace_buffer
, list
);
463 list_del_init(&buffer
->list
);
464 auxtrace_buffer__free(buffer
);
468 zfree(&queues
->queue_array
);
469 queues
->nr_queues
= 0;
472 static void auxtrace_heapify(struct auxtrace_heap_item
*heap_array
,
473 unsigned int pos
, unsigned int queue_nr
,
479 parent
= (pos
- 1) >> 1;
480 if (heap_array
[parent
].ordinal
<= ordinal
)
482 heap_array
[pos
] = heap_array
[parent
];
485 heap_array
[pos
].queue_nr
= queue_nr
;
486 heap_array
[pos
].ordinal
= ordinal
;
489 int auxtrace_heap__add(struct auxtrace_heap
*heap
, unsigned int queue_nr
,
492 struct auxtrace_heap_item
*heap_array
;
494 if (queue_nr
>= heap
->heap_sz
) {
495 unsigned int heap_sz
= AUXTRACE_INIT_NR_QUEUES
;
497 while (heap_sz
<= queue_nr
)
499 heap_array
= realloc(heap
->heap_array
,
500 heap_sz
* sizeof(struct auxtrace_heap_item
));
503 heap
->heap_array
= heap_array
;
504 heap
->heap_sz
= heap_sz
;
507 auxtrace_heapify(heap
->heap_array
, heap
->heap_cnt
++, queue_nr
, ordinal
);
512 void auxtrace_heap__free(struct auxtrace_heap
*heap
)
514 zfree(&heap
->heap_array
);
519 void auxtrace_heap__pop(struct auxtrace_heap
*heap
)
521 unsigned int pos
, last
, heap_cnt
= heap
->heap_cnt
;
522 struct auxtrace_heap_item
*heap_array
;
529 heap_array
= heap
->heap_array
;
533 unsigned int left
, right
;
535 left
= (pos
<< 1) + 1;
536 if (left
>= heap_cnt
)
539 if (right
>= heap_cnt
) {
540 heap_array
[pos
] = heap_array
[left
];
543 if (heap_array
[left
].ordinal
< heap_array
[right
].ordinal
) {
544 heap_array
[pos
] = heap_array
[left
];
547 heap_array
[pos
] = heap_array
[right
];
553 auxtrace_heapify(heap_array
, pos
, heap_array
[last
].queue_nr
,
554 heap_array
[last
].ordinal
);
557 size_t auxtrace_record__info_priv_size(struct auxtrace_record
*itr
,
558 struct evlist
*evlist
)
561 return itr
->info_priv_size(itr
, evlist
);
565 static int auxtrace_not_supported(void)
567 pr_err("AUX area tracing is not supported on this architecture\n");
571 int auxtrace_record__info_fill(struct auxtrace_record
*itr
,
572 struct perf_session
*session
,
573 struct perf_record_auxtrace_info
*auxtrace_info
,
577 return itr
->info_fill(itr
, session
, auxtrace_info
, priv_size
);
578 return auxtrace_not_supported();
581 void auxtrace_record__free(struct auxtrace_record
*itr
)
587 int auxtrace_record__snapshot_start(struct auxtrace_record
*itr
)
589 if (itr
&& itr
->snapshot_start
)
590 return itr
->snapshot_start(itr
);
594 int auxtrace_record__snapshot_finish(struct auxtrace_record
*itr
, bool on_exit
)
596 if (!on_exit
&& itr
&& itr
->snapshot_finish
)
597 return itr
->snapshot_finish(itr
);
601 int auxtrace_record__find_snapshot(struct auxtrace_record
*itr
, int idx
,
602 struct auxtrace_mmap
*mm
,
603 unsigned char *data
, u64
*head
, u64
*old
)
605 if (itr
&& itr
->find_snapshot
)
606 return itr
->find_snapshot(itr
, idx
, mm
, data
, head
, old
);
610 int auxtrace_record__options(struct auxtrace_record
*itr
,
611 struct evlist
*evlist
,
612 struct record_opts
*opts
)
615 itr
->evlist
= evlist
;
616 return itr
->recording_options(itr
, evlist
, opts
);
621 u64
auxtrace_record__reference(struct auxtrace_record
*itr
)
624 return itr
->reference(itr
);
628 int auxtrace_parse_snapshot_options(struct auxtrace_record
*itr
,
629 struct record_opts
*opts
, const char *str
)
634 /* PMU-agnostic options */
637 opts
->auxtrace_snapshot_on_exit
= true;
645 return itr
->parse_snapshot_options(itr
, opts
, str
);
647 pr_err("No AUX area tracing to snapshot\n");
651 int auxtrace_record__read_finish(struct auxtrace_record
*itr
, int idx
)
655 if (!itr
->evlist
|| !itr
->pmu
)
658 evlist__for_each_entry(itr
->evlist
, evsel
) {
659 if (evsel
->core
.attr
.type
== itr
->pmu
->type
) {
662 return perf_evlist__enable_event_idx(itr
->evlist
, evsel
,
670 * Event record size is 16-bit which results in a maximum size of about 64KiB.
671 * Allow about 4KiB for the rest of the sample record, to give a maximum
672 * AUX area sample size of 60KiB.
674 #define MAX_AUX_SAMPLE_SIZE (60 * 1024)
676 /* Arbitrary default size if no other default provided */
677 #define DEFAULT_AUX_SAMPLE_SIZE (4 * 1024)
679 static int auxtrace_validate_aux_sample_size(struct evlist
*evlist
,
680 struct record_opts
*opts
)
683 bool has_aux_leader
= false;
686 evlist__for_each_entry(evlist
, evsel
) {
687 sz
= evsel
->core
.attr
.aux_sample_size
;
688 if (evsel__is_group_leader(evsel
)) {
689 has_aux_leader
= evsel__is_aux_event(evsel
);
692 pr_err("Cannot add AUX area sampling to an AUX area event\n");
694 pr_err("Cannot add AUX area sampling to a group leader\n");
698 if (sz
> MAX_AUX_SAMPLE_SIZE
) {
699 pr_err("AUX area sample size %u too big, max. %d\n",
700 sz
, MAX_AUX_SAMPLE_SIZE
);
704 if (!has_aux_leader
) {
705 pr_err("Cannot add AUX area sampling because group leader is not an AUX area event\n");
708 evsel__set_sample_bit(evsel
, AUX
);
709 opts
->auxtrace_sample_mode
= true;
711 evsel__reset_sample_bit(evsel
, AUX
);
715 if (!opts
->auxtrace_sample_mode
) {
716 pr_err("AUX area sampling requires an AUX area event group leader plus other events to which to add samples\n");
720 if (!perf_can_aux_sample()) {
721 pr_err("AUX area sampling is not supported by kernel\n");
728 int auxtrace_parse_sample_options(struct auxtrace_record
*itr
,
729 struct evlist
*evlist
,
730 struct record_opts
*opts
, const char *str
)
732 struct perf_evsel_config_term
*term
;
733 struct evsel
*aux_evsel
;
734 bool has_aux_sample_size
= false;
735 bool has_aux_leader
= false;
744 pr_err("No AUX area event to sample\n");
748 sz
= strtoul(str
, &endptr
, 0);
749 if (*endptr
|| sz
> UINT_MAX
) {
750 pr_err("Bad AUX area sampling option: '%s'\n", str
);
755 sz
= itr
->default_aux_sample_size
;
758 sz
= DEFAULT_AUX_SAMPLE_SIZE
;
760 /* Set aux_sample_size based on --aux-sample option */
761 evlist__for_each_entry(evlist
, evsel
) {
762 if (evsel__is_group_leader(evsel
)) {
763 has_aux_leader
= evsel__is_aux_event(evsel
);
764 } else if (has_aux_leader
) {
765 evsel
->core
.attr
.aux_sample_size
= sz
;
770 /* Override with aux_sample_size from config term */
771 evlist__for_each_entry(evlist
, evsel
) {
772 if (evsel__is_aux_event(evsel
))
774 term
= perf_evsel__get_config_term(evsel
, AUX_SAMPLE_SIZE
);
776 has_aux_sample_size
= true;
777 evsel
->core
.attr
.aux_sample_size
= term
->val
.aux_sample_size
;
778 /* If possible, group with the AUX event */
779 if (aux_evsel
&& evsel
->core
.attr
.aux_sample_size
)
780 perf_evlist__regroup(evlist
, aux_evsel
, evsel
);
784 if (!str
&& !has_aux_sample_size
)
788 pr_err("No AUX area event to sample\n");
792 return auxtrace_validate_aux_sample_size(evlist
, opts
);
795 struct auxtrace_record
*__weak
796 auxtrace_record__init(struct evlist
*evlist __maybe_unused
, int *err
)
802 static int auxtrace_index__alloc(struct list_head
*head
)
804 struct auxtrace_index
*auxtrace_index
;
806 auxtrace_index
= malloc(sizeof(struct auxtrace_index
));
810 auxtrace_index
->nr
= 0;
811 INIT_LIST_HEAD(&auxtrace_index
->list
);
813 list_add_tail(&auxtrace_index
->list
, head
);
818 void auxtrace_index__free(struct list_head
*head
)
820 struct auxtrace_index
*auxtrace_index
, *n
;
822 list_for_each_entry_safe(auxtrace_index
, n
, head
, list
) {
823 list_del_init(&auxtrace_index
->list
);
824 free(auxtrace_index
);
828 static struct auxtrace_index
*auxtrace_index__last(struct list_head
*head
)
830 struct auxtrace_index
*auxtrace_index
;
833 if (list_empty(head
)) {
834 err
= auxtrace_index__alloc(head
);
839 auxtrace_index
= list_entry(head
->prev
, struct auxtrace_index
, list
);
841 if (auxtrace_index
->nr
>= PERF_AUXTRACE_INDEX_ENTRY_COUNT
) {
842 err
= auxtrace_index__alloc(head
);
845 auxtrace_index
= list_entry(head
->prev
, struct auxtrace_index
,
849 return auxtrace_index
;
852 int auxtrace_index__auxtrace_event(struct list_head
*head
,
853 union perf_event
*event
, off_t file_offset
)
855 struct auxtrace_index
*auxtrace_index
;
858 auxtrace_index
= auxtrace_index__last(head
);
862 nr
= auxtrace_index
->nr
;
863 auxtrace_index
->entries
[nr
].file_offset
= file_offset
;
864 auxtrace_index
->entries
[nr
].sz
= event
->header
.size
;
865 auxtrace_index
->nr
+= 1;
870 static int auxtrace_index__do_write(int fd
,
871 struct auxtrace_index
*auxtrace_index
)
873 struct auxtrace_index_entry ent
;
876 for (i
= 0; i
< auxtrace_index
->nr
; i
++) {
877 ent
.file_offset
= auxtrace_index
->entries
[i
].file_offset
;
878 ent
.sz
= auxtrace_index
->entries
[i
].sz
;
879 if (writen(fd
, &ent
, sizeof(ent
)) != sizeof(ent
))
885 int auxtrace_index__write(int fd
, struct list_head
*head
)
887 struct auxtrace_index
*auxtrace_index
;
891 list_for_each_entry(auxtrace_index
, head
, list
)
892 total
+= auxtrace_index
->nr
;
894 if (writen(fd
, &total
, sizeof(total
)) != sizeof(total
))
897 list_for_each_entry(auxtrace_index
, head
, list
) {
898 err
= auxtrace_index__do_write(fd
, auxtrace_index
);
906 static int auxtrace_index__process_entry(int fd
, struct list_head
*head
,
909 struct auxtrace_index
*auxtrace_index
;
910 struct auxtrace_index_entry ent
;
913 if (readn(fd
, &ent
, sizeof(ent
)) != sizeof(ent
))
916 auxtrace_index
= auxtrace_index__last(head
);
920 nr
= auxtrace_index
->nr
;
922 auxtrace_index
->entries
[nr
].file_offset
=
923 bswap_64(ent
.file_offset
);
924 auxtrace_index
->entries
[nr
].sz
= bswap_64(ent
.sz
);
926 auxtrace_index
->entries
[nr
].file_offset
= ent
.file_offset
;
927 auxtrace_index
->entries
[nr
].sz
= ent
.sz
;
930 auxtrace_index
->nr
= nr
+ 1;
935 int auxtrace_index__process(int fd
, u64 size
, struct perf_session
*session
,
938 struct list_head
*head
= &session
->auxtrace_index
;
941 if (readn(fd
, &nr
, sizeof(u64
)) != sizeof(u64
))
947 if (sizeof(u64
) + nr
* sizeof(struct auxtrace_index_entry
) > size
)
953 err
= auxtrace_index__process_entry(fd
, head
, needs_swap
);
961 static int auxtrace_queues__process_index_entry(struct auxtrace_queues
*queues
,
962 struct perf_session
*session
,
963 struct auxtrace_index_entry
*ent
)
965 return auxtrace_queues__add_indexed_event(queues
, session
,
966 ent
->file_offset
, ent
->sz
);
969 int auxtrace_queues__process_index(struct auxtrace_queues
*queues
,
970 struct perf_session
*session
)
972 struct auxtrace_index
*auxtrace_index
;
973 struct auxtrace_index_entry
*ent
;
977 if (auxtrace__dont_decode(session
))
980 list_for_each_entry(auxtrace_index
, &session
->auxtrace_index
, list
) {
981 for (i
= 0; i
< auxtrace_index
->nr
; i
++) {
982 ent
= &auxtrace_index
->entries
[i
];
983 err
= auxtrace_queues__process_index_entry(queues
,
993 struct auxtrace_buffer
*auxtrace_buffer__next(struct auxtrace_queue
*queue
,
994 struct auxtrace_buffer
*buffer
)
997 if (list_is_last(&buffer
->list
, &queue
->head
))
999 return list_entry(buffer
->list
.next
, struct auxtrace_buffer
,
1002 if (list_empty(&queue
->head
))
1004 return list_entry(queue
->head
.next
, struct auxtrace_buffer
,
1009 struct auxtrace_queue
*auxtrace_queues__sample_queue(struct auxtrace_queues
*queues
,
1010 struct perf_sample
*sample
,
1011 struct perf_session
*session
)
1013 struct perf_sample_id
*sid
;
1021 sid
= perf_evlist__id2sid(session
->evlist
, id
);
1027 if (idx
>= queues
->nr_queues
)
1030 return &queues
->queue_array
[idx
];
1033 int auxtrace_queues__add_sample(struct auxtrace_queues
*queues
,
1034 struct perf_session
*session
,
1035 struct perf_sample
*sample
, u64 data_offset
,
1038 struct auxtrace_buffer buffer
= {
1040 .data_offset
= data_offset
,
1041 .reference
= reference
,
1042 .size
= sample
->aux_sample
.size
,
1044 struct perf_sample_id
*sid
;
1045 u64 id
= sample
->id
;
1051 sid
= perf_evlist__id2sid(session
->evlist
, id
);
1056 buffer
.tid
= sid
->tid
;
1057 buffer
.cpu
= sid
->cpu
;
1059 return auxtrace_queues__add_buffer(queues
, session
, idx
, &buffer
, NULL
);
1067 static int auxtrace_queue_data_cb(struct perf_session
*session
,
1068 union perf_event
*event
, u64 offset
,
1071 struct queue_data
*qd
= data
;
1072 struct perf_sample sample
;
1075 if (qd
->events
&& event
->header
.type
== PERF_RECORD_AUXTRACE
) {
1076 if (event
->header
.size
< sizeof(struct perf_record_auxtrace
))
1078 offset
+= event
->header
.size
;
1079 return session
->auxtrace
->queue_data(session
, NULL
, event
,
1083 if (!qd
->samples
|| event
->header
.type
!= PERF_RECORD_SAMPLE
)
1086 err
= perf_evlist__parse_sample(session
->evlist
, event
, &sample
);
1090 if (!sample
.aux_sample
.size
)
1093 offset
+= sample
.aux_sample
.data
- (void *)event
;
1095 return session
->auxtrace
->queue_data(session
, &sample
, NULL
, offset
);
1098 int auxtrace_queue_data(struct perf_session
*session
, bool samples
, bool events
)
1100 struct queue_data qd
= {
1105 if (auxtrace__dont_decode(session
))
1108 if (!session
->auxtrace
|| !session
->auxtrace
->queue_data
)
1111 return perf_session__peek_events(session
, session
->header
.data_offset
,
1112 session
->header
.data_size
,
1113 auxtrace_queue_data_cb
, &qd
);
1116 void *auxtrace_buffer__get_data(struct auxtrace_buffer
*buffer
, int fd
)
1118 size_t adj
= buffer
->data_offset
& (page_size
- 1);
1119 size_t size
= buffer
->size
+ adj
;
1120 off_t file_offset
= buffer
->data_offset
- adj
;
1124 return buffer
->data
;
1126 addr
= mmap(NULL
, size
, PROT_READ
, MAP_SHARED
, fd
, file_offset
);
1127 if (addr
== MAP_FAILED
)
1130 buffer
->mmap_addr
= addr
;
1131 buffer
->mmap_size
= size
;
1133 buffer
->data
= addr
+ adj
;
1135 return buffer
->data
;
1138 void auxtrace_buffer__put_data(struct auxtrace_buffer
*buffer
)
1140 if (!buffer
->data
|| !buffer
->mmap_addr
)
1142 munmap(buffer
->mmap_addr
, buffer
->mmap_size
);
1143 buffer
->mmap_addr
= NULL
;
1144 buffer
->mmap_size
= 0;
1145 buffer
->data
= NULL
;
1146 buffer
->use_data
= NULL
;
1149 void auxtrace_buffer__drop_data(struct auxtrace_buffer
*buffer
)
1151 auxtrace_buffer__put_data(buffer
);
1152 if (buffer
->data_needs_freeing
) {
1153 buffer
->data_needs_freeing
= false;
1154 zfree(&buffer
->data
);
1155 buffer
->use_data
= NULL
;
1160 void auxtrace_buffer__free(struct auxtrace_buffer
*buffer
)
1162 auxtrace_buffer__drop_data(buffer
);
1166 void auxtrace_synth_error(struct perf_record_auxtrace_error
*auxtrace_error
, int type
,
1167 int code
, int cpu
, pid_t pid
, pid_t tid
, u64 ip
,
1168 const char *msg
, u64 timestamp
)
1172 memset(auxtrace_error
, 0, sizeof(struct perf_record_auxtrace_error
));
1174 auxtrace_error
->header
.type
= PERF_RECORD_AUXTRACE_ERROR
;
1175 auxtrace_error
->type
= type
;
1176 auxtrace_error
->code
= code
;
1177 auxtrace_error
->cpu
= cpu
;
1178 auxtrace_error
->pid
= pid
;
1179 auxtrace_error
->tid
= tid
;
1180 auxtrace_error
->fmt
= 1;
1181 auxtrace_error
->ip
= ip
;
1182 auxtrace_error
->time
= timestamp
;
1183 strlcpy(auxtrace_error
->msg
, msg
, MAX_AUXTRACE_ERROR_MSG
);
1185 size
= (void *)auxtrace_error
->msg
- (void *)auxtrace_error
+
1186 strlen(auxtrace_error
->msg
) + 1;
1187 auxtrace_error
->header
.size
= PERF_ALIGN(size
, sizeof(u64
));
1190 int perf_event__synthesize_auxtrace_info(struct auxtrace_record
*itr
,
1191 struct perf_tool
*tool
,
1192 struct perf_session
*session
,
1193 perf_event__handler_t process
)
1195 union perf_event
*ev
;
1199 pr_debug2("Synthesizing auxtrace information\n");
1200 priv_size
= auxtrace_record__info_priv_size(itr
, session
->evlist
);
1201 ev
= zalloc(sizeof(struct perf_record_auxtrace_info
) + priv_size
);
1205 ev
->auxtrace_info
.header
.type
= PERF_RECORD_AUXTRACE_INFO
;
1206 ev
->auxtrace_info
.header
.size
= sizeof(struct perf_record_auxtrace_info
) +
1208 err
= auxtrace_record__info_fill(itr
, session
, &ev
->auxtrace_info
,
1213 err
= process(tool
, ev
, NULL
, NULL
);
1219 static void unleader_evsel(struct evlist
*evlist
, struct evsel
*leader
)
1221 struct evsel
*new_leader
= NULL
;
1222 struct evsel
*evsel
;
1224 /* Find new leader for the group */
1225 evlist__for_each_entry(evlist
, evsel
) {
1226 if (evsel
->leader
!= leader
|| evsel
== leader
)
1230 evsel
->leader
= new_leader
;
1233 /* Update group information */
1235 zfree(&new_leader
->group_name
);
1236 new_leader
->group_name
= leader
->group_name
;
1237 leader
->group_name
= NULL
;
1239 new_leader
->core
.nr_members
= leader
->core
.nr_members
- 1;
1240 leader
->core
.nr_members
= 1;
1244 static void unleader_auxtrace(struct perf_session
*session
)
1246 struct evsel
*evsel
;
1248 evlist__for_each_entry(session
->evlist
, evsel
) {
1249 if (auxtrace__evsel_is_auxtrace(session
, evsel
) &&
1250 evsel__is_group_leader(evsel
)) {
1251 unleader_evsel(session
->evlist
, evsel
);
1256 int perf_event__process_auxtrace_info(struct perf_session
*session
,
1257 union perf_event
*event
)
1259 enum auxtrace_type type
= event
->auxtrace_info
.type
;
1263 fprintf(stdout
, " type: %u\n", type
);
1266 case PERF_AUXTRACE_INTEL_PT
:
1267 err
= intel_pt_process_auxtrace_info(event
, session
);
1269 case PERF_AUXTRACE_INTEL_BTS
:
1270 err
= intel_bts_process_auxtrace_info(event
, session
);
1272 case PERF_AUXTRACE_ARM_SPE
:
1273 err
= arm_spe_process_auxtrace_info(event
, session
);
1275 case PERF_AUXTRACE_CS_ETM
:
1276 err
= cs_etm__process_auxtrace_info(event
, session
);
1278 case PERF_AUXTRACE_S390_CPUMSF
:
1279 err
= s390_cpumsf_process_auxtrace_info(event
, session
);
1281 case PERF_AUXTRACE_UNKNOWN
:
1289 unleader_auxtrace(session
);
1294 s64
perf_event__process_auxtrace(struct perf_session
*session
,
1295 union perf_event
*event
)
1300 fprintf(stdout
, " size: %#"PRI_lx64
" offset: %#"PRI_lx64
" ref: %#"PRI_lx64
" idx: %u tid: %d cpu: %d\n",
1301 event
->auxtrace
.size
, event
->auxtrace
.offset
,
1302 event
->auxtrace
.reference
, event
->auxtrace
.idx
,
1303 event
->auxtrace
.tid
, event
->auxtrace
.cpu
);
1305 if (auxtrace__dont_decode(session
))
1306 return event
->auxtrace
.size
;
1308 if (!session
->auxtrace
|| event
->header
.type
!= PERF_RECORD_AUXTRACE
)
1311 err
= session
->auxtrace
->process_auxtrace_event(session
, event
, session
->tool
);
1315 return event
->auxtrace
.size
;
1318 #define PERF_ITRACE_DEFAULT_PERIOD_TYPE PERF_ITRACE_PERIOD_NANOSECS
1319 #define PERF_ITRACE_DEFAULT_PERIOD 100000
1320 #define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ 16
1321 #define PERF_ITRACE_MAX_CALLCHAIN_SZ 1024
1322 #define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ 64
1323 #define PERF_ITRACE_MAX_LAST_BRANCH_SZ 1024
1325 void itrace_synth_opts__set_default(struct itrace_synth_opts
*synth_opts
,
1328 synth_opts
->branches
= true;
1329 synth_opts
->transactions
= true;
1330 synth_opts
->ptwrites
= true;
1331 synth_opts
->pwr_events
= true;
1332 synth_opts
->other_events
= true;
1333 synth_opts
->errors
= true;
1335 synth_opts
->period_type
= PERF_ITRACE_PERIOD_INSTRUCTIONS
;
1336 synth_opts
->period
= 1;
1337 synth_opts
->calls
= true;
1339 synth_opts
->instructions
= true;
1340 synth_opts
->period_type
= PERF_ITRACE_DEFAULT_PERIOD_TYPE
;
1341 synth_opts
->period
= PERF_ITRACE_DEFAULT_PERIOD
;
1343 synth_opts
->callchain_sz
= PERF_ITRACE_DEFAULT_CALLCHAIN_SZ
;
1344 synth_opts
->last_branch_sz
= PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ
;
1345 synth_opts
->initial_skip
= 0;
1349 * Please check tools/perf/Documentation/perf-script.txt for information
1350 * about the options parsed here, which is introduced after this cset,
1351 * when support in 'perf script' for these options is introduced.
1353 int itrace_parse_synth_opts(const struct option
*opt
, const char *str
,
1356 struct itrace_synth_opts
*synth_opts
= opt
->value
;
1359 bool period_type_set
= false;
1360 bool period_set
= false;
1362 synth_opts
->set
= true;
1365 synth_opts
->dont_decode
= true;
1370 itrace_synth_opts__set_default(synth_opts
,
1371 synth_opts
->default_no_sample
);
1375 for (p
= str
; *p
;) {
1378 synth_opts
->instructions
= true;
1379 while (*p
== ' ' || *p
== ',')
1382 synth_opts
->period
= strtoull(p
, &endptr
, 10);
1385 while (*p
== ' ' || *p
== ',')
1389 synth_opts
->period_type
=
1390 PERF_ITRACE_PERIOD_INSTRUCTIONS
;
1391 period_type_set
= true;
1394 synth_opts
->period_type
=
1395 PERF_ITRACE_PERIOD_TICKS
;
1396 period_type_set
= true;
1399 synth_opts
->period
*= 1000;
1402 synth_opts
->period
*= 1000;
1407 synth_opts
->period_type
=
1408 PERF_ITRACE_PERIOD_NANOSECS
;
1409 period_type_set
= true;
1419 synth_opts
->branches
= true;
1422 synth_opts
->transactions
= true;
1425 synth_opts
->ptwrites
= true;
1428 synth_opts
->pwr_events
= true;
1431 synth_opts
->other_events
= true;
1434 synth_opts
->errors
= true;
1437 synth_opts
->log
= true;
1440 synth_opts
->branches
= true;
1441 synth_opts
->calls
= true;
1444 synth_opts
->branches
= true;
1445 synth_opts
->returns
= true;
1450 synth_opts
->add_callchain
= true;
1452 synth_opts
->callchain
= true;
1453 synth_opts
->callchain_sz
=
1454 PERF_ITRACE_DEFAULT_CALLCHAIN_SZ
;
1455 while (*p
== ' ' || *p
== ',')
1460 val
= strtoul(p
, &endptr
, 10);
1462 if (!val
|| val
> PERF_ITRACE_MAX_CALLCHAIN_SZ
)
1464 synth_opts
->callchain_sz
= val
;
1470 synth_opts
->add_last_branch
= true;
1472 synth_opts
->last_branch
= true;
1473 synth_opts
->last_branch_sz
=
1474 PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ
;
1475 while (*p
== ' ' || *p
== ',')
1480 val
= strtoul(p
, &endptr
, 10);
1483 val
> PERF_ITRACE_MAX_LAST_BRANCH_SZ
)
1485 synth_opts
->last_branch_sz
= val
;
1489 synth_opts
->initial_skip
= strtoul(p
, &endptr
, 10);
1502 if (synth_opts
->instructions
) {
1503 if (!period_type_set
)
1504 synth_opts
->period_type
=
1505 PERF_ITRACE_DEFAULT_PERIOD_TYPE
;
1507 synth_opts
->period
= PERF_ITRACE_DEFAULT_PERIOD
;
1513 pr_err("Bad Instruction Tracing options '%s'\n", str
);
1517 static const char * const auxtrace_error_type_name
[] = {
1518 [PERF_AUXTRACE_ERROR_ITRACE
] = "instruction trace",
1521 static const char *auxtrace_error_name(int type
)
1523 const char *error_type_name
= NULL
;
1525 if (type
< PERF_AUXTRACE_ERROR_MAX
)
1526 error_type_name
= auxtrace_error_type_name
[type
];
1527 if (!error_type_name
)
1528 error_type_name
= "unknown AUX";
1529 return error_type_name
;
1532 size_t perf_event__fprintf_auxtrace_error(union perf_event
*event
, FILE *fp
)
1534 struct perf_record_auxtrace_error
*e
= &event
->auxtrace_error
;
1535 unsigned long long nsecs
= e
->time
;
1536 const char *msg
= e
->msg
;
1539 ret
= fprintf(fp
, " %s error type %u",
1540 auxtrace_error_name(e
->type
), e
->type
);
1542 if (e
->fmt
&& nsecs
) {
1543 unsigned long secs
= nsecs
/ NSEC_PER_SEC
;
1545 nsecs
-= secs
* NSEC_PER_SEC
;
1546 ret
+= fprintf(fp
, " time %lu.%09llu", secs
, nsecs
);
1548 ret
+= fprintf(fp
, " time 0");
1552 msg
= (const char *)&e
->time
;
1554 ret
+= fprintf(fp
, " cpu %d pid %d tid %d ip %#"PRI_lx64
" code %u: %s\n",
1555 e
->cpu
, e
->pid
, e
->tid
, e
->ip
, e
->code
, msg
);
1559 void perf_session__auxtrace_error_inc(struct perf_session
*session
,
1560 union perf_event
*event
)
1562 struct perf_record_auxtrace_error
*e
= &event
->auxtrace_error
;
1564 if (e
->type
< PERF_AUXTRACE_ERROR_MAX
)
1565 session
->evlist
->stats
.nr_auxtrace_errors
[e
->type
] += 1;
1568 void events_stats__auxtrace_error_warn(const struct events_stats
*stats
)
1572 for (i
= 0; i
< PERF_AUXTRACE_ERROR_MAX
; i
++) {
1573 if (!stats
->nr_auxtrace_errors
[i
])
1575 ui__warning("%u %s errors\n",
1576 stats
->nr_auxtrace_errors
[i
],
1577 auxtrace_error_name(i
));
1581 int perf_event__process_auxtrace_error(struct perf_session
*session
,
1582 union perf_event
*event
)
1584 if (auxtrace__dont_decode(session
))
1587 perf_event__fprintf_auxtrace_error(event
, stdout
);
1591 static int __auxtrace_mmap__read(struct mmap
*map
,
1592 struct auxtrace_record
*itr
,
1593 struct perf_tool
*tool
, process_auxtrace_t fn
,
1594 bool snapshot
, size_t snapshot_size
)
1596 struct auxtrace_mmap
*mm
= &map
->auxtrace_mmap
;
1597 u64 head
, old
= mm
->prev
, offset
, ref
;
1598 unsigned char *data
= mm
->base
;
1599 size_t size
, head_off
, old_off
, len1
, len2
, padding
;
1600 union perf_event ev
;
1601 void *data1
, *data2
;
1604 head
= auxtrace_mmap__read_snapshot_head(mm
);
1605 if (auxtrace_record__find_snapshot(itr
, mm
->idx
, mm
, data
,
1609 head
= auxtrace_mmap__read_head(mm
);
1615 pr_debug3("auxtrace idx %d old %#"PRIx64
" head %#"PRIx64
" diff %#"PRIx64
"\n",
1616 mm
->idx
, old
, head
, head
- old
);
1619 head_off
= head
& mm
->mask
;
1620 old_off
= old
& mm
->mask
;
1622 head_off
= head
% mm
->len
;
1623 old_off
= old
% mm
->len
;
1626 if (head_off
> old_off
)
1627 size
= head_off
- old_off
;
1629 size
= mm
->len
- (old_off
- head_off
);
1631 if (snapshot
&& size
> snapshot_size
)
1632 size
= snapshot_size
;
1634 ref
= auxtrace_record__reference(itr
);
1636 if (head
> old
|| size
<= head
|| mm
->mask
) {
1637 offset
= head
- size
;
1640 * When the buffer size is not a power of 2, 'head' wraps at the
1641 * highest multiple of the buffer size, so we have to subtract
1642 * the remainder here.
1644 u64 rem
= (0ULL - mm
->len
) % mm
->len
;
1646 offset
= head
- size
- rem
;
1649 if (size
> head_off
) {
1650 len1
= size
- head_off
;
1651 data1
= &data
[mm
->len
- len1
];
1656 data1
= &data
[head_off
- len1
];
1661 if (itr
->alignment
) {
1662 unsigned int unwanted
= len1
% itr
->alignment
;
1668 /* padding must be written by fn() e.g. record__process_auxtrace() */
1669 padding
= size
& (PERF_AUXTRACE_RECORD_ALIGNMENT
- 1);
1671 padding
= PERF_AUXTRACE_RECORD_ALIGNMENT
- padding
;
1673 memset(&ev
, 0, sizeof(ev
));
1674 ev
.auxtrace
.header
.type
= PERF_RECORD_AUXTRACE
;
1675 ev
.auxtrace
.header
.size
= sizeof(ev
.auxtrace
);
1676 ev
.auxtrace
.size
= size
+ padding
;
1677 ev
.auxtrace
.offset
= offset
;
1678 ev
.auxtrace
.reference
= ref
;
1679 ev
.auxtrace
.idx
= mm
->idx
;
1680 ev
.auxtrace
.tid
= mm
->tid
;
1681 ev
.auxtrace
.cpu
= mm
->cpu
;
1683 if (fn(tool
, map
, &ev
, data1
, len1
, data2
, len2
))
1689 auxtrace_mmap__write_tail(mm
, head
);
1690 if (itr
->read_finish
) {
1693 err
= itr
->read_finish(itr
, mm
->idx
);
1702 int auxtrace_mmap__read(struct mmap
*map
, struct auxtrace_record
*itr
,
1703 struct perf_tool
*tool
, process_auxtrace_t fn
)
1705 return __auxtrace_mmap__read(map
, itr
, tool
, fn
, false, 0);
1708 int auxtrace_mmap__read_snapshot(struct mmap
*map
,
1709 struct auxtrace_record
*itr
,
1710 struct perf_tool
*tool
, process_auxtrace_t fn
,
1711 size_t snapshot_size
)
1713 return __auxtrace_mmap__read(map
, itr
, tool
, fn
, true, snapshot_size
);
1717 * struct auxtrace_cache - hash table to implement a cache
1718 * @hashtable: the hashtable
1719 * @sz: hashtable size (number of hlists)
1720 * @entry_size: size of an entry
1721 * @limit: limit the number of entries to this maximum, when reached the cache
1722 * is dropped and caching begins again with an empty cache
1723 * @cnt: current number of entries
1724 * @bits: hashtable size (@sz = 2^@bits)
1726 struct auxtrace_cache
{
1727 struct hlist_head
*hashtable
;
1735 struct auxtrace_cache
*auxtrace_cache__new(unsigned int bits
, size_t entry_size
,
1736 unsigned int limit_percent
)
1738 struct auxtrace_cache
*c
;
1739 struct hlist_head
*ht
;
1742 c
= zalloc(sizeof(struct auxtrace_cache
));
1748 ht
= calloc(sz
, sizeof(struct hlist_head
));
1752 for (i
= 0; i
< sz
; i
++)
1753 INIT_HLIST_HEAD(&ht
[i
]);
1757 c
->entry_size
= entry_size
;
1758 c
->limit
= (c
->sz
* limit_percent
) / 100;
1768 static void auxtrace_cache__drop(struct auxtrace_cache
*c
)
1770 struct auxtrace_cache_entry
*entry
;
1771 struct hlist_node
*tmp
;
1777 for (i
= 0; i
< c
->sz
; i
++) {
1778 hlist_for_each_entry_safe(entry
, tmp
, &c
->hashtable
[i
], hash
) {
1779 hlist_del(&entry
->hash
);
1780 auxtrace_cache__free_entry(c
, entry
);
1787 void auxtrace_cache__free(struct auxtrace_cache
*c
)
1792 auxtrace_cache__drop(c
);
1793 zfree(&c
->hashtable
);
1797 void *auxtrace_cache__alloc_entry(struct auxtrace_cache
*c
)
1799 return malloc(c
->entry_size
);
1802 void auxtrace_cache__free_entry(struct auxtrace_cache
*c __maybe_unused
,
1808 int auxtrace_cache__add(struct auxtrace_cache
*c
, u32 key
,
1809 struct auxtrace_cache_entry
*entry
)
1811 if (c
->limit
&& ++c
->cnt
> c
->limit
)
1812 auxtrace_cache__drop(c
);
1815 hlist_add_head(&entry
->hash
, &c
->hashtable
[hash_32(key
, c
->bits
)]);
1820 static struct auxtrace_cache_entry
*auxtrace_cache__rm(struct auxtrace_cache
*c
,
1823 struct auxtrace_cache_entry
*entry
;
1824 struct hlist_head
*hlist
;
1825 struct hlist_node
*n
;
1830 hlist
= &c
->hashtable
[hash_32(key
, c
->bits
)];
1831 hlist_for_each_entry_safe(entry
, n
, hlist
, hash
) {
1832 if (entry
->key
== key
) {
1833 hlist_del(&entry
->hash
);
1841 void auxtrace_cache__remove(struct auxtrace_cache
*c
, u32 key
)
1843 struct auxtrace_cache_entry
*entry
= auxtrace_cache__rm(c
, key
);
1845 auxtrace_cache__free_entry(c
, entry
);
1848 void *auxtrace_cache__lookup(struct auxtrace_cache
*c
, u32 key
)
1850 struct auxtrace_cache_entry
*entry
;
1851 struct hlist_head
*hlist
;
1856 hlist
= &c
->hashtable
[hash_32(key
, c
->bits
)];
1857 hlist_for_each_entry(entry
, hlist
, hash
) {
1858 if (entry
->key
== key
)
1865 static void addr_filter__free_str(struct addr_filter
*filt
)
1868 filt
->action
= NULL
;
1869 filt
->sym_from
= NULL
;
1870 filt
->sym_to
= NULL
;
1871 filt
->filename
= NULL
;
1874 static struct addr_filter
*addr_filter__new(void)
1876 struct addr_filter
*filt
= zalloc(sizeof(*filt
));
1879 INIT_LIST_HEAD(&filt
->list
);
1884 static void addr_filter__free(struct addr_filter
*filt
)
1887 addr_filter__free_str(filt
);
1891 static void addr_filters__add(struct addr_filters
*filts
,
1892 struct addr_filter
*filt
)
1894 list_add_tail(&filt
->list
, &filts
->head
);
1898 static void addr_filters__del(struct addr_filters
*filts
,
1899 struct addr_filter
*filt
)
1901 list_del_init(&filt
->list
);
1905 void addr_filters__init(struct addr_filters
*filts
)
1907 INIT_LIST_HEAD(&filts
->head
);
1911 void addr_filters__exit(struct addr_filters
*filts
)
1913 struct addr_filter
*filt
, *n
;
1915 list_for_each_entry_safe(filt
, n
, &filts
->head
, list
) {
1916 addr_filters__del(filts
, filt
);
1917 addr_filter__free(filt
);
1921 static int parse_num_or_str(char **inp
, u64
*num
, const char **str
,
1922 const char *str_delim
)
1924 *inp
+= strspn(*inp
, " ");
1926 if (isdigit(**inp
)) {
1932 *num
= strtoull(*inp
, &endptr
, 0);
1943 *inp
+= strspn(*inp
, " ");
1945 n
= strcspn(*inp
, str_delim
);
1957 static int parse_action(struct addr_filter
*filt
)
1959 if (!strcmp(filt
->action
, "filter")) {
1962 } else if (!strcmp(filt
->action
, "start")) {
1964 } else if (!strcmp(filt
->action
, "stop")) {
1965 filt
->start
= false;
1966 } else if (!strcmp(filt
->action
, "tracestop")) {
1967 filt
->start
= false;
1969 filt
->action
+= 5; /* Change 'tracestop' to 'stop' */
1976 static int parse_sym_idx(char **inp
, int *idx
)
1980 *inp
+= strspn(*inp
, " ");
1987 if (**inp
== 'g' || **inp
== 'G') {
1995 num
= strtoul(*inp
, &endptr
, 0);
1998 if (endptr
== *inp
|| num
> INT_MAX
)
2007 static int parse_addr_size(char **inp
, u64
*num
, const char **str
, int *idx
)
2009 int err
= parse_num_or_str(inp
, num
, str
, " ");
2012 err
= parse_sym_idx(inp
, idx
);
2017 static int parse_one_filter(struct addr_filter
*filt
, const char **filter_inp
)
2022 filt
->str
= fstr
= strdup(*filter_inp
);
2026 err
= parse_num_or_str(&fstr
, NULL
, &filt
->action
, " ");
2030 err
= parse_action(filt
);
2034 err
= parse_addr_size(&fstr
, &filt
->addr
, &filt
->sym_from
,
2035 &filt
->sym_from_idx
);
2039 fstr
+= strspn(fstr
, " ");
2043 err
= parse_addr_size(&fstr
, &filt
->size
, &filt
->sym_to
,
2050 fstr
+= strspn(fstr
, " ");
2054 err
= parse_num_or_str(&fstr
, NULL
, &filt
->filename
, " ,");
2059 fstr
+= strspn(fstr
, " ,");
2061 *filter_inp
+= fstr
- filt
->str
;
2066 addr_filter__free_str(filt
);
2071 int addr_filters__parse_bare_filter(struct addr_filters
*filts
,
2074 struct addr_filter
*filt
;
2075 const char *fstr
= filter
;
2079 filt
= addr_filter__new();
2080 err
= parse_one_filter(filt
, &fstr
);
2082 addr_filter__free(filt
);
2083 addr_filters__exit(filts
);
2086 addr_filters__add(filts
, filt
);
2105 static bool kern_sym_match(struct sym_args
*args
, const char *name
, char type
)
2107 /* A function with the same name, and global or the n'th found or any */
2108 return kallsyms__is_function(type
) &&
2109 !strcmp(name
, args
->name
) &&
2110 ((args
->global
&& isupper(type
)) ||
2111 (args
->selected
&& ++(args
->cnt
) == args
->idx
) ||
2112 (!args
->global
&& !args
->selected
));
2115 static int find_kern_sym_cb(void *arg
, const char *name
, char type
, u64 start
)
2117 struct sym_args
*args
= arg
;
2119 if (args
->started
) {
2121 args
->size
= start
- args
->start
;
2122 if (args
->selected
) {
2125 } else if (kern_sym_match(args
, name
, type
)) {
2126 args
->duplicate
= true;
2129 } else if (kern_sym_match(args
, name
, type
)) {
2130 args
->started
= true;
2131 args
->start
= start
;
2137 static int print_kern_sym_cb(void *arg
, const char *name
, char type
, u64 start
)
2139 struct sym_args
*args
= arg
;
2141 if (kern_sym_match(args
, name
, type
)) {
2142 pr_err("#%d\t0x%"PRIx64
"\t%c\t%s\n",
2143 ++args
->cnt
, start
, type
, name
);
2145 } else if (args
->near
) {
2147 pr_err("\t\twhich is near\t\t%s\n", name
);
2153 static int sym_not_found_error(const char *sym_name
, int idx
)
2156 pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n",
2159 pr_err("Global symbol '%s' not found.\n", sym_name
);
2161 pr_err("Symbol '%s' not found.\n", sym_name
);
2163 pr_err("Note that symbols must be functions.\n");
2168 static int find_kern_sym(const char *sym_name
, u64
*start
, u64
*size
, int idx
)
2170 struct sym_args args
= {
2174 .selected
= idx
> 0,
2181 err
= kallsyms__parse("/proc/kallsyms", &args
, find_kern_sym_cb
);
2183 pr_err("Failed to parse /proc/kallsyms\n");
2187 if (args
.duplicate
) {
2188 pr_err("Multiple kernel symbols with name '%s'\n", sym_name
);
2190 kallsyms__parse("/proc/kallsyms", &args
, print_kern_sym_cb
);
2191 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
2193 pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
2197 if (!args
.started
) {
2198 pr_err("Kernel symbol lookup: ");
2199 return sym_not_found_error(sym_name
, idx
);
2202 *start
= args
.start
;
2208 static int find_entire_kern_cb(void *arg
, const char *name __maybe_unused
,
2209 char type
, u64 start
)
2211 struct sym_args
*args
= arg
;
2213 if (!kallsyms__is_function(type
))
2216 if (!args
->started
) {
2217 args
->started
= true;
2218 args
->start
= start
;
2220 /* Don't know exactly where the kernel ends, so we add a page */
2221 args
->size
= round_up(start
, page_size
) + page_size
- args
->start
;
2226 static int addr_filter__entire_kernel(struct addr_filter
*filt
)
2228 struct sym_args args
= { .started
= false };
2231 err
= kallsyms__parse("/proc/kallsyms", &args
, find_entire_kern_cb
);
2232 if (err
< 0 || !args
.started
) {
2233 pr_err("Failed to parse /proc/kallsyms\n");
2237 filt
->addr
= args
.start
;
2238 filt
->size
= args
.size
;
2243 static int check_end_after_start(struct addr_filter
*filt
, u64 start
, u64 size
)
2245 if (start
+ size
>= filt
->addr
)
2248 if (filt
->sym_from
) {
2249 pr_err("Symbol '%s' (0x%"PRIx64
") comes before '%s' (0x%"PRIx64
")\n",
2250 filt
->sym_to
, start
, filt
->sym_from
, filt
->addr
);
2252 pr_err("Symbol '%s' (0x%"PRIx64
") comes before address 0x%"PRIx64
")\n",
2253 filt
->sym_to
, start
, filt
->addr
);
2259 static int addr_filter__resolve_kernel_syms(struct addr_filter
*filt
)
2261 bool no_size
= false;
2265 if (symbol_conf
.kptr_restrict
) {
2266 pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n");
2270 if (filt
->sym_from
&& !strcmp(filt
->sym_from
, "*"))
2271 return addr_filter__entire_kernel(filt
);
2273 if (filt
->sym_from
) {
2274 err
= find_kern_sym(filt
->sym_from
, &start
, &size
,
2275 filt
->sym_from_idx
);
2279 if (filt
->range
&& !filt
->size
&& !filt
->sym_to
) {
2286 err
= find_kern_sym(filt
->sym_to
, &start
, &size
,
2291 err
= check_end_after_start(filt
, start
, size
);
2294 filt
->size
= start
+ size
- filt
->addr
;
2298 /* The very last symbol in kallsyms does not imply a particular size */
2300 pr_err("Cannot determine size of symbol '%s'\n",
2301 filt
->sym_to
? filt
->sym_to
: filt
->sym_from
);
2308 static struct dso
*load_dso(const char *name
)
2313 map
= dso__new_map(name
);
2317 if (map__load(map
) < 0)
2318 pr_err("File '%s' not found or has no symbols.\n", name
);
2320 dso
= dso__get(map
->dso
);
2327 static bool dso_sym_match(struct symbol
*sym
, const char *name
, int *cnt
,
2330 /* Same name, and global or the n'th found or any */
2331 return !arch__compare_symbol_names(name
, sym
->name
) &&
2332 ((!idx
&& sym
->binding
== STB_GLOBAL
) ||
2333 (idx
> 0 && ++*cnt
== idx
) ||
2337 static void print_duplicate_syms(struct dso
*dso
, const char *sym_name
)
2343 pr_err("Multiple symbols with name '%s'\n", sym_name
);
2345 sym
= dso__first_symbol(dso
);
2347 if (dso_sym_match(sym
, sym_name
, &cnt
, -1)) {
2348 pr_err("#%d\t0x%"PRIx64
"\t%c\t%s\n",
2350 sym
->binding
== STB_GLOBAL
? 'g' :
2351 sym
->binding
== STB_LOCAL
? 'l' : 'w',
2356 pr_err("\t\twhich is near\t\t%s\n", sym
->name
);
2358 sym
= dso__next_symbol(sym
);
2361 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
2363 pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
2366 static int find_dso_sym(struct dso
*dso
, const char *sym_name
, u64
*start
,
2375 sym
= dso__first_symbol(dso
);
2379 *size
= sym
->start
- *start
;
2383 } else if (dso_sym_match(sym
, sym_name
, &cnt
, idx
)) {
2384 print_duplicate_syms(dso
, sym_name
);
2387 } else if (dso_sym_match(sym
, sym_name
, &cnt
, idx
)) {
2388 *start
= sym
->start
;
2389 *size
= sym
->end
- sym
->start
;
2391 sym
= dso__next_symbol(sym
);
2395 return sym_not_found_error(sym_name
, idx
);
2400 static int addr_filter__entire_dso(struct addr_filter
*filt
, struct dso
*dso
)
2402 if (dso__data_file_size(dso
, NULL
)) {
2403 pr_err("Failed to determine filter for %s\nCannot determine file size.\n",
2409 filt
->size
= dso
->data
.file_size
;
2414 static int addr_filter__resolve_syms(struct addr_filter
*filt
)
2420 if (!filt
->sym_from
&& !filt
->sym_to
)
2423 if (!filt
->filename
)
2424 return addr_filter__resolve_kernel_syms(filt
);
2426 dso
= load_dso(filt
->filename
);
2428 pr_err("Failed to load symbols from: %s\n", filt
->filename
);
2432 if (filt
->sym_from
&& !strcmp(filt
->sym_from
, "*")) {
2433 err
= addr_filter__entire_dso(filt
, dso
);
2437 if (filt
->sym_from
) {
2438 err
= find_dso_sym(dso
, filt
->sym_from
, &start
, &size
,
2439 filt
->sym_from_idx
);
2443 if (filt
->range
&& !filt
->size
&& !filt
->sym_to
)
2448 err
= find_dso_sym(dso
, filt
->sym_to
, &start
, &size
,
2453 err
= check_end_after_start(filt
, start
, size
);
2457 filt
->size
= start
+ size
- filt
->addr
;
2466 static char *addr_filter__to_str(struct addr_filter
*filt
)
2468 char filename_buf
[PATH_MAX
];
2469 const char *at
= "";
2470 const char *fn
= "";
2474 if (filt
->filename
) {
2476 fn
= realpath(filt
->filename
, filename_buf
);
2482 err
= asprintf(&filter
, "%s 0x%"PRIx64
"/0x%"PRIx64
"%s%s",
2483 filt
->action
, filt
->addr
, filt
->size
, at
, fn
);
2485 err
= asprintf(&filter
, "%s 0x%"PRIx64
"%s%s",
2486 filt
->action
, filt
->addr
, at
, fn
);
2489 return err
< 0 ? NULL
: filter
;
2492 static int parse_addr_filter(struct evsel
*evsel
, const char *filter
,
2495 struct addr_filters filts
;
2496 struct addr_filter
*filt
;
2499 addr_filters__init(&filts
);
2501 err
= addr_filters__parse_bare_filter(&filts
, filter
);
2505 if (filts
.cnt
> max_nr
) {
2506 pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n",
2512 list_for_each_entry(filt
, &filts
.head
, list
) {
2515 err
= addr_filter__resolve_syms(filt
);
2519 new_filter
= addr_filter__to_str(filt
);
2525 if (evsel__append_addr_filter(evsel
, new_filter
)) {
2532 addr_filters__exit(&filts
);
2535 pr_err("Failed to parse address filter: '%s'\n", filter
);
2536 pr_err("Filter format is: filter|start|stop|tracestop <start symbol or address> [/ <end symbol or size>] [@<file name>]\n");
2537 pr_err("Where multiple filters are separated by space or comma.\n");
2543 static int evsel__nr_addr_filter(struct evsel
*evsel
)
2545 struct perf_pmu
*pmu
= evsel__find_pmu(evsel
);
2546 int nr_addr_filters
= 0;
2551 perf_pmu__scan_file(pmu
, "nr_addr_filters", "%d", &nr_addr_filters
);
2553 return nr_addr_filters
;
2556 int auxtrace_parse_filters(struct evlist
*evlist
)
2558 struct evsel
*evsel
;
2562 evlist__for_each_entry(evlist
, evsel
) {
2563 filter
= evsel
->filter
;
2564 max_nr
= evsel__nr_addr_filter(evsel
);
2565 if (!filter
|| !max_nr
)
2567 evsel
->filter
= NULL
;
2568 err
= parse_addr_filter(evsel
, filter
, max_nr
);
2572 pr_debug("Address filter: %s\n", evsel
->filter
);
2578 int auxtrace__process_event(struct perf_session
*session
, union perf_event
*event
,
2579 struct perf_sample
*sample
, struct perf_tool
*tool
)
2581 if (!session
->auxtrace
)
2584 return session
->auxtrace
->process_event(session
, event
, sample
, tool
);
2587 void auxtrace__dump_auxtrace_sample(struct perf_session
*session
,
2588 struct perf_sample
*sample
)
2590 if (!session
->auxtrace
|| !session
->auxtrace
->dump_auxtrace_sample
||
2591 auxtrace__dont_decode(session
))
2594 session
->auxtrace
->dump_auxtrace_sample(session
, sample
);
2597 int auxtrace__flush_events(struct perf_session
*session
, struct perf_tool
*tool
)
2599 if (!session
->auxtrace
)
2602 return session
->auxtrace
->flush_events(session
, tool
);
2605 void auxtrace__free_events(struct perf_session
*session
)
2607 if (!session
->auxtrace
)
2610 return session
->auxtrace
->free_events(session
);
2613 void auxtrace__free(struct perf_session
*session
)
2615 if (!session
->auxtrace
)
2618 return session
->auxtrace
->free(session
);
2621 bool auxtrace__evsel_is_auxtrace(struct perf_session
*session
,
2622 struct evsel
*evsel
)
2624 if (!session
->auxtrace
|| !session
->auxtrace
->evsel_is_auxtrace
)
2627 return session
->auxtrace
->evsel_is_auxtrace(session
, evsel
);