1 // SPDX-License-Identifier: GPL-2.0-only
3 * intel_pt.c: Intel Processor Trace support
4 * Copyright (c) 2013-2015, Intel Corporation.
9 #include <linux/kernel.h>
10 #include <linux/types.h>
11 #include <linux/bitops.h>
12 #include <linux/log2.h>
13 #include <linux/zalloc.h>
16 #include "../../../util/session.h"
17 #include "../../../util/event.h"
18 #include "../../../util/evlist.h"
19 #include "../../../util/evsel.h"
20 #include "../../../util/evsel_config.h"
21 #include "../../../util/cpumap.h"
22 #include "../../../util/mmap.h"
23 #include <subcmd/parse-options.h>
24 #include "../../../util/parse-events.h"
25 #include "../../../util/pmu.h"
26 #include "../../../util/debug.h"
27 #include "../../../util/auxtrace.h"
28 #include "../../../util/perf_api_probe.h"
29 #include "../../../util/record.h"
30 #include "../../../util/target.h"
31 #include "../../../util/tsc.h"
32 #include <internal/lib.h> // page_size
33 #include "../../../util/intel-pt.h"
35 #define KiB(x) ((x) * 1024)
36 #define MiB(x) ((x) * 1024 * 1024)
37 #define KiB_MASK(x) (KiB(x) - 1)
38 #define MiB_MASK(x) (MiB(x) - 1)
40 #define INTEL_PT_PSB_PERIOD_NEAR 256
42 struct intel_pt_snapshot_ref
{
48 struct intel_pt_recording
{
49 struct auxtrace_record itr
;
50 struct perf_pmu
*intel_pt_pmu
;
51 int have_sched_switch
;
52 struct evlist
*evlist
;
54 bool snapshot_init_done
;
56 size_t snapshot_ref_buf_size
;
58 struct intel_pt_snapshot_ref
*snapshot_refs
;
62 static int intel_pt_parse_terms_with_default(struct list_head
*formats
,
66 struct list_head
*terms
;
67 struct perf_event_attr attr
= { .size
= 0, };
70 terms
= malloc(sizeof(struct list_head
));
74 INIT_LIST_HEAD(terms
);
76 err
= parse_events_terms(terms
, str
);
80 attr
.config
= *config
;
81 err
= perf_pmu__config_terms(formats
, &attr
, terms
, true, NULL
);
85 *config
= attr
.config
;
87 parse_events_terms__delete(terms
);
91 static int intel_pt_parse_terms(struct list_head
*formats
, const char *str
,
95 return intel_pt_parse_terms_with_default(formats
, str
, config
);
98 static u64
intel_pt_masked_bits(u64 mask
, u64 bits
)
100 const u64 top_bit
= 1ULL << 63;
104 for (i
= 0; i
< 64; i
++) {
105 if (mask
& top_bit
) {
117 static int intel_pt_read_config(struct perf_pmu
*intel_pt_pmu
, const char *str
,
118 struct evlist
*evlist
, u64
*res
)
125 mask
= perf_pmu__format_bits(&intel_pt_pmu
->format
, str
);
129 evlist__for_each_entry(evlist
, evsel
) {
130 if (evsel
->core
.attr
.type
== intel_pt_pmu
->type
) {
131 *res
= intel_pt_masked_bits(mask
, evsel
->core
.attr
.config
);
139 static size_t intel_pt_psb_period(struct perf_pmu
*intel_pt_pmu
,
140 struct evlist
*evlist
)
143 int err
, topa_multiple_entries
;
146 if (perf_pmu__scan_file(intel_pt_pmu
, "caps/topa_multiple_entries",
147 "%d", &topa_multiple_entries
) != 1)
148 topa_multiple_entries
= 0;
151 * Use caps/topa_multiple_entries to indicate early hardware that had
152 * extra frequent PSBs.
154 if (!topa_multiple_entries
) {
159 err
= intel_pt_read_config(intel_pt_pmu
, "psb_period", evlist
, &val
);
163 psb_period
= 1 << (val
+ 11);
165 pr_debug2("%s psb_period %zu\n", intel_pt_pmu
->name
, psb_period
);
169 static int intel_pt_pick_bit(int bits
, int target
)
173 for (pos
= 0; bits
; bits
>>= 1, pos
++) {
175 if (pos
<= target
|| pick
< 0)
185 static u64
intel_pt_default_config(struct perf_pmu
*intel_pt_pmu
)
188 int mtc
, mtc_periods
= 0, mtc_period
;
189 int psb_cyc
, psb_periods
, psb_period
;
194 pos
+= scnprintf(buf
+ pos
, sizeof(buf
) - pos
, "tsc");
196 if (perf_pmu__scan_file(intel_pt_pmu
, "caps/mtc", "%d",
201 if (perf_pmu__scan_file(intel_pt_pmu
, "caps/mtc_periods", "%x",
205 mtc_period
= intel_pt_pick_bit(mtc_periods
, 3);
206 pos
+= scnprintf(buf
+ pos
, sizeof(buf
) - pos
,
207 ",mtc,mtc_period=%d", mtc_period
);
211 if (perf_pmu__scan_file(intel_pt_pmu
, "caps/psb_cyc", "%d",
215 if (psb_cyc
&& mtc_periods
) {
216 if (perf_pmu__scan_file(intel_pt_pmu
, "caps/psb_periods", "%x",
220 psb_period
= intel_pt_pick_bit(psb_periods
, 3);
221 pos
+= scnprintf(buf
+ pos
, sizeof(buf
) - pos
,
222 ",psb_period=%d", psb_period
);
226 if (perf_pmu__scan_file(intel_pt_pmu
, "format/pt", "%c", &c
) == 1 &&
227 perf_pmu__scan_file(intel_pt_pmu
, "format/branch", "%c", &c
) == 1)
228 pos
+= scnprintf(buf
+ pos
, sizeof(buf
) - pos
, ",pt,branch");
230 pr_debug2("%s default config: %s\n", intel_pt_pmu
->name
, buf
);
232 intel_pt_parse_terms(&intel_pt_pmu
->format
, buf
, &config
);
237 static int intel_pt_parse_snapshot_options(struct auxtrace_record
*itr
,
238 struct record_opts
*opts
,
241 struct intel_pt_recording
*ptr
=
242 container_of(itr
, struct intel_pt_recording
, itr
);
243 unsigned long long snapshot_size
= 0;
247 snapshot_size
= strtoull(str
, &endptr
, 0);
248 if (*endptr
|| snapshot_size
> SIZE_MAX
)
252 opts
->auxtrace_snapshot_mode
= true;
253 opts
->auxtrace_snapshot_size
= snapshot_size
;
255 ptr
->snapshot_size
= snapshot_size
;
260 struct perf_event_attr
*
261 intel_pt_pmu_default_config(struct perf_pmu
*intel_pt_pmu
)
263 struct perf_event_attr
*attr
;
265 attr
= zalloc(sizeof(struct perf_event_attr
));
269 attr
->config
= intel_pt_default_config(intel_pt_pmu
);
271 intel_pt_pmu
->selectable
= true;
276 static const char *intel_pt_find_filter(struct evlist
*evlist
,
277 struct perf_pmu
*intel_pt_pmu
)
281 evlist__for_each_entry(evlist
, evsel
) {
282 if (evsel
->core
.attr
.type
== intel_pt_pmu
->type
)
283 return evsel
->filter
;
289 static size_t intel_pt_filter_bytes(const char *filter
)
291 size_t len
= filter
? strlen(filter
) : 0;
293 return len
? roundup(len
+ 1, 8) : 0;
297 intel_pt_info_priv_size(struct auxtrace_record
*itr
, struct evlist
*evlist
)
299 struct intel_pt_recording
*ptr
=
300 container_of(itr
, struct intel_pt_recording
, itr
);
301 const char *filter
= intel_pt_find_filter(evlist
, ptr
->intel_pt_pmu
);
303 ptr
->priv_size
= (INTEL_PT_AUXTRACE_PRIV_MAX
* sizeof(u64
)) +
304 intel_pt_filter_bytes(filter
);
306 return ptr
->priv_size
;
309 static void intel_pt_tsc_ctc_ratio(u32
*n
, u32
*d
)
311 unsigned int eax
= 0, ebx
= 0, ecx
= 0, edx
= 0;
313 __get_cpuid(0x15, &eax
, &ebx
, &ecx
, &edx
);
318 static int intel_pt_info_fill(struct auxtrace_record
*itr
,
319 struct perf_session
*session
,
320 struct perf_record_auxtrace_info
*auxtrace_info
,
323 struct intel_pt_recording
*ptr
=
324 container_of(itr
, struct intel_pt_recording
, itr
);
325 struct perf_pmu
*intel_pt_pmu
= ptr
->intel_pt_pmu
;
326 struct perf_event_mmap_page
*pc
;
327 struct perf_tsc_conversion tc
= { .time_mult
= 0, };
328 bool cap_user_time_zero
= false, per_cpu_mmaps
;
329 u64 tsc_bit
, mtc_bit
, mtc_freq_bits
, cyc_bit
, noretcomp_bit
;
330 u32 tsc_ctc_ratio_n
, tsc_ctc_ratio_d
;
331 unsigned long max_non_turbo_ratio
;
332 size_t filter_str_len
;
337 if (priv_size
!= ptr
->priv_size
)
340 intel_pt_parse_terms(&intel_pt_pmu
->format
, "tsc", &tsc_bit
);
341 intel_pt_parse_terms(&intel_pt_pmu
->format
, "noretcomp",
343 intel_pt_parse_terms(&intel_pt_pmu
->format
, "mtc", &mtc_bit
);
344 mtc_freq_bits
= perf_pmu__format_bits(&intel_pt_pmu
->format
,
346 intel_pt_parse_terms(&intel_pt_pmu
->format
, "cyc", &cyc_bit
);
348 intel_pt_tsc_ctc_ratio(&tsc_ctc_ratio_n
, &tsc_ctc_ratio_d
);
350 if (perf_pmu__scan_file(intel_pt_pmu
, "max_nonturbo_ratio",
351 "%lu", &max_non_turbo_ratio
) != 1)
352 max_non_turbo_ratio
= 0;
354 filter
= intel_pt_find_filter(session
->evlist
, ptr
->intel_pt_pmu
);
355 filter_str_len
= filter
? strlen(filter
) : 0;
357 if (!session
->evlist
->core
.nr_mmaps
)
360 pc
= session
->evlist
->mmap
[0].core
.base
;
362 err
= perf_read_tsc_conversion(pc
, &tc
);
364 if (err
!= -EOPNOTSUPP
)
367 cap_user_time_zero
= tc
.time_mult
!= 0;
369 if (!cap_user_time_zero
)
370 ui__warning("Intel Processor Trace: TSC not available\n");
373 per_cpu_mmaps
= !perf_cpu_map__empty(session
->evlist
->core
.cpus
);
375 auxtrace_info
->type
= PERF_AUXTRACE_INTEL_PT
;
376 auxtrace_info
->priv
[INTEL_PT_PMU_TYPE
] = intel_pt_pmu
->type
;
377 auxtrace_info
->priv
[INTEL_PT_TIME_SHIFT
] = tc
.time_shift
;
378 auxtrace_info
->priv
[INTEL_PT_TIME_MULT
] = tc
.time_mult
;
379 auxtrace_info
->priv
[INTEL_PT_TIME_ZERO
] = tc
.time_zero
;
380 auxtrace_info
->priv
[INTEL_PT_CAP_USER_TIME_ZERO
] = cap_user_time_zero
;
381 auxtrace_info
->priv
[INTEL_PT_TSC_BIT
] = tsc_bit
;
382 auxtrace_info
->priv
[INTEL_PT_NORETCOMP_BIT
] = noretcomp_bit
;
383 auxtrace_info
->priv
[INTEL_PT_HAVE_SCHED_SWITCH
] = ptr
->have_sched_switch
;
384 auxtrace_info
->priv
[INTEL_PT_SNAPSHOT_MODE
] = ptr
->snapshot_mode
;
385 auxtrace_info
->priv
[INTEL_PT_PER_CPU_MMAPS
] = per_cpu_mmaps
;
386 auxtrace_info
->priv
[INTEL_PT_MTC_BIT
] = mtc_bit
;
387 auxtrace_info
->priv
[INTEL_PT_MTC_FREQ_BITS
] = mtc_freq_bits
;
388 auxtrace_info
->priv
[INTEL_PT_TSC_CTC_N
] = tsc_ctc_ratio_n
;
389 auxtrace_info
->priv
[INTEL_PT_TSC_CTC_D
] = tsc_ctc_ratio_d
;
390 auxtrace_info
->priv
[INTEL_PT_CYC_BIT
] = cyc_bit
;
391 auxtrace_info
->priv
[INTEL_PT_MAX_NONTURBO_RATIO
] = max_non_turbo_ratio
;
392 auxtrace_info
->priv
[INTEL_PT_FILTER_STR_LEN
] = filter_str_len
;
394 info
= &auxtrace_info
->priv
[INTEL_PT_FILTER_STR_LEN
] + 1;
396 if (filter_str_len
) {
397 size_t len
= intel_pt_filter_bytes(filter
);
399 strncpy((char *)info
, filter
, len
);
406 static int intel_pt_track_switches(struct evlist
*evlist
)
408 const char *sched_switch
= "sched:sched_switch";
412 if (!perf_evlist__can_select_event(evlist
, sched_switch
))
415 err
= parse_events(evlist
, sched_switch
, NULL
);
417 pr_debug2("%s: failed to parse %s, error %d\n",
418 __func__
, sched_switch
, err
);
422 evsel
= evlist__last(evlist
);
424 evsel__set_sample_bit(evsel
, CPU
);
425 evsel__set_sample_bit(evsel
, TIME
);
427 evsel
->core
.system_wide
= true;
428 evsel
->no_aux_samples
= true;
429 evsel
->immediate
= true;
434 static void intel_pt_valid_str(char *str
, size_t len
, u64 valid
)
436 unsigned int val
, last
= 0, state
= 1;
441 for (val
= 0; val
<= 64; val
++, valid
>>= 1) {
446 p
+= scnprintf(str
+ p
, len
- p
, ",");
449 p
+= scnprintf(str
+ p
, len
- p
, "%u", val
);
464 p
+= scnprintf(str
+ p
, len
- p
, ",%u", last
);
468 p
+= scnprintf(str
+ p
, len
- p
, "-%u", last
);
480 static int intel_pt_val_config_term(struct perf_pmu
*intel_pt_pmu
,
481 const char *caps
, const char *name
,
482 const char *supported
, u64 config
)
486 unsigned long long valid
;
490 if (perf_pmu__scan_file(intel_pt_pmu
, caps
, "%llx", &valid
) != 1)
494 perf_pmu__scan_file(intel_pt_pmu
, supported
, "%d", &ok
) == 1 && !ok
)
499 bits
= perf_pmu__format_bits(&intel_pt_pmu
->format
, name
);
503 for (shift
= 0; bits
&& !(bits
& 1); shift
++)
511 if (valid
& (1 << config
))
514 intel_pt_valid_str(valid_str
, sizeof(valid_str
), valid
);
515 pr_err("Invalid %s for %s. Valid values are: %s\n",
516 name
, INTEL_PT_PMU_NAME
, valid_str
);
520 static int intel_pt_validate_config(struct perf_pmu
*intel_pt_pmu
,
530 * If supported, force pass-through config term (pt=1) even if user
531 * sets pt=0, which avoids senseless kernel errors.
533 if (perf_pmu__scan_file(intel_pt_pmu
, "format/pt", "%c", &c
) == 1 &&
534 !(evsel
->core
.attr
.config
& 1)) {
535 pr_warning("pt=0 doesn't make sense, forcing pt=1\n");
536 evsel
->core
.attr
.config
|= 1;
539 err
= intel_pt_val_config_term(intel_pt_pmu
, "caps/cycle_thresholds",
540 "cyc_thresh", "caps/psb_cyc",
541 evsel
->core
.attr
.config
);
545 err
= intel_pt_val_config_term(intel_pt_pmu
, "caps/mtc_periods",
546 "mtc_period", "caps/mtc",
547 evsel
->core
.attr
.config
);
551 return intel_pt_val_config_term(intel_pt_pmu
, "caps/psb_periods",
552 "psb_period", "caps/psb_cyc",
553 evsel
->core
.attr
.config
);
556 static void intel_pt_config_sample_mode(struct perf_pmu
*intel_pt_pmu
,
559 struct perf_evsel_config_term
*term
;
560 u64 user_bits
= 0, bits
;
562 term
= perf_evsel__get_config_term(evsel
, CFG_CHG
);
564 user_bits
= term
->val
.cfg_chg
;
566 bits
= perf_pmu__format_bits(&intel_pt_pmu
->format
, "psb_period");
568 /* Did user change psb_period */
569 if (bits
& user_bits
)
572 /* Set psb_period to 0 */
573 evsel
->core
.attr
.config
&= ~bits
;
576 static void intel_pt_min_max_sample_sz(struct evlist
*evlist
,
577 size_t *min_sz
, size_t *max_sz
)
581 evlist__for_each_entry(evlist
, evsel
) {
582 size_t sz
= evsel
->core
.attr
.aux_sample_size
;
586 if (min_sz
&& (sz
< *min_sz
|| !*min_sz
))
588 if (max_sz
&& sz
> *max_sz
)
594 * Currently, there is not enough information to disambiguate different PEBS
595 * events, so only allow one.
597 static bool intel_pt_too_many_aux_output(struct evlist
*evlist
)
600 int aux_output_cnt
= 0;
602 evlist__for_each_entry(evlist
, evsel
)
603 aux_output_cnt
+= !!evsel
->core
.attr
.aux_output
;
605 if (aux_output_cnt
> 1) {
606 pr_err(INTEL_PT_PMU_NAME
" supports at most one event with aux-output\n");
613 static int intel_pt_recording_options(struct auxtrace_record
*itr
,
614 struct evlist
*evlist
,
615 struct record_opts
*opts
)
617 struct intel_pt_recording
*ptr
=
618 container_of(itr
, struct intel_pt_recording
, itr
);
619 struct perf_pmu
*intel_pt_pmu
= ptr
->intel_pt_pmu
;
620 bool have_timing_info
, need_immediate
= false;
621 struct evsel
*evsel
, *intel_pt_evsel
= NULL
;
622 const struct perf_cpu_map
*cpus
= evlist
->core
.cpus
;
623 bool privileged
= perf_event_paranoid_check(-1);
627 ptr
->evlist
= evlist
;
628 ptr
->snapshot_mode
= opts
->auxtrace_snapshot_mode
;
630 evlist__for_each_entry(evlist
, evsel
) {
631 if (evsel
->core
.attr
.type
== intel_pt_pmu
->type
) {
632 if (intel_pt_evsel
) {
633 pr_err("There may be only one " INTEL_PT_PMU_NAME
" event\n");
636 evsel
->core
.attr
.freq
= 0;
637 evsel
->core
.attr
.sample_period
= 1;
638 intel_pt_evsel
= evsel
;
639 opts
->full_auxtrace
= true;
643 if (opts
->auxtrace_snapshot_mode
&& !opts
->full_auxtrace
) {
644 pr_err("Snapshot mode (-S option) requires " INTEL_PT_PMU_NAME
" PMU event (-e " INTEL_PT_PMU_NAME
")\n");
648 if (opts
->auxtrace_snapshot_mode
&& opts
->auxtrace_sample_mode
) {
649 pr_err("Snapshot mode (" INTEL_PT_PMU_NAME
" PMU) and sample trace cannot be used together\n");
653 if (opts
->use_clockid
) {
654 pr_err("Cannot use clockid (-k option) with " INTEL_PT_PMU_NAME
"\n");
658 if (intel_pt_too_many_aux_output(evlist
))
661 if (!opts
->full_auxtrace
)
664 if (opts
->auxtrace_sample_mode
)
665 intel_pt_config_sample_mode(intel_pt_pmu
, intel_pt_evsel
);
667 err
= intel_pt_validate_config(intel_pt_pmu
, intel_pt_evsel
);
671 /* Set default sizes for snapshot mode */
672 if (opts
->auxtrace_snapshot_mode
) {
673 size_t psb_period
= intel_pt_psb_period(intel_pt_pmu
, evlist
);
675 if (!opts
->auxtrace_snapshot_size
&& !opts
->auxtrace_mmap_pages
) {
677 opts
->auxtrace_mmap_pages
= MiB(4) / page_size
;
679 opts
->auxtrace_mmap_pages
= KiB(128) / page_size
;
680 if (opts
->mmap_pages
== UINT_MAX
)
681 opts
->mmap_pages
= KiB(256) / page_size
;
683 } else if (!opts
->auxtrace_mmap_pages
&& !privileged
&&
684 opts
->mmap_pages
== UINT_MAX
) {
685 opts
->mmap_pages
= KiB(256) / page_size
;
687 if (!opts
->auxtrace_snapshot_size
)
688 opts
->auxtrace_snapshot_size
=
689 opts
->auxtrace_mmap_pages
* (size_t)page_size
;
690 if (!opts
->auxtrace_mmap_pages
) {
691 size_t sz
= opts
->auxtrace_snapshot_size
;
693 sz
= round_up(sz
, page_size
) / page_size
;
694 opts
->auxtrace_mmap_pages
= roundup_pow_of_two(sz
);
696 if (opts
->auxtrace_snapshot_size
>
697 opts
->auxtrace_mmap_pages
* (size_t)page_size
) {
698 pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
699 opts
->auxtrace_snapshot_size
,
700 opts
->auxtrace_mmap_pages
* (size_t)page_size
);
703 if (!opts
->auxtrace_snapshot_size
|| !opts
->auxtrace_mmap_pages
) {
704 pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
707 pr_debug2("Intel PT snapshot size: %zu\n",
708 opts
->auxtrace_snapshot_size
);
710 opts
->auxtrace_snapshot_size
<= psb_period
+
711 INTEL_PT_PSB_PERIOD_NEAR
)
712 ui__warning("Intel PT snapshot size (%zu) may be too small for PSB period (%zu)\n",
713 opts
->auxtrace_snapshot_size
, psb_period
);
716 /* Set default sizes for sample mode */
717 if (opts
->auxtrace_sample_mode
) {
718 size_t psb_period
= intel_pt_psb_period(intel_pt_pmu
, evlist
);
719 size_t min_sz
= 0, max_sz
= 0;
721 intel_pt_min_max_sample_sz(evlist
, &min_sz
, &max_sz
);
722 if (!opts
->auxtrace_mmap_pages
&& !privileged
&&
723 opts
->mmap_pages
== UINT_MAX
)
724 opts
->mmap_pages
= KiB(256) / page_size
;
725 if (!opts
->auxtrace_mmap_pages
) {
726 size_t sz
= round_up(max_sz
, page_size
) / page_size
;
728 opts
->auxtrace_mmap_pages
= roundup_pow_of_two(sz
);
730 if (max_sz
> opts
->auxtrace_mmap_pages
* (size_t)page_size
) {
731 pr_err("Sample size %zu must not be greater than AUX area tracing mmap size %zu\n",
733 opts
->auxtrace_mmap_pages
* (size_t)page_size
);
736 pr_debug2("Intel PT min. sample size: %zu max. sample size: %zu\n",
739 min_sz
<= psb_period
+ INTEL_PT_PSB_PERIOD_NEAR
)
740 ui__warning("Intel PT sample size (%zu) may be too small for PSB period (%zu)\n",
744 /* Set default sizes for full trace mode */
745 if (opts
->full_auxtrace
&& !opts
->auxtrace_mmap_pages
) {
747 opts
->auxtrace_mmap_pages
= MiB(4) / page_size
;
749 opts
->auxtrace_mmap_pages
= KiB(128) / page_size
;
750 if (opts
->mmap_pages
== UINT_MAX
)
751 opts
->mmap_pages
= KiB(256) / page_size
;
755 /* Validate auxtrace_mmap_pages */
756 if (opts
->auxtrace_mmap_pages
) {
757 size_t sz
= opts
->auxtrace_mmap_pages
* (size_t)page_size
;
760 if (opts
->auxtrace_snapshot_mode
|| opts
->auxtrace_sample_mode
)
765 if (sz
< min_sz
|| !is_power_of_2(sz
)) {
766 pr_err("Invalid mmap size for Intel Processor Trace: must be at least %zuKiB and a power of 2\n",
772 intel_pt_parse_terms(&intel_pt_pmu
->format
, "tsc", &tsc_bit
);
774 if (opts
->full_auxtrace
&& (intel_pt_evsel
->core
.attr
.config
& tsc_bit
))
775 have_timing_info
= true;
777 have_timing_info
= false;
780 * Per-cpu recording needs sched_switch events to distinguish different
783 if (have_timing_info
&& !perf_cpu_map__empty(cpus
)) {
784 if (perf_can_record_switch_events()) {
785 bool cpu_wide
= !target__none(&opts
->target
) &&
786 !target__has_task(&opts
->target
);
788 if (!cpu_wide
&& perf_can_record_cpu_wide()) {
789 struct evsel
*switch_evsel
;
791 err
= parse_events(evlist
, "dummy:u", NULL
);
795 switch_evsel
= evlist__last(evlist
);
797 switch_evsel
->core
.attr
.freq
= 0;
798 switch_evsel
->core
.attr
.sample_period
= 1;
799 switch_evsel
->core
.attr
.context_switch
= 1;
801 switch_evsel
->core
.system_wide
= true;
802 switch_evsel
->no_aux_samples
= true;
803 switch_evsel
->immediate
= true;
805 evsel__set_sample_bit(switch_evsel
, TID
);
806 evsel__set_sample_bit(switch_evsel
, TIME
);
807 evsel__set_sample_bit(switch_evsel
, CPU
);
808 evsel__reset_sample_bit(switch_evsel
, BRANCH_STACK
);
810 opts
->record_switch_events
= false;
811 ptr
->have_sched_switch
= 3;
813 opts
->record_switch_events
= true;
814 need_immediate
= true;
816 ptr
->have_sched_switch
= 3;
818 ptr
->have_sched_switch
= 2;
821 err
= intel_pt_track_switches(evlist
);
823 pr_debug2("Unable to select sched:sched_switch\n");
827 ptr
->have_sched_switch
= 1;
831 if (intel_pt_evsel
) {
833 * To obtain the auxtrace buffer file descriptor, the auxtrace
834 * event must come first.
836 perf_evlist__to_front(evlist
, intel_pt_evsel
);
838 * In the case of per-cpu mmaps, we need the CPU on the
841 if (!perf_cpu_map__empty(cpus
))
842 evsel__set_sample_bit(intel_pt_evsel
, CPU
);
845 /* Add dummy event to keep tracking */
846 if (opts
->full_auxtrace
) {
847 struct evsel
*tracking_evsel
;
849 err
= parse_events(evlist
, "dummy:u", NULL
);
853 tracking_evsel
= evlist__last(evlist
);
855 perf_evlist__set_tracking_event(evlist
, tracking_evsel
);
857 tracking_evsel
->core
.attr
.freq
= 0;
858 tracking_evsel
->core
.attr
.sample_period
= 1;
860 tracking_evsel
->no_aux_samples
= true;
862 tracking_evsel
->immediate
= true;
864 /* In per-cpu case, always need the time of mmap events etc */
865 if (!perf_cpu_map__empty(cpus
)) {
866 evsel__set_sample_bit(tracking_evsel
, TIME
);
867 /* And the CPU for switch events */
868 evsel__set_sample_bit(tracking_evsel
, CPU
);
870 evsel__reset_sample_bit(tracking_evsel
, BRANCH_STACK
);
874 * Warn the user when we do not have enough information to decode i.e.
875 * per-cpu with no sched_switch (except workload-only).
877 if (!ptr
->have_sched_switch
&& !perf_cpu_map__empty(cpus
) &&
878 !target__none(&opts
->target
))
879 ui__warning("Intel Processor Trace decoding will not be possible except for kernel tracing!\n");
884 static int intel_pt_snapshot_start(struct auxtrace_record
*itr
)
886 struct intel_pt_recording
*ptr
=
887 container_of(itr
, struct intel_pt_recording
, itr
);
890 evlist__for_each_entry(ptr
->evlist
, evsel
) {
891 if (evsel
->core
.attr
.type
== ptr
->intel_pt_pmu
->type
)
892 return evsel__disable(evsel
);
897 static int intel_pt_snapshot_finish(struct auxtrace_record
*itr
)
899 struct intel_pt_recording
*ptr
=
900 container_of(itr
, struct intel_pt_recording
, itr
);
903 evlist__for_each_entry(ptr
->evlist
, evsel
) {
904 if (evsel
->core
.attr
.type
== ptr
->intel_pt_pmu
->type
)
905 return evsel__enable(evsel
);
910 static int intel_pt_alloc_snapshot_refs(struct intel_pt_recording
*ptr
, int idx
)
912 const size_t sz
= sizeof(struct intel_pt_snapshot_ref
);
913 int cnt
= ptr
->snapshot_ref_cnt
, new_cnt
= cnt
* 2;
914 struct intel_pt_snapshot_ref
*refs
;
919 while (new_cnt
<= idx
)
922 refs
= calloc(new_cnt
, sz
);
926 memcpy(refs
, ptr
->snapshot_refs
, cnt
* sz
);
928 ptr
->snapshot_refs
= refs
;
929 ptr
->snapshot_ref_cnt
= new_cnt
;
934 static void intel_pt_free_snapshot_refs(struct intel_pt_recording
*ptr
)
938 for (i
= 0; i
< ptr
->snapshot_ref_cnt
; i
++)
939 zfree(&ptr
->snapshot_refs
[i
].ref_buf
);
940 zfree(&ptr
->snapshot_refs
);
943 static void intel_pt_recording_free(struct auxtrace_record
*itr
)
945 struct intel_pt_recording
*ptr
=
946 container_of(itr
, struct intel_pt_recording
, itr
);
948 intel_pt_free_snapshot_refs(ptr
);
952 static int intel_pt_alloc_snapshot_ref(struct intel_pt_recording
*ptr
, int idx
,
953 size_t snapshot_buf_size
)
955 size_t ref_buf_size
= ptr
->snapshot_ref_buf_size
;
958 ref_buf
= zalloc(ref_buf_size
);
962 ptr
->snapshot_refs
[idx
].ref_buf
= ref_buf
;
963 ptr
->snapshot_refs
[idx
].ref_offset
= snapshot_buf_size
- ref_buf_size
;
968 static size_t intel_pt_snapshot_ref_buf_size(struct intel_pt_recording
*ptr
,
969 size_t snapshot_buf_size
)
971 const size_t max_size
= 256 * 1024;
972 size_t buf_size
= 0, psb_period
;
974 if (ptr
->snapshot_size
<= 64 * 1024)
977 psb_period
= intel_pt_psb_period(ptr
->intel_pt_pmu
, ptr
->evlist
);
979 buf_size
= psb_period
* 2;
981 if (!buf_size
|| buf_size
> max_size
)
984 if (buf_size
>= snapshot_buf_size
)
987 if (buf_size
>= ptr
->snapshot_size
/ 2)
993 static int intel_pt_snapshot_init(struct intel_pt_recording
*ptr
,
994 size_t snapshot_buf_size
)
996 if (ptr
->snapshot_init_done
)
999 ptr
->snapshot_init_done
= true;
1001 ptr
->snapshot_ref_buf_size
= intel_pt_snapshot_ref_buf_size(ptr
,
1008 * intel_pt_compare_buffers - compare bytes in a buffer to a circular buffer.
1009 * @buf1: first buffer
1010 * @compare_size: number of bytes to compare
1011 * @buf2: second buffer (a circular buffer)
1012 * @offs2: offset in second buffer
1013 * @buf2_size: size of second buffer
1015 * The comparison allows for the possibility that the bytes to compare in the
1016 * circular buffer are not contiguous. It is assumed that @compare_size <=
1017 * @buf2_size. This function returns %false if the bytes are identical, %true
1020 static bool intel_pt_compare_buffers(void *buf1
, size_t compare_size
,
1021 void *buf2
, size_t offs2
, size_t buf2_size
)
1023 size_t end2
= offs2
+ compare_size
, part_size
;
1025 if (end2
<= buf2_size
)
1026 return memcmp(buf1
, buf2
+ offs2
, compare_size
);
1028 part_size
= end2
- buf2_size
;
1029 if (memcmp(buf1
, buf2
+ offs2
, part_size
))
1032 compare_size
-= part_size
;
1034 return memcmp(buf1
+ part_size
, buf2
, compare_size
);
1037 static bool intel_pt_compare_ref(void *ref_buf
, size_t ref_offset
,
1038 size_t ref_size
, size_t buf_size
,
1039 void *data
, size_t head
)
1041 size_t ref_end
= ref_offset
+ ref_size
;
1043 if (ref_end
> buf_size
) {
1044 if (head
> ref_offset
|| head
< ref_end
- buf_size
)
1046 } else if (head
> ref_offset
&& head
< ref_end
) {
1050 return intel_pt_compare_buffers(ref_buf
, ref_size
, data
, ref_offset
,
1054 static void intel_pt_copy_ref(void *ref_buf
, size_t ref_size
, size_t buf_size
,
1055 void *data
, size_t head
)
1057 if (head
>= ref_size
) {
1058 memcpy(ref_buf
, data
+ head
- ref_size
, ref_size
);
1060 memcpy(ref_buf
, data
, head
);
1062 memcpy(ref_buf
+ head
, data
+ buf_size
- ref_size
, ref_size
);
1066 static bool intel_pt_wrapped(struct intel_pt_recording
*ptr
, int idx
,
1067 struct auxtrace_mmap
*mm
, unsigned char *data
,
1070 struct intel_pt_snapshot_ref
*ref
= &ptr
->snapshot_refs
[idx
];
1073 wrapped
= intel_pt_compare_ref(ref
->ref_buf
, ref
->ref_offset
,
1074 ptr
->snapshot_ref_buf_size
, mm
->len
,
1077 intel_pt_copy_ref(ref
->ref_buf
, ptr
->snapshot_ref_buf_size
, mm
->len
,
1083 static bool intel_pt_first_wrap(u64
*data
, size_t buf_size
)
1092 for (i
= a
; i
< b
; i
++) {
1100 static int intel_pt_find_snapshot(struct auxtrace_record
*itr
, int idx
,
1101 struct auxtrace_mmap
*mm
, unsigned char *data
,
1102 u64
*head
, u64
*old
)
1104 struct intel_pt_recording
*ptr
=
1105 container_of(itr
, struct intel_pt_recording
, itr
);
1109 pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
1110 __func__
, idx
, (size_t)*old
, (size_t)*head
);
1112 err
= intel_pt_snapshot_init(ptr
, mm
->len
);
1116 if (idx
>= ptr
->snapshot_ref_cnt
) {
1117 err
= intel_pt_alloc_snapshot_refs(ptr
, idx
);
1122 if (ptr
->snapshot_ref_buf_size
) {
1123 if (!ptr
->snapshot_refs
[idx
].ref_buf
) {
1124 err
= intel_pt_alloc_snapshot_ref(ptr
, idx
, mm
->len
);
1128 wrapped
= intel_pt_wrapped(ptr
, idx
, mm
, data
, *head
);
1130 wrapped
= ptr
->snapshot_refs
[idx
].wrapped
;
1131 if (!wrapped
&& intel_pt_first_wrap((u64
*)data
, mm
->len
)) {
1132 ptr
->snapshot_refs
[idx
].wrapped
= true;
1138 * In full trace mode 'head' continually increases. However in snapshot
1139 * mode 'head' is an offset within the buffer. Here 'old' and 'head'
1140 * are adjusted to match the full trace case which expects that 'old' is
1141 * always less than 'head'.
1155 pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
1156 __func__
, wrapped
? "" : "not ", (size_t)*old
, (size_t)*head
);
1161 pr_err("%s: failed, error %d\n", __func__
, err
);
1165 static u64
intel_pt_reference(struct auxtrace_record
*itr __maybe_unused
)
1170 struct auxtrace_record
*intel_pt_recording_init(int *err
)
1172 struct perf_pmu
*intel_pt_pmu
= perf_pmu__find(INTEL_PT_PMU_NAME
);
1173 struct intel_pt_recording
*ptr
;
1178 if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) {
1183 ptr
= zalloc(sizeof(struct intel_pt_recording
));
1189 ptr
->intel_pt_pmu
= intel_pt_pmu
;
1190 ptr
->itr
.pmu
= intel_pt_pmu
;
1191 ptr
->itr
.recording_options
= intel_pt_recording_options
;
1192 ptr
->itr
.info_priv_size
= intel_pt_info_priv_size
;
1193 ptr
->itr
.info_fill
= intel_pt_info_fill
;
1194 ptr
->itr
.free
= intel_pt_recording_free
;
1195 ptr
->itr
.snapshot_start
= intel_pt_snapshot_start
;
1196 ptr
->itr
.snapshot_finish
= intel_pt_snapshot_finish
;
1197 ptr
->itr
.find_snapshot
= intel_pt_find_snapshot
;
1198 ptr
->itr
.parse_snapshot_options
= intel_pt_parse_snapshot_options
;
1199 ptr
->itr
.reference
= intel_pt_reference
;
1200 ptr
->itr
.read_finish
= auxtrace_record__read_finish
;
1202 * Decoding starts at a PSB packet. Minimum PSB period is 2K so 4K
1203 * should give at least 1 PSB per sample.
1205 ptr
->itr
.default_aux_sample_size
= 4096;