[thirdparty/linux.git] / tools / perf / arch / x86 / util / intel-bts.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * intel-bts.c: Intel Processor Trace support
 * Copyright (c) 2013-2015, Intel Corporation.
 */

#include <errno.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/bitops.h>
#include <linux/log2.h>
#include <linux/zalloc.h>

#include "../../../util/cpumap.h"
#include "../../../util/event.h"
#include "../../../util/evsel.h"
#include "../../../util/evlist.h"
#include "../../../util/mmap.h"
#include "../../../util/session.h"
#include "../../../util/pmu.h"
#include "../../../util/debug.h"
#include "../../../util/record.h"
#include "../../../util/tsc.h"
#include "../../../util/auxtrace.h"
#include "../../../util/intel-bts.h"
#include <internal/lib.h> // page_size

#define KiB(x) ((x) * 1024)
#define MiB(x) ((x) * 1024 * 1024)
#define KiB_MASK(x) (KiB(x) - 1)
#define MiB_MASK(x) (MiB(x) - 1)

struct intel_bts_snapshot_ref {
	void	*ref_buf;
	size_t	ref_offset;
	bool	wrapped;
};

struct intel_bts_recording {
	struct auxtrace_record		itr;
	struct perf_pmu			*intel_bts_pmu;
	struct evlist		*evlist;
	bool				snapshot_mode;
	size_t				snapshot_size;
	int				snapshot_ref_cnt;
	struct intel_bts_snapshot_ref	*snapshot_refs;
};

struct branch {
	u64 from;
	u64 to;
	u64 misc;
};

static size_t
intel_bts_info_priv_size(struct auxtrace_record *itr __maybe_unused,
			 struct evlist *evlist __maybe_unused)
{
	return INTEL_BTS_AUXTRACE_PRIV_SIZE;
}

static int intel_bts_info_fill(struct auxtrace_record *itr,
			       struct perf_session *session,
			       struct perf_record_auxtrace_info *auxtrace_info,
			       size_t priv_size)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
	struct perf_event_mmap_page *pc;
	struct perf_tsc_conversion tc = { .time_mult = 0, };
	bool cap_user_time_zero = false;
	int err;

	if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE)
		return -EINVAL;

	if (!session->evlist->core.nr_mmaps)
		return -EINVAL;

	pc = session->evlist->mmap[0].core.base;
	if (pc) {
		err = perf_read_tsc_conversion(pc, &tc);
		if (err) {
			if (err != -EOPNOTSUPP)
				return err;
		} else {
			cap_user_time_zero = tc.time_mult != 0;
		}
		if (!cap_user_time_zero)
			ui__warning("Intel BTS: TSC not available\n");
	}

	auxtrace_info->type = PERF_AUXTRACE_INTEL_BTS;
	auxtrace_info->priv[INTEL_BTS_PMU_TYPE] = intel_bts_pmu->type;
	auxtrace_info->priv[INTEL_BTS_TIME_SHIFT] = tc.time_shift;
	auxtrace_info->priv[INTEL_BTS_TIME_MULT] = tc.time_mult;
	auxtrace_info->priv[INTEL_BTS_TIME_ZERO] = tc.time_zero;
	auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO] = cap_user_time_zero;
	auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE] = btsr->snapshot_mode;

	return 0;
}

static int intel_bts_recording_options(struct auxtrace_record *itr,
				       struct evlist *evlist,
				       struct record_opts *opts)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
	struct evsel *evsel, *intel_bts_evsel = NULL;
	const struct perf_cpu_map *cpus = evlist->core.cpus;
	bool privileged = perf_event_paranoid_check(-1);

	if (opts->auxtrace_sample_mode) {
		pr_err("Intel BTS does not support AUX area sampling\n");
		return -EINVAL;
	}

	btsr->evlist = evlist;
	btsr->snapshot_mode = opts->auxtrace_snapshot_mode;

	evlist__for_each_entry(evlist, evsel) {
		if (evsel->core.attr.type == intel_bts_pmu->type) {
			if (intel_bts_evsel) {
				pr_err("There may be only one " INTEL_BTS_PMU_NAME " event\n");
				return -EINVAL;
			}
			evsel->core.attr.freq = 0;
			evsel->core.attr.sample_period = 1;
			intel_bts_evsel = evsel;
			opts->full_auxtrace = true;
		}
	}

	if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) {
		pr_err("Snapshot mode (-S option) requires " INTEL_BTS_PMU_NAME " PMU event (-e " INTEL_BTS_PMU_NAME ")\n");
		return -EINVAL;
	}

	if (!opts->full_auxtrace)
		return 0;

	if (opts->full_auxtrace && !perf_cpu_map__empty(cpus)) {
		pr_err(INTEL_BTS_PMU_NAME " does not support per-cpu recording\n");
		return -EINVAL;
	}

	/* Set default sizes for snapshot mode */
	if (opts->auxtrace_snapshot_mode) {
		if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) {
			if (privileged) {
				opts->auxtrace_mmap_pages = MiB(4) / page_size;
			} else {
				opts->auxtrace_mmap_pages = KiB(128) / page_size;
				if (opts->mmap_pages == UINT_MAX)
					opts->mmap_pages = KiB(256) / page_size;
			}
		} else if (!opts->auxtrace_mmap_pages && !privileged &&
			   opts->mmap_pages == UINT_MAX) {
			opts->mmap_pages = KiB(256) / page_size;
		}
		if (!opts->auxtrace_snapshot_size)
			opts->auxtrace_snapshot_size =
				opts->auxtrace_mmap_pages * (size_t)page_size;
		if (!opts->auxtrace_mmap_pages) {
			size_t sz = opts->auxtrace_snapshot_size;

			sz = round_up(sz, page_size) / page_size;
			opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
		}
		if (opts->auxtrace_snapshot_size >
				opts->auxtrace_mmap_pages * (size_t)page_size) {
			pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
			       opts->auxtrace_snapshot_size,
			       opts->auxtrace_mmap_pages * (size_t)page_size);
			return -EINVAL;
		}
		if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) {
			pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
			return -EINVAL;
		}
		pr_debug2("Intel BTS snapshot size: %zu\n",
			  opts->auxtrace_snapshot_size);
	}

	/* Set default sizes for full trace mode */
	if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
		if (privileged) {
			opts->auxtrace_mmap_pages = MiB(4) / page_size;
		} else {
			opts->auxtrace_mmap_pages = KiB(128) / page_size;
			if (opts->mmap_pages == UINT_MAX)
				opts->mmap_pages = KiB(256) / page_size;
		}
	}

	/* Validate auxtrace_mmap_pages */
	if (opts->auxtrace_mmap_pages) {
		size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
		size_t min_sz;

		if (opts->auxtrace_snapshot_mode)
			min_sz = KiB(4);
		else
			min_sz = KiB(8);

		if (sz < min_sz || !is_power_of_2(sz)) {
			pr_err("Invalid mmap size for Intel BTS: must be at least %zuKiB and a power of 2\n",
			       min_sz / 1024);
			return -EINVAL;
		}
	}

	if (intel_bts_evsel) {
		/*
		 * To obtain the auxtrace buffer file descriptor, the auxtrace event
		 * must come first.
		 */
		perf_evlist__to_front(evlist, intel_bts_evsel);
		/*
		 * In the case of per-cpu mmaps, we need the CPU on the
		 * AUX event.
		 */
		if (!perf_cpu_map__empty(cpus))
			evsel__set_sample_bit(intel_bts_evsel, CPU);
	}

	/* Add dummy event to keep tracking */
	if (opts->full_auxtrace) {
		struct evsel *tracking_evsel;
		int err;

		err = parse_events(evlist, "dummy:u", NULL);
		if (err)
			return err;

		tracking_evsel = evlist__last(evlist);

		perf_evlist__set_tracking_event(evlist, tracking_evsel);

		tracking_evsel->core.attr.freq = 0;
		tracking_evsel->core.attr.sample_period = 1;
	}

	return 0;
}

static int intel_bts_parse_snapshot_options(struct auxtrace_record *itr,
					    struct record_opts *opts,
					    const char *str)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	unsigned long long snapshot_size = 0;
	char *endptr;

	if (str) {
		snapshot_size = strtoull(str, &endptr, 0);
		if (*endptr || snapshot_size > SIZE_MAX)
			return -1;
	}

	opts->auxtrace_snapshot_mode = true;
	opts->auxtrace_snapshot_size = snapshot_size;

	btsr->snapshot_size = snapshot_size;

	return 0;
}

static u64 intel_bts_reference(struct auxtrace_record *itr __maybe_unused)
{
	return rdtsc();
}

static int intel_bts_alloc_snapshot_refs(struct intel_bts_recording *btsr,
					 int idx)
{
	const size_t sz = sizeof(struct intel_bts_snapshot_ref);
	int cnt = btsr->snapshot_ref_cnt, new_cnt = cnt * 2;
	struct intel_bts_snapshot_ref *refs;

	if (!new_cnt)
		new_cnt = 16;

	while (new_cnt <= idx)
		new_cnt *= 2;

	refs = calloc(new_cnt, sz);
	if (!refs)
		return -ENOMEM;

	memcpy(refs, btsr->snapshot_refs, cnt * sz);

	btsr->snapshot_refs = refs;
	btsr->snapshot_ref_cnt = new_cnt;

	return 0;
}

static void intel_bts_free_snapshot_refs(struct intel_bts_recording *btsr)
{
	int i;

	for (i = 0; i < btsr->snapshot_ref_cnt; i++)
		zfree(&btsr->snapshot_refs[i].ref_buf);
	zfree(&btsr->snapshot_refs);
}

static void intel_bts_recording_free(struct auxtrace_record *itr)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);

	intel_bts_free_snapshot_refs(btsr);
	free(btsr);
}

static int intel_bts_snapshot_start(struct auxtrace_record *itr)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct evsel *evsel;

	evlist__for_each_entry(btsr->evlist, evsel) {
		if (evsel->core.attr.type == btsr->intel_bts_pmu->type)
			return evsel__disable(evsel);
	}
	return -EINVAL;
}

static int intel_bts_snapshot_finish(struct auxtrace_record *itr)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct evsel *evsel;

	evlist__for_each_entry(btsr->evlist, evsel) {
		if (evsel->core.attr.type == btsr->intel_bts_pmu->type)
			return evsel__enable(evsel);
	}
	return -EINVAL;
}

static bool intel_bts_first_wrap(u64 *data, size_t buf_size)
{
	int i, a, b;

	b = buf_size >> 3;
	a = b - 512;
	if (a < 0)
		a = 0;

	for (i = a; i < b; i++) {
		if (data[i])
			return true;
	}

	return false;
}

static int intel_bts_find_snapshot(struct auxtrace_record *itr, int idx,
				   struct auxtrace_mmap *mm, unsigned char *data,
				   u64 *head, u64 *old)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	bool wrapped;
	int err;

	pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
		  __func__, idx, (size_t)*old, (size_t)*head);

	if (idx >= btsr->snapshot_ref_cnt) {
		err = intel_bts_alloc_snapshot_refs(btsr, idx);
		if (err)
			goto out_err;
	}

	wrapped = btsr->snapshot_refs[idx].wrapped;
	if (!wrapped && intel_bts_first_wrap((u64 *)data, mm->len)) {
		btsr->snapshot_refs[idx].wrapped = true;
		wrapped = true;
	}

	/*
	 * In full trace mode 'head' continually increases.  However in snapshot
	 * mode 'head' is an offset within the buffer.  Here 'old' and 'head'
	 * are adjusted to match the full trace case which expects that 'old' is
	 * always less than 'head'.
	 */
	if (wrapped) {
		*old = *head;
		*head += mm->len;
	} else {
		if (mm->mask)
			*old &= mm->mask;
		else
			*old %= mm->len;
		if (*old > *head)
			*head += mm->len;
	}

	pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
		  __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head);

	return 0;

out_err:
	pr_err("%s: failed, error %d\n", __func__, err);
	return err;
}

struct auxtrace_record *intel_bts_recording_init(int *err)
{
	struct perf_pmu *intel_bts_pmu = perf_pmu__find(INTEL_BTS_PMU_NAME);
	struct intel_bts_recording *btsr;

	if (!intel_bts_pmu)
		return NULL;

	if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) {
		*err = -errno;
		return NULL;
	}

	btsr = zalloc(sizeof(struct intel_bts_recording));
	if (!btsr) {
		*err = -ENOMEM;
		return NULL;
	}

	btsr->intel_bts_pmu = intel_bts_pmu;
	btsr->itr.pmu = intel_bts_pmu;
	btsr->itr.recording_options = intel_bts_recording_options;
	btsr->itr.info_priv_size = intel_bts_info_priv_size;
	btsr->itr.info_fill = intel_bts_info_fill;
	btsr->itr.free = intel_bts_recording_free;
	btsr->itr.snapshot_start = intel_bts_snapshot_start;
	btsr->itr.snapshot_finish = intel_bts_snapshot_finish;
	btsr->itr.find_snapshot = intel_bts_find_snapshot;
	btsr->itr.parse_snapshot_options = intel_bts_parse_snapshot_options;
	btsr->itr.reference = intel_bts_reference;
	btsr->itr.read_finish = auxtrace_record__read_finish;
	btsr->itr.alignment = sizeof(struct branch);
	return &btsr->itr;
}
Commit	Line	Data
2025cf9e	1	// SPDX-License-Identifier: GPL-2.0-only
d0170af7 AH	2	/*
	3	* intel-bts.c: Intel Processor Trace support
	4	* Copyright (c) 2013-2015, Intel Corporation.
d0170af7 AH	5	*/
d0170af7 AH	6
a43783ae	7	#include <errno.h>
d0170af7 AH	8	#include <linux/kernel.h>
	9	#include <linux/types.h>
	10	#include <linux/bitops.h>
	11	#include <linux/log2.h>
7f7c536f	12	#include <linux/zalloc.h>
d0170af7	13
441b62ac IR	14	#include "../../../util/cpumap.h"
	15	#include "../../../util/event.h"
	16	#include "../../../util/evsel.h"
	17	#include "../../../util/evlist.h"
	18	#include "../../../util/mmap.h"
	19	#include "../../../util/session.h"
	20	#include "../../../util/pmu.h"
	21	#include "../../../util/debug.h"
	22	#include "../../../util/record.h"
	23	#include "../../../util/tsc.h"
	24	#include "../../../util/auxtrace.h"
	25	#include "../../../util/intel-bts.h"
20f2be1d	26	#include <internal/lib.h> // page_size
d0170af7 AH	27
	28	#define KiB(x) ((x) * 1024)
	29	#define MiB(x) ((x) * 1024 * 1024)
	30	#define KiB_MASK(x) (KiB(x) - 1)
	31	#define MiB_MASK(x) (MiB(x) - 1)
	32
d0170af7 AH	33	struct intel_bts_snapshot_ref {
	34	void *ref_buf;
	35	size_t ref_offset;
	36	bool wrapped;
	37	};
	38
	39	struct intel_bts_recording {
	40	struct auxtrace_record itr;
	41	struct perf_pmu *intel_bts_pmu;
63503dba	42	struct evlist *evlist;
d0170af7 AH	43	bool snapshot_mode;
	44	size_t snapshot_size;
	45	int snapshot_ref_cnt;
	46	struct intel_bts_snapshot_ref *snapshot_refs;
	47	};
	48
	49	struct branch {
	50	u64 from;
	51	u64 to;
	52	u64 misc;
	53	};
	54
14a05e13 MP	55	static size_t
14a05e13 MP	56	intel_bts_info_priv_size(struct auxtrace_record *itr __maybe_unused,
63503dba	57	struct evlist *evlist __maybe_unused)
d0170af7 AH	58	{
	59	return INTEL_BTS_AUXTRACE_PRIV_SIZE;
	60	}
	61
	62	static int intel_bts_info_fill(struct auxtrace_record *itr,
	63	struct perf_session *session,
72932371	64	struct perf_record_auxtrace_info *auxtrace_info,
d0170af7 AH	65	size_t priv_size)
	66	{
	67	struct intel_bts_recording *btsr =
	68	container_of(itr, struct intel_bts_recording, itr);
	69	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
	70	struct perf_event_mmap_page *pc;
	71	struct perf_tsc_conversion tc = { .time_mult = 0, };
	72	bool cap_user_time_zero = false;
	73	int err;
	74
	75	if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE)
	76	return -EINVAL;
	77
c976ee11	78	if (!session->evlist->core.nr_mmaps)
d0170af7 AH	79	return -EINVAL;
d0170af7 AH	80
547740f7	81	pc = session->evlist->mmap[0].core.base;
d0170af7 AH	82	if (pc) {
	83	err = perf_read_tsc_conversion(pc, &tc);
	84	if (err) {
	85	if (err != -EOPNOTSUPP)
	86	return err;
	87	} else {
	88	cap_user_time_zero = tc.time_mult != 0;
	89	}
	90	if (!cap_user_time_zero)
	91	ui__warning("Intel BTS: TSC not available\n");
	92	}
	93
	94	auxtrace_info->type = PERF_AUXTRACE_INTEL_BTS;
	95	auxtrace_info->priv[INTEL_BTS_PMU_TYPE] = intel_bts_pmu->type;
	96	auxtrace_info->priv[INTEL_BTS_TIME_SHIFT] = tc.time_shift;
	97	auxtrace_info->priv[INTEL_BTS_TIME_MULT] = tc.time_mult;
	98	auxtrace_info->priv[INTEL_BTS_TIME_ZERO] = tc.time_zero;
	99	auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO] = cap_user_time_zero;
	100	auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE] = btsr->snapshot_mode;
	101
	102	return 0;
	103	}
	104
	105	static int intel_bts_recording_options(struct auxtrace_record *itr,
63503dba	106	struct evlist *evlist,
d0170af7 AH	107	struct record_opts *opts)
	108	{
	109	struct intel_bts_recording *btsr =
	110	container_of(itr, struct intel_bts_recording, itr);
	111	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
32dcd021	112	struct evsel evsel, intel_bts_evsel = NULL;
f72f901d	113	const struct perf_cpu_map *cpus = evlist->core.cpus;
dda1bf8e	114	bool privileged = perf_event_paranoid_check(-1);
d0170af7	115
32a1ece4 AH	116	if (opts->auxtrace_sample_mode) {
	117	pr_err("Intel BTS does not support AUX area sampling\n");
	118	return -EINVAL;
	119	}
	120
d0170af7 AH	121	btsr->evlist = evlist;
	122	btsr->snapshot_mode = opts->auxtrace_snapshot_mode;
	123
e5cadb93	124	evlist__for_each_entry(evlist, evsel) {
1fc632ce	125	if (evsel->core.attr.type == intel_bts_pmu->type) {
d0170af7 AH	126	if (intel_bts_evsel) {
	127	pr_err("There may be only one " INTEL_BTS_PMU_NAME " event\n");
	128	return -EINVAL;
	129	}
1fc632ce JO	130	evsel->core.attr.freq = 0;
1fc632ce JO	131	evsel->core.attr.sample_period = 1;
d0170af7 AH	132	intel_bts_evsel = evsel;
	133	opts->full_auxtrace = true;
	134	}
	135	}
	136
	137	if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) {
	138	pr_err("Snapshot mode (-S option) requires " INTEL_BTS_PMU_NAME " PMU event (-e " INTEL_BTS_PMU_NAME ")\n");
	139	return -EINVAL;
	140	}
	141
	142	if (!opts->full_auxtrace)
	143	return 0;
	144
315c0a1f	145	if (opts->full_auxtrace && !perf_cpu_map__empty(cpus)) {
d0170af7 AH	146	pr_err(INTEL_BTS_PMU_NAME " does not support per-cpu recording\n");
	147	return -EINVAL;
	148	}
	149
	150	/* Set default sizes for snapshot mode */
	151	if (opts->auxtrace_snapshot_mode) {
	152	if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) {
	153	if (privileged) {
	154	opts->auxtrace_mmap_pages = MiB(4) / page_size;
	155	} else {
	156	opts->auxtrace_mmap_pages = KiB(128) / page_size;
	157	if (opts->mmap_pages == UINT_MAX)
	158	opts->mmap_pages = KiB(256) / page_size;
	159	}
	160	} else if (!opts->auxtrace_mmap_pages && !privileged &&
	161	opts->mmap_pages == UINT_MAX) {
	162	opts->mmap_pages = KiB(256) / page_size;
	163	}
	164	if (!opts->auxtrace_snapshot_size)
	165	opts->auxtrace_snapshot_size =
	166	opts->auxtrace_mmap_pages * (size_t)page_size;
	167	if (!opts->auxtrace_mmap_pages) {
	168	size_t sz = opts->auxtrace_snapshot_size;
	169
	170	sz = round_up(sz, page_size) / page_size;
	171	opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
	172	}
	173	if (opts->auxtrace_snapshot_size >
	174	opts->auxtrace_mmap_pages * (size_t)page_size) {
	175	pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
	176	opts->auxtrace_snapshot_size,
	177	opts->auxtrace_mmap_pages * (size_t)page_size);
	178	return -EINVAL;
	179	}
	180	if (!opts->auxtrace_snapshot_size \|\| !opts->auxtrace_mmap_pages) {
	181	pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
	182	return -EINVAL;
	183	}
	184	pr_debug2("Intel BTS snapshot size: %zu\n",
	185	opts->auxtrace_snapshot_size);
	186	}
	187
	188	/* Set default sizes for full trace mode */
	189	if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
	190	if (privileged) {
	191	opts->auxtrace_mmap_pages = MiB(4) / page_size;
	192	} else {
	193	opts->auxtrace_mmap_pages = KiB(128) / page_size;
	194	if (opts->mmap_pages == UINT_MAX)
	195	opts->mmap_pages = KiB(256) / page_size;
	196	}
	197	}
	198
	199	/* Validate auxtrace_mmap_pages */
	200	if (opts->auxtrace_mmap_pages) {
	201	size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
	202	size_t min_sz;
	203
	204	if (opts->auxtrace_snapshot_mode)
	205	min_sz = KiB(4);
	206	else
	207	min_sz = KiB(8);
	208
	209	if (sz < min_sz \|\| !is_power_of_2(sz)) {
210	pr_err("Invalid mmap size for Intel BTS: must be at least %zuKiB and a power of 2\n",
211	min_sz / 1024);
212	return -EINVAL;
213	}
214	}
215
216	if (intel_bts_evsel) {
217	/*
218	* To obtain the auxtrace buffer file descriptor, the auxtrace event
219	* must come first.
220	*/
221	perf_evlist__to_front(evlist, intel_bts_evsel);
222	/*
223	* In the case of per-cpu mmaps, we need the CPU on the
224	* AUX event.
225	*/
315c0a1f	226	if (!perf_cpu_map__empty(cpus))
862b2f8f	227	evsel__set_sample_bit(intel_bts_evsel, CPU);
d0170af7 AH	228	}
	229
	230	/* Add dummy event to keep tracking */
	231	if (opts->full_auxtrace) {
32dcd021	232	struct evsel *tracking_evsel;
d0170af7 AH	233	int err;
	234
	235	err = parse_events(evlist, "dummy:u", NULL);
	236	if (err)
	237	return err;
	238
515dbe48	239	tracking_evsel = evlist__last(evlist);
d0170af7 AH	240
	241	perf_evlist__set_tracking_event(evlist, tracking_evsel);
	242
1fc632ce JO	243	tracking_evsel->core.attr.freq = 0;
1fc632ce JO	244	tracking_evsel->core.attr.sample_period = 1;
d0170af7 AH	245	}
	246
	247	return 0;
	248	}
	249
	250	static int intel_bts_parse_snapshot_options(struct auxtrace_record *itr,
	251	struct record_opts *opts,
	252	const char *str)
	253	{
	254	struct intel_bts_recording *btsr =
	255	container_of(itr, struct intel_bts_recording, itr);
	256	unsigned long long snapshot_size = 0;
	257	char *endptr;
	258
	259	if (str) {
	260	snapshot_size = strtoull(str, &endptr, 0);
	261	if (*endptr \|\| snapshot_size > SIZE_MAX)
	262	return -1;
	263	}
	264
	265	opts->auxtrace_snapshot_mode = true;
	266	opts->auxtrace_snapshot_size = snapshot_size;
	267
	268	btsr->snapshot_size = snapshot_size;
	269
	270	return 0;
	271	}
	272
	273	static u64 intel_bts_reference(struct auxtrace_record *itr __maybe_unused)
	274	{
	275	return rdtsc();
	276	}
	277
	278	static int intel_bts_alloc_snapshot_refs(struct intel_bts_recording *btsr,
	279	int idx)
	280	{
	281	const size_t sz = sizeof(struct intel_bts_snapshot_ref);
	282	int cnt = btsr->snapshot_ref_cnt, new_cnt = cnt * 2;
	283	struct intel_bts_snapshot_ref *refs;
	284
	285	if (!new_cnt)
	286	new_cnt = 16;
	287
	288	while (new_cnt <= idx)
	289	new_cnt *= 2;
	290
	291	refs = calloc(new_cnt, sz);
	292	if (!refs)
	293	return -ENOMEM;
	294
	295	memcpy(refs, btsr->snapshot_refs, cnt * sz);
	296
	297	btsr->snapshot_refs = refs;
	298	btsr->snapshot_ref_cnt = new_cnt;
	299
	300	return 0;
	301	}
	302
	303	static void intel_bts_free_snapshot_refs(struct intel_bts_recording *btsr)
	304	{
	305	int i;
	306
	307	for (i = 0; i < btsr->snapshot_ref_cnt; i++)
	308	zfree(&btsr->snapshot_refs[i].ref_buf);
309	zfree(&btsr->snapshot_refs);
310	}
311
312	static void intel_bts_recording_free(struct auxtrace_record *itr)
313	{
314	struct intel_bts_recording *btsr =
315	container_of(itr, struct intel_bts_recording, itr);
316
317	intel_bts_free_snapshot_refs(btsr);
318	free(btsr);
319	}
320
321	static int intel_bts_snapshot_start(struct auxtrace_record *itr)
322	{
323	struct intel_bts_recording *btsr =
324	container_of(itr, struct intel_bts_recording, itr);
32dcd021	325	struct evsel *evsel;
d0170af7	326
e5cadb93	327	evlist__for_each_entry(btsr->evlist, evsel) {
1fc632ce	328	if (evsel->core.attr.type == btsr->intel_bts_pmu->type)
9a10bb22	329	return evsel__disable(evsel);
d0170af7 AH	330	}
	331	return -EINVAL;
	332	}
	333
	334	static int intel_bts_snapshot_finish(struct auxtrace_record *itr)
	335	{
	336	struct intel_bts_recording *btsr =
	337	container_of(itr, struct intel_bts_recording, itr);
32dcd021	338	struct evsel *evsel;
d0170af7	339
e5cadb93	340	evlist__for_each_entry(btsr->evlist, evsel) {
1fc632ce	341	if (evsel->core.attr.type == btsr->intel_bts_pmu->type)
ec7f24ef	342	return evsel__enable(evsel);
d0170af7 AH	343	}
	344	return -EINVAL;
	345	}
	346
	347	static bool intel_bts_first_wrap(u64 *data, size_t buf_size)
	348	{
	349	int i, a, b;
	350
	351	b = buf_size >> 3;
	352	a = b - 512;
	353	if (a < 0)
	354	a = 0;
	355
	356	for (i = a; i < b; i++) {
	357	if (data[i])
	358	return true;
	359	}
	360
	361	return false;
	362	}
	363
	364	static int intel_bts_find_snapshot(struct auxtrace_record *itr, int idx,
	365	struct auxtrace_mmap mm, unsigned char data,
	366	u64 head, u64 old)
	367	{
	368	struct intel_bts_recording *btsr =
	369	container_of(itr, struct intel_bts_recording, itr);
	370	bool wrapped;
	371	int err;
	372
	373	pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
	374	__func__, idx, (size_t)old, (size_t)head);
	375
	376	if (idx >= btsr->snapshot_ref_cnt) {
	377	err = intel_bts_alloc_snapshot_refs(btsr, idx);
	378	if (err)
	379	goto out_err;
	380	}
	381
	382	wrapped = btsr->snapshot_refs[idx].wrapped;
	383	if (!wrapped && intel_bts_first_wrap((u64 *)data, mm->len)) {
	384	btsr->snapshot_refs[idx].wrapped = true;
	385	wrapped = true;
	386	}
	387
	388	/*
	389	* In full trace mode 'head' continually increases. However in snapshot
	390	* mode 'head' is an offset within the buffer. Here 'old' and 'head'
	391	* are adjusted to match the full trace case which expects that 'old' is
	392	* always less than 'head'.
	393	*/
	394	if (wrapped) {
	395	old = head;
	396	*head += mm->len;
	397	} else {
	398	if (mm->mask)
	399	*old &= mm->mask;
	400	else
	401	*old %= mm->len;
	402	if (old > head)
	403	*head += mm->len;
	404	}
	405
	406	pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
407	__func__, wrapped ? "" : "not ", (size_t)old, (size_t)head);
408
409	return 0;
410
411	out_err:
412	pr_err("%s: failed, error %d\n", __func__, err);
413	return err;
414	}
415
d0170af7 AH	416	struct auxtrace_record intel_bts_recording_init(int err)
	417	{
	418	struct perf_pmu *intel_bts_pmu = perf_pmu__find(INTEL_BTS_PMU_NAME);
	419	struct intel_bts_recording *btsr;
	420
	421	if (!intel_bts_pmu)
	422	return NULL;
	423
bd0c7a54 AH	424	if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) {
	425	*err = -errno;
	426	return NULL;
	427	}
	428
d0170af7 AH	429	btsr = zalloc(sizeof(struct intel_bts_recording));
	430	if (!btsr) {
	431	*err = -ENOMEM;
	432	return NULL;
	433	}
	434
	435	btsr->intel_bts_pmu = intel_bts_pmu;
ad60ba0c	436	btsr->itr.pmu = intel_bts_pmu;
d0170af7 AH	437	btsr->itr.recording_options = intel_bts_recording_options;
	438	btsr->itr.info_priv_size = intel_bts_info_priv_size;
	439	btsr->itr.info_fill = intel_bts_info_fill;
	440	btsr->itr.free = intel_bts_recording_free;
	441	btsr->itr.snapshot_start = intel_bts_snapshot_start;
	442	btsr->itr.snapshot_finish = intel_bts_snapshot_finish;
	443	btsr->itr.find_snapshot = intel_bts_find_snapshot;
	444	btsr->itr.parse_snapshot_options = intel_bts_parse_snapshot_options;
	445	btsr->itr.reference = intel_bts_reference;
ad60ba0c	446	btsr->itr.read_finish = auxtrace_record__read_finish;
d0170af7 AH	447	btsr->itr.alignment = sizeof(struct branch);
	448	return &btsr->itr;
	449	}