perf/core: Save raw sample data conditionally based on sample type

Currently, space for raw sample data is always allocated within sample
records for both BPF output and tracepoint events. This leads to unused
space in sample records when raw sample data is not requested.

This patch enforces checking sample type of an event in
perf_sample_save_raw_data(). So raw sample data will only be saved if
explicitly requested, reducing overhead when it is not needed.

Fixes: 0a9081cf0a11 ("perf/core: Add perf_sample_save_raw_data() helper")
Signed-off-by: Yabin Cui <yabinc@google.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Ian Rogers <irogers@google.com>
Acked-by: Namhyung Kim <namhyung@kernel.org>
Link: https://lore.kernel.org/r/20240515193610.2350456-2-yabinc@google.com

diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c
index e2e0aa463fbd1e5dd228a78fcd6ecb122856440e..c3075e4a8efc315711bf2d317ae2ac56e3ec5b7e 100644 (file)
--- a/arch/s390/kernel/perf_cpum_cf.c
+++ b/arch/s390/kernel/perf_cpum_cf.c
@@ -981,7 +981,7 @@ static int cfdiag_push_sample(struct perf_event *event,
        if (event->attr.sample_type & PERF_SAMPLE_RAW) {
                raw.frag.size = cpuhw->usedss;
                raw.frag.data = cpuhw->stop;
-               perf_sample_save_raw_data(&data, &raw);
+               perf_sample_save_raw_data(&data, event, &raw);
        }
 
        overflow = perf_event_overflow(event, &data, &regs);

diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c
index fa732545426611f6f8c96be60d6e520b861ad6ed..10725f5a6f0fd12fd4fe46c5aa7af26298a54b90 100644 (file)
--- a/arch/s390/kernel/perf_pai_crypto.c
+++ b/arch/s390/kernel/perf_pai_crypto.c
@@ -478,7 +478,7 @@ static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump,
        if (event->attr.sample_type & PERF_SAMPLE_RAW) {
                raw.frag.size = rawsize;
                raw.frag.data = cpump->save;
-               perf_sample_save_raw_data(&data, &raw);
+               perf_sample_save_raw_data(&data, event, &raw);
        }
 
        overflow = perf_event_overflow(event, &data, &regs);

diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c
index 7f462bef1fc07533b2f4e85937417c7b7cfbca2e..a8f0bad99cf04fa1b4c1fb45d9b33345f36cc23a 100644 (file)
--- a/arch/s390/kernel/perf_pai_ext.c
+++ b/arch/s390/kernel/perf_pai_ext.c
@@ -503,7 +503,7 @@ static int paiext_push_sample(size_t rawsize, struct paiext_map *cpump,
        if (event->attr.sample_type & PERF_SAMPLE_RAW) {
                raw.frag.size = rawsize;
                raw.frag.data = cpump->save;
-               perf_sample_save_raw_data(&data, &raw);
+               perf_sample_save_raw_data(&data, event, &raw);
        }
 
        overflow = perf_event_overflow(event, &data, &regs);

diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index e91970b01d6243e44d14e7e7e732aa44a59d2188..c3a2f6f57770ab9c037ac57eb45d1acdb40cb559 100644 (file)
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -1118,7 +1118,7 @@ fail:
                                .data = ibs_data.data,
                        },
                };
-               perf_sample_save_raw_data(&data, &raw);
+               perf_sample_save_raw_data(&data, event, &raw);
        }
 
        if (perf_ibs == &perf_ibs_op)

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index cb99ec8c9e96f63c64eeeb4470c019d86bc6e50f..f7c0a3f2f502d0ef927030779a98d4491006c7de 100644 (file)
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1287,12 +1287,18 @@ static inline void perf_sample_save_callchain(struct perf_sample_data *data,
 }
 
 static inline void perf_sample_save_raw_data(struct perf_sample_data *data,
+                                            struct perf_event *event,
                                             struct perf_raw_record *raw)
 {
        struct perf_raw_frag *frag = &raw->frag;
        u32 sum = 0;
        int size;
 
+       if (!(event->attr.sample_type & PERF_SAMPLE_RAW))
+               return;
+       if (WARN_ON_ONCE(data->sample_flags & PERF_SAMPLE_RAW))
+               return;
+
        do {
                sum += frag->size;
                if (perf_raw_frag_last(frag))

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 1869164a4e991319ca8c576a3d9ff4a14ac6cfa9..3670b91f182fbdc75dbbdd631186d77e297b879f 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -10451,9 +10451,9 @@ static struct pmu perf_tracepoint = {
 };
 
 static int perf_tp_filter_match(struct perf_event *event,
-                               struct perf_sample_data *data)
+                               struct perf_raw_record *raw)
 {
-       void *record = data->raw->frag.data;
+       void *record = raw->frag.data;
 
        /* only top level events have filters set */
        if (event->parent)
@@ -10465,7 +10465,7 @@ static int perf_tp_filter_match(struct perf_event *event,
 }
 
 static int perf_tp_event_match(struct perf_event *event,
-                               struct perf_sample_data *data,
+                               struct perf_raw_record *raw,
                                struct pt_regs *regs)
 {
        if (event->hw.state & PERF_HES_STOPPED)
@@ -10476,7 +10476,7 @@ static int perf_tp_event_match(struct perf_event *event,
        if (event->attr.exclude_kernel && !user_mode(regs))
                return 0;
 
-       if (!perf_tp_filter_match(event, data))
+       if (!perf_tp_filter_match(event, raw))
                return 0;
 
        return 1;
@@ -10502,6 +10502,7 @@ EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
 static void __perf_tp_event_target_task(u64 count, void *record,
                                        struct pt_regs *regs,
                                        struct perf_sample_data *data,
+                                       struct perf_raw_record *raw,
                                        struct perf_event *event)
 {
        struct trace_entry *entry = record;
@@ -10511,13 +10512,17 @@ static void __perf_tp_event_target_task(u64 count, void *record,
        /* Cannot deliver synchronous signal to other task. */
        if (event->attr.sigtrap)
                return;
-       if (perf_tp_event_match(event, data, regs))
+       if (perf_tp_event_match(event, raw, regs)) {
+               perf_sample_data_init(data, 0, 0);
+               perf_sample_save_raw_data(data, event, raw);
                perf_swevent_event(event, count, data, regs);
+       }
 }
 
 static void perf_tp_event_target_task(u64 count, void *record,
                                      struct pt_regs *regs,
                                      struct perf_sample_data *data,
+                                     struct perf_raw_record *raw,
                                      struct perf_event_context *ctx)
 {
        unsigned int cpu = smp_processor_id();
@@ -10525,15 +10530,15 @@ static void perf_tp_event_target_task(u64 count, void *record,
        struct perf_event *event, *sibling;
 
        perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {
-               __perf_tp_event_target_task(count, record, regs, data, event);
+               __perf_tp_event_target_task(count, record, regs, data, raw, event);
                for_each_sibling_event(sibling, event)
-                       __perf_tp_event_target_task(count, record, regs, data, sibling);
+                       __perf_tp_event_target_task(count, record, regs, data, raw, sibling);
        }
 
        perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {
-               __perf_tp_event_target_task(count, record, regs, data, event);
+               __perf_tp_event_target_task(count, record, regs, data, raw, event);
                for_each_sibling_event(sibling, event)
-                       __perf_tp_event_target_task(count, record, regs, data, sibling);
+                       __perf_tp_event_target_task(count, record, regs, data, raw, sibling);
        }
 }
 
@@ -10551,15 +10556,10 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
                },
        };
 
-       perf_sample_data_init(&data, 0, 0);
-       perf_sample_save_raw_data(&data, &raw);
-
        perf_trace_buf_update(record, event_type);
 
        hlist_for_each_entry_rcu(event, head, hlist_entry) {
-               if (perf_tp_event_match(event, &data, regs)) {
-                       perf_swevent_event(event, count, &data, regs);
-
+               if (perf_tp_event_match(event, &raw, regs)) {
                        /*
                         * Here use the same on-stack perf_sample_data,
                         * some members in data are event-specific and
@@ -10569,7 +10569,8 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
                         * because data->sample_flags is set.
                         */
                        perf_sample_data_init(&data, 0, 0);
-                       perf_sample_save_raw_data(&data, &raw);
+                       perf_sample_save_raw_data(&data, event, &raw);
+                       perf_swevent_event(event, count, &data, regs);
                }
        }
 
@@ -10586,7 +10587,7 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
                        goto unlock;
 
                raw_spin_lock(&ctx->lock);
-               perf_tp_event_target_task(count, record, regs, &data, ctx);
+               perf_tp_event_target_task(count, record, regs, &data, &raw, ctx);
                raw_spin_unlock(&ctx->lock);
 unlock:
                rcu_read_unlock();

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index fdab7ecd8dfa068a09f8899d56cb90081b5fb812..162bacf8aa5d1b16da45bfedf38708896c1f0c97 100644 (file)
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -619,7 +619,8 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
 
 static __always_inline u64
 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
-                       u64 flags, struct perf_sample_data *sd)
+                       u64 flags, struct perf_raw_record *raw,
+                       struct perf_sample_data *sd)
 {
        struct bpf_array *array = container_of(map, struct bpf_array, map);
        unsigned int cpu = smp_processor_id();
@@ -644,6 +645,8 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
        if (unlikely(event->oncpu != cpu))
                return -EOPNOTSUPP;
 
+       perf_sample_save_raw_data(sd, event, raw);
+
        return perf_event_output(event, sd, regs);
 }
 
@@ -687,9 +690,8 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
        }
 
        perf_sample_data_init(sd, 0, 0);
-       perf_sample_save_raw_data(sd, &raw);
 
-       err = __bpf_perf_event_output(regs, map, flags, sd);
+       err = __bpf_perf_event_output(regs, map, flags, &raw, sd);
 out:
        this_cpu_dec(bpf_trace_nest_level);
        preempt_enable();
@@ -748,9 +750,8 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
 
        perf_fetch_caller_regs(regs);
        perf_sample_data_init(sd, 0, 0);
-       perf_sample_save_raw_data(sd, &raw);
 
-       ret = __bpf_perf_event_output(regs, map, flags, sd);
+       ret = __bpf_perf_event_output(regs, map, flags, &raw, sd);
 out:
        this_cpu_dec(bpf_event_output_nest_level);
        preempt_enable();