perf: Supply task information to sched_task()

To save/restore LBR call stack data in system-wide mode, the task_struct
information is required.

Extend the parameters of sched_task() to supply task_struct information.

When schedule in, the LBR call stack data for new task will be restored.
When schedule out, the LBR call stack data for old task will be saved.
Only need to pass the required task_struct information.

Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250314172700.438923-4-kan.liang@linux.intel.com

diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
index 2b79171ee185be55f2145157620f01fbca26345f..f4e03aaabb4c36daac11f03bb7b226ad624e35d6 100644 (file)
--- a/arch/powerpc/perf/core-book3s.c
+++ b/arch/powerpc/perf/core-book3s.c
@@ -132,7 +132,10 @@ static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
 
 static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
 static inline void power_pmu_bhrb_disable(struct perf_event *event) {}
-static void power_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) {}
+static void power_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                                struct task_struct *task, bool sched_in)
+{
+}
 static inline void power_pmu_bhrb_read(struct perf_event *event, struct cpu_hw_events *cpuhw) {}
 static void pmao_restore_workaround(bool ebb) { }
 #endif /* CONFIG_PPC32 */
@@ -444,7 +447,8 @@ static void power_pmu_bhrb_disable(struct perf_event *event)
 /* Called from ctxsw to prevent one process's branch entries to
  * mingle with the other process's entries during context switch.
  */
-static void power_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+static void power_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                                struct task_struct *task, bool sched_in)
 {
        if (!ppmu->bhrb_nr)
                return;

diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c
index 10725f5a6f0fd12fd4fe46c5aa7af26298a54b90..63875270941bc4f6134aa4cc708b5ea692b90bef 100644 (file)
--- a/arch/s390/kernel/perf_pai_crypto.c
+++ b/arch/s390/kernel/perf_pai_crypto.c
@@ -518,7 +518,8 @@ static void paicrypt_have_samples(void)
 /* Called on schedule-in and schedule-out. No access to event structure,
  * but for sampling only event CRYPTO_ALL is allowed.
  */
-static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                               struct task_struct *task, bool sched_in)
 {
        /* We started with a clean page on event installation. So read out
         * results on schedule_out and if page was dirty, save old values.

diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c
index a8f0bad99cf04fa1b4c1fb45d9b33345f36cc23a..fd14d5ebccbca0f2539abb752184dc304e5704d3 100644 (file)
--- a/arch/s390/kernel/perf_pai_ext.c
+++ b/arch/s390/kernel/perf_pai_ext.c
@@ -542,7 +542,8 @@ static void paiext_have_samples(void)
 /* Called on schedule-in and schedule-out. No access to event structure,
  * but for sampling only event NNPA_ALL is allowed.
  */
-static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                             struct task_struct *task, bool sched_in)
 {
        /* We started with a clean page on event installation. So read out
         * results on schedule_out and if page was dirty, save old values.

diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c
index 780acd3dff22a2d62525a4fbf2bfb8148533d47c..ec34274633824e7a850d44872cebb07f7191e500 100644 (file)
--- a/arch/x86/events/amd/brs.c
+++ b/arch/x86/events/amd/brs.c
@@ -381,7 +381,8 @@ static void amd_brs_poison_buffer(void)
  * On ctxswin, sched_in = true, called after the PMU has started
  * On ctxswout, sched_in = false, called before the PMU is stopped
  */
-void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                           struct task_struct *task, bool sched_in)
 {
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 

diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c
index 19c7b76e21bcb4b2736bdb0a538f641de29c1b02..c06ccca96851f3367b5839ae24c43e5005d4dca4 100644 (file)
--- a/arch/x86/events/amd/lbr.c
+++ b/arch/x86/events/amd/lbr.c
@@ -371,7 +371,8 @@ void amd_pmu_lbr_del(struct perf_event *event)
        perf_sched_cb_dec(event->pmu);
 }
 
-void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                           struct task_struct *task, bool sched_in)
 {
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 

diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 20ad5cca6ad26cbb60faad09cc3923dfff774ca8..ae8c90adca0f0fa6ed0c14f336528421f8c7bd26 100644 (file)
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -2638,9 +2638,10 @@ static const struct attribute_group *x86_pmu_attr_groups[] = {
        NULL,
 };
 
-static void x86_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+static void x86_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                              struct task_struct *task, bool sched_in)
 {
-       static_call_cond(x86_pmu_sched_task)(pmu_ctx, sched_in);
+       static_call_cond(x86_pmu_sched_task)(pmu_ctx, task, sched_in);
 }
 
 static void x86_pmu_swap_task_ctx(struct perf_event_pmu_context *prev_epc,

diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 5a8d6e1a933448639603059b6dcfa1ff442ab53c..3efbb03fd77ee24846ec2af8f6b0ec4caadea52f 100644 (file)
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -5294,10 +5294,10 @@ static void intel_pmu_cpu_dead(int cpu)
 }
 
 static void intel_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
-                                bool sched_in)
+                                struct task_struct *task, bool sched_in)
 {
        intel_pmu_pebs_sched_task(pmu_ctx, sched_in);
-       intel_pmu_lbr_sched_task(pmu_ctx, sched_in);
+       intel_pmu_lbr_sched_task(pmu_ctx, task, sched_in);
 }
 
 static void intel_pmu_swap_task_ctx(struct perf_event_pmu_context *prev_epc,

diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index dc641b50814e21e35df66d56874ef7610b2fd1bc..dafeee216f3b6ce886c17ddda5d41508a8b59858 100644 (file)
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -539,7 +539,8 @@ void intel_pmu_lbr_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
             task_context_opt(next_ctx_data)->lbr_callstack_users);
 }
 
-void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                             struct task_struct *task, bool sched_in)
 {
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        void *task_ctx;

diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index a698e6484b3b629f65e3b62345b94f4522d8560f..0d5019fb3ad237fdc9af2eb049972c70ca43fd38 100644 (file)
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -875,7 +875,7 @@ struct x86_pmu {
 
        void            (*check_microcode)(void);
        void            (*sched_task)(struct perf_event_pmu_context *pmu_ctx,
-                                     bool sched_in);
+                                     struct task_struct *task, bool sched_in);
 
        /*
         * Intel Arch Perfmon v2+
@@ -1408,7 +1408,8 @@ void amd_pmu_lbr_reset(void);
 void amd_pmu_lbr_read(void);
 void amd_pmu_lbr_add(struct perf_event *event);
 void amd_pmu_lbr_del(struct perf_event *event);
-void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
+void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                           struct task_struct *task, bool sched_in);
 void amd_pmu_lbr_enable_all(void);
 void amd_pmu_lbr_disable_all(void);
 int amd_pmu_lbr_hw_config(struct perf_event *event);
@@ -1462,7 +1463,8 @@ static inline void amd_pmu_brs_del(struct perf_event *event)
        perf_sched_cb_dec(event->pmu);
 }
 
-void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
+void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                           struct task_struct *task, bool sched_in);
 #else
 static inline int amd_brs_init(void)
 {
@@ -1487,7 +1489,8 @@ static inline void amd_pmu_brs_del(struct perf_event *event)
 {
 }
 
-static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
+static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                                         struct task_struct *task, bool sched_in)
 {
 }
 
@@ -1670,7 +1673,8 @@ void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
 void intel_pmu_lbr_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
                                 struct perf_event_pmu_context *next_epc);
 
-void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
+void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
+                             struct task_struct *task, bool sched_in);
 
 u64 lbr_from_signext_quirk_wr(u64 val);
 

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 2551170c0d18915b6678c9c7bb1ade1f89742a49..58f40c89d728ee2441c035d243f6b41f3edebc35 100644 (file)
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -494,7 +494,7 @@ struct pmu {
         * context-switches callback
         */
        void (*sched_task)              (struct perf_event_pmu_context *pmu_ctx,
-                                       bool sched_in);
+                                        struct task_struct *task, bool sched_in);
 
        /*
         * Kmem cache of PMU specific data

diff --git a/kernel/events/core.c b/kernel/events/core.c
index e86d35e4f27119c68a99d7c63fddb494ec2a0bcd..992829264577395820dc26c603c5f260f3e7d20b 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -3613,7 +3613,8 @@ static void perf_event_swap_task_ctx_data(struct perf_event_context *prev_ctx,
        }
 }
 
-static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, bool sched_in)
+static void perf_ctx_sched_task_cb(struct perf_event_context *ctx,
+                                  struct task_struct *task, bool sched_in)
 {
        struct perf_event_pmu_context *pmu_ctx;
        struct perf_cpu_pmu_context *cpc;
@@ -3622,7 +3623,7 @@ static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, bool sched_in
                cpc = this_cpc(pmu_ctx->pmu);
 
                if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)
-                       pmu_ctx->pmu->sched_task(pmu_ctx, sched_in);
+                       pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in);
        }
 }
 
@@ -3685,7 +3686,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
                        WRITE_ONCE(ctx->task, next);
                        WRITE_ONCE(next_ctx->task, task);
 
-                       perf_ctx_sched_task_cb(ctx, false);
+                       perf_ctx_sched_task_cb(ctx, task, false);
                        perf_event_swap_task_ctx_data(ctx, next_ctx);
 
                        perf_ctx_enable(ctx, false);
@@ -3715,7 +3716,7 @@ unlock:
                perf_ctx_disable(ctx, false);
 
 inside_switch:
-               perf_ctx_sched_task_cb(ctx, false);
+               perf_ctx_sched_task_cb(ctx, task, false);
                task_ctx_sched_out(ctx, NULL, EVENT_ALL);
 
                perf_ctx_enable(ctx, false);
@@ -3757,7 +3758,8 @@ void perf_sched_cb_inc(struct pmu *pmu)
  * PEBS requires this to provide PID/TID information. This requires we flush
  * all queued PEBS records before we context switch to a new task.
  */
-static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, bool sched_in)
+static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc,
+                                 struct task_struct *task, bool sched_in)
 {
        struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
        struct pmu *pmu;
@@ -3771,7 +3773,7 @@ static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, bool sched_i
        perf_ctx_lock(cpuctx, cpuctx->task_ctx);
        perf_pmu_disable(pmu);
 
-       pmu->sched_task(cpc->task_epc, sched_in);
+       pmu->sched_task(cpc->task_epc, task, sched_in);
 
        perf_pmu_enable(pmu);
        perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
@@ -3789,7 +3791,7 @@ static void perf_pmu_sched_task(struct task_struct *prev,
                return;
 
        list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry)
-               __perf_pmu_sched_task(cpc, sched_in);
+               __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in);
 }
 
 static void perf_event_switch(struct task_struct *task,
@@ -4088,7 +4090,7 @@ static void perf_event_context_sched_in(struct task_struct *task)
                perf_ctx_lock(cpuctx, ctx);
                perf_ctx_disable(ctx, false);
 
-               perf_ctx_sched_task_cb(ctx, true);
+               perf_ctx_sched_task_cb(ctx, task, true);
 
                perf_ctx_enable(ctx, false);
                perf_ctx_unlock(cpuctx, ctx);
@@ -4119,7 +4121,7 @@ static void perf_event_context_sched_in(struct task_struct *task)
 
        perf_event_sched_in(cpuctx, ctx, NULL);
 
-       perf_ctx_sched_task_cb(cpuctx->task_ctx, true);
+       perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true);
 
        if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
                perf_ctx_enable(&cpuctx->ctx, false);