perf: Clean up perf ctx time

The current perf tracks two timestamps for the normal ctx and cgroup.
The same type of variables and similar codes are used to track the
timestamps. In the following patch, the third timestamp to track the
guest time will be introduced.
To avoid the code duplication, add a new struct perf_time_ctx and factor
out a generic function update_perf_time_ctx().

No functional change.

Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Mingwei Zhang <mizhang@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: Xudong Hao <xudong.hao@intel.com>
Link: https://patch.msgid.link/20251206001720.468579-6-seanjc@google.com

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 31929da6e711101cb755183d4ac94dd493d72d41..d5aa1bc3f0884b42f6d34dd62eeee986f7df179d 100644 (file)
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -999,6 +999,11 @@ struct perf_event_groups {
        u64                             index;
 };
 
+struct perf_time_ctx {
+       u64             time;
+       u64             stamp;
+       u64             offset;
+};
 
 /**
  * struct perf_event_context - event context structure
@@ -1037,9 +1042,7 @@ struct perf_event_context {
        /*
         * Context clock, runs when context enabled.
         */
-       u64                             time;
-       u64                             timestamp;
-       u64                             timeoffset;
+       struct perf_time_ctx            time;
 
        /*
         * These fields let us detect when two contexts have both
@@ -1172,9 +1175,7 @@ struct bpf_perf_event_data_kern {
  * This is a per-cpu dynamically allocated data structure.
  */
 struct perf_cgroup_info {
-       u64                             time;
-       u64                             timestamp;
-       u64                             timeoffset;
+       struct perf_time_ctx            time;
        int                             active;
 };
 

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 5a2166ba61389d3684c4af089871c352edc34d12..95f118230ff5ec35b30f89ef3fa2bfa150854262 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -816,6 +816,24 @@ static void perf_ctx_enable(struct perf_event_context *ctx,
 static void ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type);
 static void ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type);
 
+static inline void update_perf_time_ctx(struct perf_time_ctx *time, u64 now, bool adv)
+{
+       if (adv)
+               time->time += now - time->stamp;
+       time->stamp = now;
+
+       /*
+        * The above: time' = time + (now - timestamp), can be re-arranged
+        * into: time` = now + (time - timestamp), which gives a single value
+        * offset to compute future time without locks on.
+        *
+        * See perf_event_time_now(), which can be used from NMI context where
+        * it's (obviously) not possible to acquire ctx->lock in order to read
+        * both the above values in a consistent manner.
+        */
+       WRITE_ONCE(time->offset, time->time - time->stamp);
+}
+
 #ifdef CONFIG_CGROUP_PERF
 
 static inline bool
@@ -857,7 +875,7 @@ static inline u64 perf_cgroup_event_time(struct perf_event *event)
        struct perf_cgroup_info *t;
 
        t = per_cpu_ptr(event->cgrp->info, event->cpu);
-       return t->time;
+       return t->time.time;
 }
 
 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
@@ -866,22 +884,11 @@ static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
 
        t = per_cpu_ptr(event->cgrp->info, event->cpu);
        if (!__load_acquire(&t->active))
-               return t->time;
-       now += READ_ONCE(t->timeoffset);
+               return t->time.time;
+       now += READ_ONCE(t->time.offset);
        return now;
 }
 
-static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv)
-{
-       if (adv)
-               info->time += now - info->timestamp;
-       info->timestamp = now;
-       /*
-        * see update_context_time()
-        */
-       WRITE_ONCE(info->timeoffset, info->time - info->timestamp);
-}
-
 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
 {
        struct perf_cgroup *cgrp = cpuctx->cgrp;
@@ -895,7 +902,7 @@ static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
                        cgrp = container_of(css, struct perf_cgroup, css);
                        info = this_cpu_ptr(cgrp->info);
 
-                       __update_cgrp_time(info, now, true);
+                       update_perf_time_ctx(&info->time, now, true);
                        if (final)
                                __store_release(&info->active, 0);
                }
@@ -918,7 +925,7 @@ static inline void update_cgrp_time_from_event(struct perf_event *event)
         * Do not update time when cgroup is not active
         */
        if (info->active)
-               __update_cgrp_time(info, perf_clock(), true);
+               update_perf_time_ctx(&info->time, perf_clock(), true);
 }
 
 static inline void
@@ -942,7 +949,7 @@ perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
        for (css = &cgrp->css; css; css = css->parent) {
                cgrp = container_of(css, struct perf_cgroup, css);
                info = this_cpu_ptr(cgrp->info);
-               __update_cgrp_time(info, ctx->timestamp, false);
+               update_perf_time_ctx(&info->time, ctx->time.stamp, false);
                __store_release(&info->active, 1);
        }
 }
@@ -1563,20 +1570,7 @@ static void __update_context_time(struct perf_event_context *ctx, bool adv)
 
        lockdep_assert_held(&ctx->lock);
 
-       if (adv)
-               ctx->time += now - ctx->timestamp;
-       ctx->timestamp = now;
-
-       /*
-        * The above: time' = time + (now - timestamp), can be re-arranged
-        * into: time` = now + (time - timestamp), which gives a single value
-        * offset to compute future time without locks on.
-        *
-        * See perf_event_time_now(), which can be used from NMI context where
-        * it's (obviously) not possible to acquire ctx->lock in order to read
-        * both the above values in a consistent manner.
-        */
-       WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);
+       update_perf_time_ctx(&ctx->time, now, adv);
 }
 
 static void update_context_time(struct perf_event_context *ctx)
@@ -1594,7 +1588,7 @@ static u64 perf_event_time(struct perf_event *event)
        if (is_cgroup_event(event))
                return perf_cgroup_event_time(event);
 
-       return ctx->time;
+       return ctx->time.time;
 }
 
 static u64 perf_event_time_now(struct perf_event *event, u64 now)
@@ -1608,9 +1602,9 @@ static u64 perf_event_time_now(struct perf_event *event, u64 now)
                return perf_cgroup_event_time_now(event, now);
 
        if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
-               return ctx->time;
+               return ctx->time.time;
 
-       now += READ_ONCE(ctx->timeoffset);
+       now += READ_ONCE(ctx->time.offset);
        return now;
 }
 
@@ -12113,7 +12107,7 @@ static void task_clock_event_update(struct perf_event *event, u64 now)
 static void task_clock_event_start(struct perf_event *event, int flags)
 {
        event->hw.state = 0;
-       local64_set(&event->hw.prev_count, event->ctx->time);
+       local64_set(&event->hw.prev_count, event->ctx->time.time);
        perf_swevent_start_hrtimer(event);
 }
 
@@ -12122,7 +12116,7 @@ static void task_clock_event_stop(struct perf_event *event, int flags)
        event->hw.state = PERF_HES_STOPPED;
        perf_swevent_cancel_hrtimer(event);
        if (flags & PERF_EF_UPDATE)
-               task_clock_event_update(event, event->ctx->time);
+               task_clock_event_update(event, event->ctx->time.time);
 }
 
 static int task_clock_event_add(struct perf_event *event, int flags)
@@ -12142,8 +12136,8 @@ static void task_clock_event_del(struct perf_event *event, int flags)
 static void task_clock_event_read(struct perf_event *event)
 {
        u64 now = perf_clock();
-       u64 delta = now - event->ctx->timestamp;
-       u64 time = event->ctx->time + delta;
+       u64 delta = now - event->ctx->time.stamp;
+       u64 time = event->ctx->time.time + delta;
 
        task_clock_event_update(event, time);
 }