hrtimer: Optimize for local timers

The decision whether to keep timers on the local CPU or on the CPU they are
associated to is suboptimal and causes the expensive switch_hrtimer_base()
mechanism to be invoked more than necessary. This is especially true for
pinned timers.

Rewrite the decision logic so that the current base is kept if:

   1) The callback is running on the base

   2) The timer is associated to the local CPU and the first expiring timer as
      that allows to optimize for reprogramming avoidance

   3) The timer is associated to the local CPU and pinned

   4) The timer is associated to the local CPU and timer migration is
      disabled.

Only #2 was covered by the original code, but especially #3 makes a
difference for high frequency rearming timers like the scheduler hrtick
timer. If timer migration is disabled, then #4 avoids most of the base
switches.

Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163430.607935269@kernel.org

diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 6bab3b7eb0dee07e649c19ee724ed63f8018a62a..b87995f3ae2a8807d3953707b2b7d8f808778a91 100644 (file)
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -1147,7 +1147,7 @@ static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *b
 }
 
 static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
-                                bool restart, bool keep_local)
+                                bool restart, bool keep_base)
 {
        bool queued_state = timer->is_queued;
 
@@ -1177,7 +1177,7 @@ static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_ba
                if (!restart)
                        queued_state = HRTIMER_STATE_INACTIVE;
                else
-                       reprogram &= !keep_local;
+                       reprogram &= !keep_base;
 
                __remove_hrtimer(timer, base, queued_state, reprogram);
                return true;
@@ -1220,29 +1220,57 @@ static void hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool
        hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
 }
 
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned)
+{
+       if (static_branch_likely(&timers_migration_enabled)) {
+               /*
+                * If it is local and the first expiring timer keep it on the local
+                * CPU to optimize reprogramming of the clockevent device. Also
+                * avoid switch_hrtimer_base() overhead when local and pinned.
+                */
+               if (!is_local)
+                       return false;
+               return is_first || is_pinned;
+       }
+       return is_local;
+}
+#else
+static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned)
+{
+       return is_local;
+}
+#endif
+
+static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool is_first,
+                                    bool is_pinned)
+{
+       /* If the timer is running the callback it has to stay on its CPU base. */
+       if (unlikely(timer->base->running == timer))
+               return true;
+
+       return hrtimer_prefer_local(is_local, is_first, is_pinned);
+}
+
 static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
                                     const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
 {
        struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
-       struct hrtimer_clock_base *new_base;
-       bool force_local, first, was_armed;
+       bool is_pinned, first, was_first, was_armed, keep_base = false;
+       struct hrtimer_cpu_base *cpu_base = base->cpu_base;
 
-       /*
-        * If the timer is on the local cpu base and is the first expiring
-        * timer then this might end up reprogramming the hardware twice
-        * (on removal and on enqueue). To avoid that prevent the reprogram
-        * on removal, keep the timer local to the current CPU and enforce
-        * reprogramming after it is queued no matter whether it is the new
-        * first expiring timer again or not.
-        */
-       force_local = base->cpu_base == this_cpu_base;
-       force_local &= base->cpu_base->next_timer == timer;
+       was_first = cpu_base->next_timer == timer;
+       is_pinned = !!(mode & HRTIMER_MODE_PINNED);
 
        /*
-        * Don't force local queuing if this enqueue happens on a unplugged
-        * CPU after hrtimer_cpu_dying() has been invoked.
+        * Don't keep it local if this enqueue happens on a unplugged CPU
+        * after hrtimer_cpu_dying() has been invoked.
         */
-       force_local &= this_cpu_base->online;
+       if (likely(this_cpu_base->online)) {
+               bool is_local = cpu_base == this_cpu_base;
+
+               keep_base = hrtimer_keep_base(timer, is_local, was_first, is_pinned);
+       }
 
        /*
         * Remove an active timer from the queue. In case it is not queued
@@ -1254,8 +1282,11 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
         * reprogramming later if it was the first expiring timer.  This
         * avoids programming the underlying clock event twice (once at
         * removal and once after enqueue).
+        *
+        * @keep_base is also true if the timer callback is running on a
+        * remote CPU and for local pinned timers.
         */
-       was_armed = remove_hrtimer(timer, base, true, force_local);
+       was_armed = remove_hrtimer(timer, base, true, keep_base);
 
        if (mode & HRTIMER_MODE_REL)
                tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
@@ -1265,21 +1296,21 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
        hrtimer_set_expires_range_ns(timer, tim, delta_ns);
 
        /* Switch the timer base, if necessary: */
-       if (!force_local)
-               new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
-       else
-               new_base = base;
+       if (!keep_base) {
+               base = switch_hrtimer_base(timer, base, is_pinned);
+               cpu_base = base->cpu_base;
+       }
 
-       first = enqueue_hrtimer(timer, new_base, mode, was_armed);
+       first = enqueue_hrtimer(timer, base, mode, was_armed);
 
        /*
         * If the hrtimer interrupt is running, then it will reevaluate the
         * clock bases and reprogram the clock event device.
         */
-       if (new_base->cpu_base->in_hrtirq)
+       if (cpu_base->in_hrtirq)
                return false;
 
-       if (!force_local) {
+       if (!was_first || cpu_base != this_cpu_base) {
                /*
                 * If the current CPU base is online, then the timer is never
                 * queued on a remote CPU if it would be the first expiring
@@ -1288,7 +1319,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
                 * re-evaluate the first expiring timer after completing the
                 * callbacks.
                 */
-               if (hrtimer_base_is_online(this_cpu_base))
+               if (likely(hrtimer_base_is_online(this_cpu_base)))
                        return first;
 
                /*
@@ -1296,11 +1327,8 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
                 * already offline. If the timer is the first to expire,
                 * kick the remote CPU to reprogram the clock event.
                 */
-               if (first) {
-                       struct hrtimer_cpu_base *new_cpu_base = new_base->cpu_base;
-
-                       smp_call_function_single_async(new_cpu_base->cpu, &new_cpu_base->csd);
-               }
+               if (first)
+                       smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd);
                return false;
        }
 
@@ -1314,16 +1342,17 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
         * required.
         */
        if (timer->is_lazy) {
-               if (new_base->cpu_base->expires_next <= hrtimer_get_expires(timer))
+               if (cpu_base->expires_next <= hrtimer_get_expires(timer))
                        return false;
        }
 
        /*
-        * Timer was forced to stay on the current CPU to avoid
-        * reprogramming on removal and enqueue. Force reprogram the
-        * hardware by evaluating the new first expiring timer.
+        * Timer was the first expiring timer and forced to stay on the
+        * current CPU to avoid reprogramming on removal and enqueue. Force
+        * reprogram the hardware by evaluating the new first expiring
+        * timer.
         */
-       hrtimer_force_reprogram(new_base->cpu_base, /* skip_equal */ true);
+       hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
        return false;
 }