hrtimer: Try to modify timers in place

When modifying the expiry of a armed timer it is first dequeued, then the
expiry value is updated and then it is queued again.

This can be avoided when the new expiry value is within the range of the
previous and the next timer as that does not change the position in the RB
tree.

The linked timerqueue allows to peak ahead to the neighbours and check
whether the new expiry time is within the range of the previous and next
timer. If so just modify the timer in place and spare the enqueue and
requeue effort, which might end up rotating the RB tree twice for nothing.

This speeds up the handling of frequently rearmed hrtimers, like the hrtick
scheduler timer significantly.

Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163431.873359816@kernel.org

diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 5e45982363ce0a505a5c1091aa8098e9199831a2..b94bd56b739f61b1cd3790a0ac1948baf0e4d653 100644 (file)
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -1203,6 +1203,31 @@ static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_ba
        return false;
 }
 
+/*
+ * Update in place has to retrieve the expiry times of the neighbour nodes
+ * if they exist. That is cache line neutral because the dequeue/enqueue
+ * operation is going to need the same cache lines. But there is a big win
+ * when the dequeue/enqueue can be avoided because the RB tree does not
+ * have to be rebalanced twice.
+ */
+static inline bool
+hrtimer_can_update_in_place(struct hrtimer *timer, struct hrtimer_clock_base *base, ktime_t expires)
+{
+       struct timerqueue_linked_node *next = timerqueue_linked_next(&timer->node);
+       struct timerqueue_linked_node *prev = timerqueue_linked_prev(&timer->node);
+
+       /* If the new expiry goes behind the next timer, requeue is required */
+       if (next && expires > next->expires)
+               return false;
+
+       /* If this is the first timer, update in place */
+       if (!prev)
+               return true;
+
+       /* Update in place when it does not go ahead of the previous one */
+       return expires >= prev->expires;
+}
+
 static inline bool
 remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
                             const enum hrtimer_mode mode, ktime_t expires, u64 delta_ns)
@@ -1211,8 +1236,18 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b
 
        /* Remove it from the timer queue if active */
        if (timer->is_queued) {
-               debug_hrtimer_deactivate(timer);
                was_first = !timerqueue_linked_prev(&timer->node);
+
+               /* Try to update in place to avoid the de/enqueue dance */
+               if (hrtimer_can_update_in_place(timer, base, expires)) {
+                       hrtimer_set_expires_range_ns(timer, expires, delta_ns);
+                       trace_hrtimer_start(timer, mode, true);
+                       if (was_first)
+                               base->expires_next = expires;
+                       return was_first;
+               }
+
+               debug_hrtimer_deactivate(timer);
                timerqueue_linked_del(&base->active, &timer->node);
        }