sched/fair: Use throttled_csd_list for local unthrottle

When distribute_cfs_runtime() encounters a local cfs_rq, it adds it to a
local list and unthrottles it at the end, when it is done unthrottling
other cfs_rq(s) on cfs_b->throttled_cfs_rq until the bandwidth runs out.

Instead of using a local list, reuse the local CPU's
rq->throttled_csd_list and the __cfsb_csd_unthrottle() path for
unthrottle.

If this is the first cfs_rq to be queued on the "throttled_csd_list", it
prevents the need for a remote CPUs to interrupt this local CPU if they
themselves are performing async unthrottle.

If this is not the first cfs_rq on the list, there is an async unthrottle
operation pending on this local CPU and the unthrottle can be batched
together.

No functional changes intended.

Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Benjamin Segall <bsegall@google.com>
Tested-by: Aaron Lu <ziqianlu@bytedance.com>
Link: https://patch.msgid.link/20260602050005.11160-3-kprateek.nayak@amd.com

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 261e5cedc717c8081c3a6bc1643696e5354cf166..26a8bbb9e1e23d93b77d1feb67476987c0441aa2 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6991,12 +6991,11 @@ static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
 
 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
 {
+       bool throttled = false, unthrottle_local = false;
        int this_cpu = smp_processor_id();
        u64 runtime, remaining = 1;
-       bool throttled = false;
-       struct cfs_rq *cfs_rq, *tmp;
+       struct cfs_rq *cfs_rq;
        struct rq *rq;
-       LIST_HEAD(local_unthrottle);
 
        guard(rcu)();
 
@@ -7047,24 +7046,23 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
                }
 
                /*
-                * We currently only expect to be unthrottling
-                * a single cfs_rq locally.
+                * Allow a parallel async unthrottle to unthrottle
+                * this cfs_rq too via __cfsb_csd_unthrottle().
+                * If we are first, do it ourselves at the end and
+                * save on an IPI from remote CPUs.
                 */
-               WARN_ON_ONCE(!list_empty(&local_unthrottle));
-               list_add_tail(&cfs_rq->throttled_csd_list, &local_unthrottle);
+               unthrottle_local = list_empty(&rq->cfsb_csd_list);
+               list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list);
        }
 
-       list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle,
-                                throttled_csd_list) {
-               struct rq *rq = rq_of(cfs_rq);
-
-               guard(rq_lock_irqsave)(rq);
-
-               list_del_init(&cfs_rq->throttled_csd_list);
-               if (cfs_rq_throttled(cfs_rq))
-                       unthrottle_cfs_rq(cfs_rq);
+       if (unthrottle_local) {
+               /*
+                * Protect against an IPI that is also trying to flush
+                * the unthrottled cfs_rq(s) from this CPU's csd_list.
+                */
+               scoped_guard(irqsave)
+                       __cfsb_csd_unthrottle(cpu_rq(this_cpu));
        }
-       WARN_ON_ONCE(!list_empty(&local_unthrottle));
 
        return throttled;
 }