sched/fair: Drop redundant RCU read lock in NOHZ kick path

nohz_balancer_kick() is reached from sched_balance_trigger(), which is
called from sched_tick(). sched_tick() runs with IRQs disabled, so the
additional rcu_read_lock/unlock() used around sched_domain accesses in
this path is redundant. Rely on the existing IRQ-disabled context (and
the rcu_dereference_all() checking) instead.

The same applies to set_cpu_sd_state_idle(), called from the idle entry
path with IRQs disabled, and to set_cpu_sd_state_busy(), reachable via
nohz_balance_exit_idle() from two contexts: nohz_balancer_kick() (IRQs
disabled, as above) and sched_cpu_deactivate() (the CPUHP_AP_ACTIVE
teardown, which runs under cpus_write_lock(), so it cannot race with
sched-domain rebuilds). In both cases the rcu_dereference_all()
validation is sufficient.

No functional change intended.

Suggested-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Andrea Righi <arighi@nvidia.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: K Prateek Nayak <kprateek.nayak@amd.com>
Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://patch.msgid.link/20260509180955.1840064-2-arighi@nvidia.com

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index bcaadddf862408b27cc52d5095fea86d278ce33a..03f63b094ff984090a47cf73b51b744e266bfc9f 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -12715,8 +12715,6 @@ static void nohz_balancer_kick(struct rq *rq)
                goto out;
        }
 
-       rcu_read_lock();
-
        sd = rcu_dereference_all(rq->sd);
        if (sd) {
                /*
@@ -12724,8 +12722,8 @@ static void nohz_balancer_kick(struct rq *rq)
                 * capacity, kick the ILB to see if there's a better CPU to run on:
                 */
                if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) {
-                       flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
-                       goto unlock;
+                       flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
+                       goto out;
                }
        }
 
@@ -12741,8 +12739,8 @@ static void nohz_balancer_kick(struct rq *rq)
                 */
                for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
                        if (sched_asym(sd, i, cpu)) {
-                               flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
-                               goto unlock;
+                               flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
+                               goto out;
                        }
                }
        }
@@ -12753,10 +12751,8 @@ static void nohz_balancer_kick(struct rq *rq)
                 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
                 * to run the misfit task on.
                 */
-               if (check_misfit_status(rq)) {
-                       flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
-                       goto unlock;
-               }
+               if (check_misfit_status(rq))
+                       flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
 
                /*
                 * For asymmetric systems, we do not want to nicely balance
@@ -12765,7 +12761,7 @@ static void nohz_balancer_kick(struct rq *rq)
                 *
                 * Skip the LLC logic because it's not relevant in that case.
                 */
-               goto unlock;
+               goto out;
        }
 
        sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
@@ -12780,13 +12776,9 @@ static void nohz_balancer_kick(struct rq *rq)
                 * like this LLC domain has tasks we could move.
                 */
                nr_busy = atomic_read(&sds->nr_busy_cpus);
-               if (nr_busy > 1) {
-                       flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
-                       goto unlock;
-               }
+               if (nr_busy > 1)
+                       flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
        }
-unlock:
-       rcu_read_unlock();
 out:
        if (READ_ONCE(nohz.needs_update))
                flags |= NOHZ_NEXT_KICK;
@@ -12798,17 +12790,13 @@ out:
 static void set_cpu_sd_state_busy(int cpu)
 {
        struct sched_domain *sd;
-
-       rcu_read_lock();
        sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
 
        if (!sd || !sd->nohz_idle)
-               goto unlock;
+               return;
        sd->nohz_idle = 0;
 
        atomic_inc(&sd->shared->nr_busy_cpus);
-unlock:
-       rcu_read_unlock();
 }
 
 void nohz_balance_exit_idle(struct rq *rq)
@@ -12827,17 +12815,13 @@ void nohz_balance_exit_idle(struct rq *rq)
 static void set_cpu_sd_state_idle(int cpu)
 {
        struct sched_domain *sd;
-
-       rcu_read_lock();
        sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
 
        if (!sd || sd->nohz_idle)
-               goto unlock;
+               return;
        sd->nohz_idle = 1;
 
        atomic_dec(&sd->shared->nr_busy_cpus);
-unlock:
-       rcu_read_unlock();
 }
 
 /*