sched/fair: Add EAS checks before updating root_domain::overutilized

root_domain::overutilized is only used for EAS(energy aware scheduler)
to decide whether to do load balance or not. It is not used if EAS
not possible.

Currently enqueue_task_fair and task_tick_fair accesses, sometime updates
this field. In update_sd_lb_stats it is updated often. This causes cache
contention due to true sharing and burns a lot of cycles. ::overload and
::overutilized are part of the same cacheline. Updating it often invalidates
the cacheline. That causes access to ::overload to slow down due to
false sharing. Hence add EAS check before accessing/updating this field.
EAS check is optimized at compile time or it is a static branch.
Hence it shouldn't cost much.

With the patch, both enqueue_task_fair and newidle_balance don't show
up as hot routines in perf profile.

  6.8-rc4:
  7.18%  swapper          [kernel.vmlinux]              [k] enqueue_task_fair
  6.78%  s                [kernel.vmlinux]              [k] newidle_balance

  +patch:
  0.14%  swapper          [kernel.vmlinux]              [k] enqueue_task_fair
  0.00%  swapper          [kernel.vmlinux]              [k] newidle_balance

While at it: trace_sched_overutilized_tp expect that second argument to
be bool. So do a int to bool conversion for that.

Fixes: 2802bf3cd936 ("sched/fair: Add over-utilization/tipping point indicator")
Signed-off-by: Shrikanth Hegde <sshegde@linux.ibm.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Qais Yousef <qyousef@layalina.io>
Reviewed-by: Srikar Dronamraju <srikar@linux.ibm.com>
Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lore.kernel.org/r/20240307085725.444486-2-sshegde@linux.ibm.com

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index dbf4f1c442594071de12c768e05f4f65b89d4d54..1afa4f87fb2516f1784ff13c096660883b99dce1 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6673,22 +6673,42 @@ static inline void hrtick_update(struct rq *rq)
 #ifdef CONFIG_SMP
 static inline bool cpu_overutilized(int cpu)
 {
-       unsigned long rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN);
-       unsigned long rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);
+       unsigned long  rq_util_min, rq_util_max;
+
+       if (!sched_energy_enabled())
+               return false;
+
+       rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN);
+       rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);
 
        /* Return true only if the utilization doesn't fit CPU's capacity */
        return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu);
 }
 
-static inline void update_overutilized_status(struct rq *rq)
+static inline void set_rd_overutilized_status(struct root_domain *rd,
+                                             unsigned int status)
 {
-       if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) {
-               WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED);
-               trace_sched_overutilized_tp(rq->rd, SG_OVERUTILIZED);
-       }
+       if (!sched_energy_enabled())
+               return;
+
+       WRITE_ONCE(rd->overutilized, status);
+       trace_sched_overutilized_tp(rd, !!status);
+}
+
+static inline void check_update_overutilized_status(struct rq *rq)
+{
+       /*
+        * overutilized field is used for load balancing decisions only
+        * if energy aware scheduler is being used
+        */
+       if (!sched_energy_enabled())
+               return;
+
+       if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu))
+               set_rd_overutilized_status(rq->rd, SG_OVERUTILIZED);
 }
 #else
-static inline void update_overutilized_status(struct rq *rq) { }
+static inline void check_update_overutilized_status(struct rq *rq) { }
 #endif
 
 /* Runqueue only has SCHED_IDLE tasks enqueued */
@@ -6789,7 +6809,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
         * and the following generally works well enough in practice.
         */
        if (!task_new)
-               update_overutilized_status(rq);
+               check_update_overutilized_status(rq);
 
 enqueue_throttle:
        assert_list_leaf_cfs_rq(rq);
@@ -10630,19 +10650,14 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd
                env->fbq_type = fbq_classify_group(&sds->busiest_stat);
 
        if (!env->sd->parent) {
-               struct root_domain *rd = env->dst_rq->rd;
-
                /* update overload indicator if we are at root domain */
-               WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD);
+               WRITE_ONCE(env->dst_rq->rd->overload, sg_status & SG_OVERLOAD);
 
                /* Update over-utilization (tipping point, U >= 0) indicator */
-               WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED);
-               trace_sched_overutilized_tp(rd, sg_status & SG_OVERUTILIZED);
+               set_rd_overutilized_status(env->dst_rq->rd,
+                                          sg_status & SG_OVERUTILIZED);
        } else if (sg_status & SG_OVERUTILIZED) {
-               struct root_domain *rd = env->dst_rq->rd;
-
-               WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED);
-               trace_sched_overutilized_tp(rd, SG_OVERUTILIZED);
+               set_rd_overutilized_status(env->dst_rq->rd, SG_OVERUTILIZED);
        }
 
        update_idle_cpu_scan(env, sum_util);
@@ -12667,7 +12682,7 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
                task_tick_numa(rq, curr);
 
        update_misfit_status(curr, rq);
-       update_overutilized_status(task_rq(curr));
+       check_update_overutilized_status(task_rq(curr));
 
        task_tick_core(rq, curr);
 }