sched/topology: Introduce sched_update_asym_prefer_cpu()

A subset of AMD Processors supporting Preferred Core Rankings also
feature the ability to dynamically switch these rankings at runtime to
bias load balancing towards or away from the LLC domain with larger
cache.

To support dynamically updating "sg->asym_prefer_cpu" without needing to
rebuild the sched domain, introduce sched_update_asym_prefer_cpu() which
recomutes the "asym_prefer_cpu" when the core-ranking of a CPU changes.

sched_update_asym_prefer_cpu() swaps the "sg->asym_prefer_cpu" with the
CPU whose ranking has changed if the new ranking is greater than that of
the "asym_prefer_cpu". If CPU whose ranking has changed is the current
"asym_prefer_cpu", it scans the CPUs of the sched groups to find the new
"asym_prefer_cpu" and sets it accordingly.

get_group() for non-overlapping sched domains returns the sched group
for the first CPU in the sched_group_span() which ensures all CPUs in
the group see the updated value of "asym_prefer_cpu".

Overlapping groups are allocated differently and will require moving the
"asym_prefer_cpu" to "sg->sgc" but since the current implementations do
not set "SD_ASYM_PACKING" at NUMA domains, skip additional
indirection and place a SCHED_WARN_ON() to alert any future users.

Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250409053446.23367-3-kprateek.nayak@amd.com

diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index 7b4301b7235f6df3ced1964e0de9afd9e867b3d0..198bb5cc1774b3c6985af9b8d629f7f6e4280eb4 100644 (file)
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -195,6 +195,8 @@ struct sched_domain_topology_level {
 };
 
 extern void __init set_sched_topology(struct sched_domain_topology_level *tl);
+extern void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio);
+
 
 # define SD_INIT_NAME(type)            .name = #type
 
@@ -223,6 +225,10 @@ static inline bool cpus_share_resources(int this_cpu, int that_cpu)
        return true;
 }
 
+static inline void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio)
+{
+}
+
 #endif /* !CONFIG_SMP */
 
 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)

diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index bbc2fc2c7c22441807f2aa13314d964c6848e76a..a2a38e1b6f188fc1d93f854eefecb6a3625c508e 100644 (file)
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -1333,6 +1333,64 @@ next:
        update_group_capacity(sd, cpu);
 }
 
+#ifdef CONFIG_SMP
+
+/* Update the "asym_prefer_cpu" when arch_asym_cpu_priority() changes. */
+void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio)
+{
+       int asym_prefer_cpu = cpu;
+       struct sched_domain *sd;
+
+       guard(rcu)();
+
+       for_each_domain(cpu, sd) {
+               struct sched_group *sg;
+               int group_cpu;
+
+               if (!(sd->flags & SD_ASYM_PACKING))
+                       continue;
+
+               /*
+                * Groups of overlapping domain are replicated per NUMA
+                * node and will require updating "asym_prefer_cpu" on
+                * each local copy.
+                *
+                * If you are hitting this warning, consider moving
+                * "sg->asym_prefer_cpu" to "sg->sgc->asym_prefer_cpu"
+                * which is shared by all the overlapping groups.
+                */
+               WARN_ON_ONCE(sd->flags & SD_OVERLAP);
+
+               sg = sd->groups;
+               if (cpu != sg->asym_prefer_cpu) {
+                       /*
+                        * Since the parent is a superset of the current group,
+                        * if the cpu is not the "asym_prefer_cpu" at the
+                        * current level, it cannot be the preferred CPU at a
+                        * higher levels either.
+                        */
+                       if (!sched_asym_prefer(cpu, sg->asym_prefer_cpu))
+                               return;
+
+                       WRITE_ONCE(sg->asym_prefer_cpu, cpu);
+                       continue;
+               }
+
+               /* Ranking has improved; CPU is still the preferred one. */
+               if (new_prio >= old_prio)
+                       continue;
+
+               for_each_cpu(group_cpu, sched_group_span(sg)) {
+                       if (sched_asym_prefer(group_cpu, asym_prefer_cpu))
+                               asym_prefer_cpu = group_cpu;
+               }
+
+               WRITE_ONCE(sg->asym_prefer_cpu, asym_prefer_cpu);
+       }
+}
+
+#endif /* CONFIG_SMP */
+
 /*
  * Set of available CPUs grouped by their corresponding capacities
  * Each list entry contains a CPU mask reflecting CPUs that share the same