sched/cache: Introduce per CPU's tasks LLC preference counter

The lowest level of sched domain for each CPU is assigned an
array where each element tracks the number of tasks preferring
a given LLC, indexed from 0 to max_lid. Since each CPU
has its dedicated sd, this implies that each CPU will have
a dedicated task LLC preference counter.

For example, sd->llc_counts[3] = 2 signifies that there
are 2 tasks on this runqueue which prefer to run within LLC3.

The load balancer can use this information to identify busy
runqueues and migrate tasks to their preferred LLC domains.
This array will be reallocated at runtime during sched domain
rebuild.

Introduce the buffer allocation mechanism, and the statistics
will be calculated in the subsequent patch.

Note: the LLC preference statistics of each CPU are reset on
sched domain rebuild and may under count temporarily, until the
CPU becomes idle and the count is cleared. This is a trade off
to avoid complex data synchronization across sched domain builds.

Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Suggested-by: K Prateek Nayak <kprateek.nayak@amd.com>
Co-developed-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/42e79eceb8cd6be8a032401d481d101913bc5703.1775065312.git.tim.c.chen@linux.intel.com

diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index 159716fa0d3aae8261def11e69d0cbda518baa84..0036d6b4bd67460d6a2907cba70d947a5a8b2a55 100644 (file)
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -103,6 +103,11 @@ struct sched_domain {
        u64 max_newidle_lb_cost;
        unsigned long last_decay_max_lb_cost;
 
+#ifdef CONFIG_SCHED_CACHE
+       unsigned int llc_max;
+       unsigned int *llc_counts __counted_by_ptr(llc_max);
+#endif
+
 #ifdef CONFIG_SCHEDSTATS
        /* sched_balance_rq() stats */
        unsigned int lb_count[CPU_MAX_IDLE_TYPES];

diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 1200670969bb0df6c16f6889c0e818718588a879..8954bf7900ffa48e5db30cf4aaeb75767304d134 100644 (file)
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -634,6 +634,11 @@ static void destroy_sched_domain(struct sched_domain *sd)
 
        if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
                kfree(sd->shared);
+
+#ifdef CONFIG_SCHED_CACHE
+       /* only the bottom sd has llc_counts array */
+       kfree(sd->llc_counts);
+#endif
        kfree(sd);
 }
 
@@ -763,10 +768,18 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
        if (sd && sd_degenerate(sd)) {
                tmp = sd;
                sd = sd->parent;
-               destroy_sched_domain(tmp);
+
                if (sd) {
                        struct sched_group *sg = sd->groups;
 
+#ifdef CONFIG_SCHED_CACHE
+                       /* move buffer to parent as child is being destroyed */
+                       sd->llc_counts = tmp->llc_counts;
+                       sd->llc_max = tmp->llc_max;
+                       /* make sure destroy_sched_domain() does not free it */
+                       tmp->llc_counts = NULL;
+                       tmp->llc_max = 0;
+#endif
                        /*
                         * sched groups hold the flags of the child sched
                         * domain for convenience. Clear such flags since
@@ -778,6 +791,8 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 
                        sd->child = NULL;
                }
+
+               destroy_sched_domain(tmp);
        }
 
        sched_domain_debug(sd, cpu);
@@ -805,6 +820,49 @@ enum s_alloc {
        sa_none,
 };
 
+#ifdef CONFIG_SCHED_CACHE
+static bool alloc_sd_llc(const struct cpumask *cpu_map,
+                        struct s_data *d)
+{
+       struct sched_domain *sd;
+       unsigned int *p;
+       int i;
+
+       for_each_cpu(i, cpu_map) {
+               sd = *per_cpu_ptr(d->sd, i);
+               if (!sd)
+                       goto err;
+
+               p = kcalloc_node(max_lid + 1, sizeof(unsigned int),
+                                GFP_KERNEL, cpu_to_node(i));
+               if (!p)
+                       goto err;
+
+               sd->llc_max = max_lid + 1;
+               sd->llc_counts = p;
+       }
+
+       return true;
+err:
+       for_each_cpu(i, cpu_map) {
+               sd = *per_cpu_ptr(d->sd, i);
+               if (sd) {
+                       kfree(sd->llc_counts);
+                       sd->llc_counts = NULL;
+                       sd->llc_max = 0;
+               }
+       }
+
+       return false;
+}
+#else
+static bool alloc_sd_llc(const struct cpumask *cpu_map,
+                        struct s_data *d)
+{
+       return false;
+}
+#endif
+
 /*
  * Return the canonical balance CPU for this group, this is the first CPU
  * of this group that's also in the balance mask.
@@ -2828,6 +2886,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
                        init_sched_groups_capacity(i, sd);
        }
 
+       alloc_sd_llc(cpu_map, &d);
+
        /* Attach the domains */
        rcu_read_lock();
        for_each_cpu(i, cpu_map) {