sched/cache: Track LLC-preferred tasks per runqueue

For each runqueue, track the number of tasks with an LLC preference
and how many of them are running on their preferred LLC. This mirrors
nr_numa_running and nr_preferred_running for NUMA balancing, and will
be used by cache-aware load balancing in later patches.

Co-developed-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/459a37102f3d74a4e09ea58401d2094ac731d044.1775065312.git.tim.c.chen@linux.intel.com

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index d11e27be76978c35d5d885a181a0c706b0b31bce..eb542c97266ad52bf8b25d3a5765773250d8c6c6 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -539,6 +539,11 @@ void __trace_set_current_state(int state_value)
 }
 EXPORT_SYMBOL(__trace_set_current_state);
 
+int task_llc(const struct task_struct *p)
+{
+       return per_cpu(sd_llc_id, task_cpu(p));
+}
+
 /*
  * Serialization rules:
  *

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6e78ecfb560e9ae0f94ca453d2d4b28da880e246..e66da7a6be3e6a893227ca890b532ed044ddcbcc 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1384,6 +1384,30 @@ static int llc_id(int cpu)
        return per_cpu(sd_llc_id, cpu);
 }
 
+static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
+{
+       int pref_llc;
+
+       pref_llc = p->preferred_llc;
+       if (pref_llc < 0)
+               return;
+
+       rq->nr_llc_running++;
+       rq->nr_pref_llc_running += (pref_llc == task_llc(p));
+}
+
+static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
+{
+       int pref_llc;
+
+       pref_llc = p->preferred_llc;
+       if (pref_llc < 0)
+               return;
+
+       rq->nr_llc_running--;
+       rq->nr_pref_llc_running -= (pref_llc == task_llc(p));
+}
+
 void mm_init_sched(struct mm_struct *mm,
                   struct sched_cache_time __percpu *_pcpu_sched)
 {
@@ -1490,6 +1514,8 @@ static int get_pref_llc(struct task_struct *p, struct mm_struct *mm)
        return mm_sched_llc;
 }
 
+static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
+
 static inline
 void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
 {
@@ -1530,8 +1556,13 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
 
        mm_sched_llc = get_pref_llc(p, mm);
 
-       if (READ_ONCE(p->preferred_llc) != mm_sched_llc)
+       /* task not on rq accounted later in account_entity_enqueue() */
+       if (task_running_on_cpu(rq->cpu, p) &&
+           READ_ONCE(p->preferred_llc) != mm_sched_llc) {
+               account_llc_dequeue(rq, p);
                WRITE_ONCE(p->preferred_llc, mm_sched_llc);
+               account_llc_enqueue(rq, p);
+       }
 }
 
 static void task_tick_cache(struct rq *rq, struct task_struct *p)
@@ -1714,6 +1745,10 @@ static inline int get_pref_llc(struct task_struct *p,
        return -1;
 }
 
+static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {}
+
+static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {}
+
 #endif /* CONFIG_SCHED_CACHE */
 
 /*
@@ -4200,9 +4235,11 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
        update_load_add(&cfs_rq->load, se->load.weight);
        if (entity_is_task(se)) {
+               struct task_struct *p = task_of(se);
                struct rq *rq = rq_of(cfs_rq);
 
-               account_numa_enqueue(rq, task_of(se));
+               account_numa_enqueue(rq, p);
+               account_llc_enqueue(rq, p);
                list_add(&se->group_node, &rq->cfs_tasks);
        }
        cfs_rq->nr_queued++;
@@ -4213,7 +4250,11 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
        update_load_sub(&cfs_rq->load, se->load.weight);
        if (entity_is_task(se)) {
-               account_numa_dequeue(rq_of(cfs_rq), task_of(se));
+               struct task_struct *p = task_of(se);
+               struct rq *rq = rq_of(cfs_rq);
+
+               account_numa_dequeue(rq, p);
+               account_llc_dequeue(rq, p);
                list_del_init(&se->group_node);
        }
        cfs_rq->nr_queued--;

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 3cb3ab02b1ebc0c286c8397fedb1ed0ef2912362..3c9e92b7904142a73458e8d52b458fbff7be35d0 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1196,6 +1196,12 @@ struct rq {
 #ifdef CONFIG_NUMA_BALANCING
        unsigned int            numa_migrate_on;
 #endif
+
+#ifdef CONFIG_SCHED_CACHE
+       unsigned int            nr_pref_llc_running;
+       unsigned int            nr_llc_running;
+#endif
+
        /*
         * This is part of a global counter where only the total sum
         * over all CPUs matters. A task can increase this counter on
@@ -2077,6 +2083,8 @@ init_numa_balancing(u64 clone_flags, struct task_struct *p)
 
 #endif /* !CONFIG_NUMA_BALANCING */
 
+int task_llc(const struct task_struct *p);
+
 static inline void
 queue_balance_callback(struct rq *rq,
                       struct balance_callback *head,