sched/cache: Assign preferred LLC ID to processes

With cache-aware scheduling enabled, each task is assigned a
preferred LLC ID. This allows quick identification of the LLC domain
where the task prefers to run, similar to numa_preferred_nid in
NUMA balancing.

Co-developed-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/f2ceecba5858680349ad4ce9303a2121f0bb7272.1775065312.git.tim.c.chen@linux.intel.com

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 2bf261bcd7b6fa844068b3602e3d8e58bb7fc2ce..d2010483cd77c43a877757fe16e68b9c56fcba78 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1409,6 +1409,7 @@ struct task_struct {
 
 #ifdef CONFIG_SCHED_CACHE
        struct callback_head            cache_work;
+       int                             preferred_llc;
 #endif
 
        struct rseq_data                rseq;

diff --git a/init/init_task.c b/init/init_task.c
index b5f48ebdc2b6eb21a61cc901f5b2b865865a4662..5d90db4ff1f8be196a64c098a2842c25a5ff6221 100644 (file)
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -215,6 +215,9 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
        .numa_group     = NULL,
        .numa_faults    = NULL,
 #endif
+#ifdef CONFIG_SCHED_CACHE
+       .preferred_llc  = -1,
+#endif
 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
        .kasan_depth    = 1,
 #endif

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7860c5bc12d7b18f12403fec0f23f418ad0c1fb3..6e78ecfb560e9ae0f94ca453d2d4b28da880e246 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1459,11 +1459,43 @@ static unsigned long fraction_mm_sched(struct rq *rq,
        return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1);
 }
 
+static int get_pref_llc(struct task_struct *p, struct mm_struct *mm)
+{
+       int mm_sched_llc = -1;
+
+       if (!mm)
+               return -1;
+
+       if (mm->sc_stat.cpu != -1) {
+               mm_sched_llc = llc_id(mm->sc_stat.cpu);
+
+#ifdef CONFIG_NUMA_BALANCING
+               /*
+                * Don't assign preferred LLC if it
+                * conflicts with NUMA balancing.
+                * This can happen when sched_setnuma() gets
+                * called, however it is not much of an issue
+                * because we expect account_mm_sched() to get
+                * called fairly regularly -- at a higher rate
+                * than sched_setnuma() at least -- and thus the
+                * conflict only exists for a short period of time.
+                */
+               if (static_branch_likely(&sched_numa_balancing) &&
+                   p->numa_preferred_nid >= 0 &&
+                   cpu_to_node(mm->sc_stat.cpu) != p->numa_preferred_nid)
+                       mm_sched_llc = -1;
+#endif
+       }
+
+       return mm_sched_llc;
+}
+
 static inline
 void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
 {
        struct sched_cache_time *pcpu_sched;
        struct mm_struct *mm = p->mm;
+       int mm_sched_llc = -1;
        unsigned long epoch;
 
        if (!sched_cache_enabled())
@@ -1495,6 +1527,11 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
                if (mm->sc_stat.cpu != -1)
                        mm->sc_stat.cpu = -1;
        }
+
+       mm_sched_llc = get_pref_llc(p, mm);
+
+       if (READ_ONCE(p->preferred_llc) != mm_sched_llc)
+               WRITE_ONCE(p->preferred_llc, mm_sched_llc);
 }
 
 static void task_tick_cache(struct rq *rq, struct task_struct *p)
@@ -1671,6 +1708,12 @@ void init_sched_mm(struct task_struct *p) { }
 
 static void task_tick_cache(struct rq *rq, struct task_struct *p) { }
 
+static inline int get_pref_llc(struct task_struct *p,
+                              struct mm_struct *mm)
+{
+       return -1;
+}
+
 #endif /* CONFIG_SCHED_CACHE */
 
 /*