sched/cache: Allow only 1 thread of the process to calculate the LLC occupancy

Scanning online CPUs to calculate the occupancy might be
time-consuming. Only allow 1 thread of the process to scan
the CPUs at the same time, which is similar to what
NUMA balance does in task_numa_work().

Signed-off-by: Jianyong Wu <wujianyong@hygon.cn>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/5672b52e588b855b01e5a1a17822f7c6c7237a3d.1778703694.git.tim.c.chen@linux.intel.com

diff --git a/include/linux/sched.h b/include/linux/sched.h
index d2010483cd77c43a877757fe16e68b9c56fcba78..6d883f109ba378f7a1076c2b8a4188a1632072bd 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2423,6 +2423,7 @@ struct sched_cache_stat {
        struct sched_cache_time __percpu *pcpu_sched;
        raw_spinlock_t lock;
        unsigned long epoch;
+       unsigned long next_scan;
        int cpu;
 } ____cacheline_aligned_in_smp;
 

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 5f22e5a097cfe399c158513b6b0c6b3949fc795b..a759ea669d74d257573af727d0c3370b47e4a79b 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1451,6 +1451,7 @@ void mm_init_sched(struct mm_struct *mm,
        raw_spin_lock_init(&mm->sc_stat.lock);
        mm->sc_stat.epoch = epoch;
        mm->sc_stat.cpu = -1;
+       mm->sc_stat.next_scan = jiffies;
 
        /*
         * The update to mm->sc_stat should not be reordered
@@ -1661,6 +1662,7 @@ out:
 
 static void task_cache_work(struct callback_head *work)
 {
+       unsigned long next_scan, now = jiffies;
        struct task_struct *p = current;
        struct mm_struct *mm = p->mm;
        unsigned long m_a_occ = 0;
@@ -1675,6 +1677,15 @@ static void task_cache_work(struct callback_head *work)
        if (p->flags & PF_EXITING)
                return;
 
+       next_scan = READ_ONCE(mm->sc_stat.next_scan);
+       if (time_before(now, next_scan))
+               return;
+
+       /* only 1 thread is allowed to scan */
+       if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan,
+                        now + EPOCH_PERIOD))
+               return;
+
        if (!zalloc_cpumask_var(&cpus, GFP_KERNEL))
                return;