sched/cache: Allow the user space to turn on and off cache aware scheduling

Provide a debugfs directory llc_balancing, and a knob named
"enabled" under it to allow the user to turn off and on the
cache aware scheduling at runtime.

Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/0aa56f7fc48db2f8f700cd1aa34dedd0ec88351b.1775065312.git.tim.c.chen@linux.intel.com

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 74c1617cf65234cc78bd98ec70eecc0e8bccf83f..2eae67cd2ba2891da4062c7eb573ae4a340f031f 100644 (file)
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -210,6 +210,46 @@ static const struct file_operations sched_scaling_fops = {
        .release        = single_release,
 };
 
+#ifdef CONFIG_SCHED_CACHE
+static ssize_t
+sched_cache_enable_write(struct file *filp, const char __user *ubuf,
+                        size_t cnt, loff_t *ppos)
+{
+       bool val;
+       int ret;
+
+       ret = kstrtobool_from_user(ubuf, cnt, &val);
+       if (ret)
+               return ret;
+
+       sysctl_sched_cache_user = val;
+
+       sched_cache_active_set_unlocked();
+
+       return cnt;
+}
+
+static int sched_cache_enable_show(struct seq_file *m, void *v)
+{
+       seq_printf(m, "%d\n", sysctl_sched_cache_user);
+       return 0;
+}
+
+static int sched_cache_enable_open(struct inode *inode,
+                                  struct file *filp)
+{
+       return single_open(filp, sched_cache_enable_show, NULL);
+}
+
+static const struct file_operations sched_cache_enable_fops = {
+       .open           = sched_cache_enable_open,
+       .write          = sched_cache_enable_write,
+       .read           = seq_read,
+       .llseek         = seq_lseek,
+       .release        = single_release,
+};
+#endif
+
 #ifdef CONFIG_PREEMPT_DYNAMIC
 
 static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
@@ -593,7 +633,7 @@ static void debugfs_ext_server_init(void)
 
 static __init int sched_init_debug(void)
 {
-       struct dentry __maybe_unused *numa;
+       struct dentry __maybe_unused *numa, *llc;
 
        debugfs_sched = debugfs_create_dir("sched", NULL);
 
@@ -626,6 +666,12 @@ static __init int sched_init_debug(void)
        debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
 #endif /* CONFIG_NUMA_BALANCING */
 
+#ifdef CONFIG_SCHED_CACHE
+       llc = debugfs_create_dir("llc_balancing", debugfs_sched);
+       debugfs_create_file("enabled", 0644, llc, NULL,
+                           &sched_cache_enable_fops);
+#endif
+
        debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
 
        debugfs_fair_server_init();

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 71f6077da4662ecd5cf513ac1f1c3546fc4d50f1..f499d5dd113083fb220114d3f569eacb7ae2e372 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4070,11 +4070,16 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
 
 #ifdef CONFIG_SCHED_CACHE
 DECLARE_STATIC_KEY_FALSE(sched_cache_present);
+DECLARE_STATIC_KEY_FALSE(sched_cache_active);
+extern int sysctl_sched_cache_user;
 
 static inline bool sched_cache_enabled(void)
 {
-       return static_branch_unlikely(&sched_cache_present);
+       return static_branch_unlikely(&sched_cache_active);
 }
+
+extern void sched_cache_active_set_unlocked(void);
+
 #endif
 
 void sched_domains_free_llc_id(int cpu);

diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 6a36f8f6b7b103cf457a725868e5c50a5d344013..9fc99346ef4f943c39650e97b07609bed7452027 100644 (file)
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -821,7 +821,16 @@ enum s_alloc {
 };
 
 #ifdef CONFIG_SCHED_CACHE
+/* hardware support for cache aware scheduling */
 DEFINE_STATIC_KEY_FALSE(sched_cache_present);
+/*
+ * Indicator of whether cache aware scheduling
+ * is active, used by the scheduler.
+ */
+DEFINE_STATIC_KEY_FALSE(sched_cache_active);
+/* user wants cache aware scheduling [0 or 1] */
+int sysctl_sched_cache_user = 1;
+
 static bool alloc_sd_llc(const struct cpumask *cpu_map,
                         struct s_data *d)
 {
@@ -856,6 +865,60 @@ err:
 
        return false;
 }
+
+static void _sched_cache_active_set(bool enable, bool locked)
+{
+       if (enable) {
+               if (locked)
+                       static_branch_enable_cpuslocked(&sched_cache_active);
+               else
+                       static_branch_enable(&sched_cache_active);
+       } else {
+               if (locked)
+                       static_branch_disable_cpuslocked(&sched_cache_active);
+               else
+                       static_branch_disable(&sched_cache_active);
+       }
+}
+
+/*
+ * Enable/disable cache aware scheduling according to
+ * user input and the presence of hardware support.
+ */
+static void sched_cache_active_set(bool locked)
+{
+       /* hardware does not support */
+       if (!static_branch_likely(&sched_cache_present)) {
+               _sched_cache_active_set(false, locked);
+               return;
+       }
+
+       /*
+        * user wants it or not ?
+        * TBD: read before writing the static key.
+        * It is not in the critical path, leave as-is
+        * for now.
+        */
+       if (sysctl_sched_cache_user) {
+               _sched_cache_active_set(true, locked);
+               if (sched_debug())
+                       pr_info("%s: enabling cache aware scheduling\n", __func__);
+       } else {
+               _sched_cache_active_set(false, locked);
+               if (sched_debug())
+                       pr_info("%s: disabling cache aware scheduling\n", __func__);
+       }
+}
+
+static void sched_cache_active_set_locked(void)
+{
+       return sched_cache_active_set(true);
+}
+
+void sched_cache_active_set_unlocked(void)
+{
+       return sched_cache_active_set(false);
+}
 #else
 static bool alloc_sd_llc(const struct cpumask *cpu_map,
                         struct s_data *d)
@@ -2926,6 +2989,8 @@ error:
                static_branch_enable_cpuslocked(&sched_cache_present);
        else
                static_branch_disable_cpuslocked(&sched_cache_present);
+
+       sched_cache_active_set_locked();
 #endif
        __free_domain_allocs(&d, alloc_state, cpu_map);