sched: Add CONFIG_SCHED_PROXY_EXEC & boot argument to enable/disable

Add a CONFIG_SCHED_PROXY_EXEC option, along with a boot argument
sched_proxy_exec= that can be used to disable the feature at boot
time if CONFIG_SCHED_PROXY_EXEC was enabled.

Also uses this option to allow the rq->donor to be different from
rq->curr.

Signed-off-by: John Stultz <jstultz@google.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: K Prateek Nayak <kprateek.nayak@amd.com>
Link: https://lkml.kernel.org/r/20250712033407.2383110-2-jstultz@google.com

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 07e22ba5bfe34354033127ca21031963cd11745b..00b835717860ced653330398d8884362f8d7fb9b 100644 (file)
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -6387,6 +6387,11 @@
        sa1100ir        [NET]
                        See drivers/net/irda/sa1100_ir.c.
 
+       sched_proxy_exec= [KNL]
+                       Enables or disables "proxy execution" style
+                       solution to mutex-based priority inversion.
+                       Format: <bool>
+
        sched_verbose   [KNL,EARLY] Enables verbose scheduler debug messages.
 
        schedstats=     [KNL,X86] Enable or disable scheduled statistics.

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 54a91261e99b36eded9b1203d61f881793123868..f225b6b1baa3558207e0d579706ec59769a63439 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1656,6 +1656,19 @@ struct task_struct {
        randomized_struct_fields_end
 } __attribute__ ((aligned (64)));
 
+#ifdef CONFIG_SCHED_PROXY_EXEC
+DECLARE_STATIC_KEY_TRUE(__sched_proxy_exec);
+static inline bool sched_proxy_exec(void)
+{
+       return static_branch_likely(&__sched_proxy_exec);
+}
+#else
+static inline bool sched_proxy_exec(void)
+{
+       return false;
+}
+#endif
+
 #define TASK_REPORT_IDLE       (TASK_REPORT + 1)
 #define TASK_REPORT_MAX                (TASK_REPORT_IDLE << 1)
 

diff --git a/init/Kconfig b/init/Kconfig
index 965699c0a6d949ca44f84b0ffc783d2212f98f8e..24dd42d3808d218148055a0ae45f8d75c9eb1601 100644 (file)
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -878,6 +878,18 @@ config UCLAMP_BUCKETS_COUNT
 
          If in doubt, use the default value.
 
+config SCHED_PROXY_EXEC
+       bool "Proxy Execution"
+       # Avoid some build failures w/ PREEMPT_RT until it can be fixed
+       depends on !PREEMPT_RT
+       # Need to investigate how to inform sched_ext of split contexts
+       depends on !SCHED_CLASS_EXT
+       # Not particularly useful until we get to multi-rq proxying
+       depends on EXPERT
+       help
+         This option enables proxy execution, a mechanism for mutex-owning
+         tasks to inherit the scheduling context of higher priority waiters.
+
 endmenu
 
 #

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index e9c8bda84d80ab447bf9851e7fd89b60e7796ea2..dd9f5c08b56358901c7246b50153fd61e454d620 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -119,6 +119,35 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp);
 
 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 
+#ifdef CONFIG_SCHED_PROXY_EXEC
+DEFINE_STATIC_KEY_TRUE(__sched_proxy_exec);
+static int __init setup_proxy_exec(char *str)
+{
+       bool proxy_enable = true;
+
+       if (*str && kstrtobool(str + 1, &proxy_enable)) {
+               pr_warn("Unable to parse sched_proxy_exec=\n");
+               return 0;
+       }
+
+       if (proxy_enable) {
+               pr_info("sched_proxy_exec enabled via boot arg\n");
+               static_branch_enable(&__sched_proxy_exec);
+       } else {
+               pr_info("sched_proxy_exec disabled via boot arg\n");
+               static_branch_disable(&__sched_proxy_exec);
+       }
+       return 1;
+}
+#else
+static int __init setup_proxy_exec(char *str)
+{
+       pr_warn("CONFIG_SCHED_PROXY_EXEC=n, so it cannot be enabled or disabled at boot time\n");
+       return 0;
+}
+#endif
+__setup("sched_proxy_exec", setup_proxy_exec);
+
 /*
  * Debugging: various feature bits
  *

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index ac953fad8c21940bfad80f4f6a00584c92b37d34..e53d0b87f780dce28d2eac566270f2500b4036b0 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1142,10 +1142,15 @@ struct rq {
         */
        unsigned long           nr_uninterruptible;
 
+#ifdef CONFIG_SCHED_PROXY_EXEC
+       struct task_struct __rcu        *donor;  /* Scheduling context */
+       struct task_struct __rcu        *curr;   /* Execution context */
+#else
        union {
                struct task_struct __rcu *donor; /* Scheduler context */
                struct task_struct __rcu *curr;  /* Execution context */
        };
+#endif
        struct sched_dl_entity  *dl_server;
        struct task_struct      *idle;
        struct task_struct      *stop;
@@ -1326,10 +1331,17 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 #define cpu_curr(cpu)          (cpu_rq(cpu)->curr)
 #define raw_rq()               raw_cpu_ptr(&runqueues)
 
+#ifdef CONFIG_SCHED_PROXY_EXEC
+static inline void rq_set_donor(struct rq *rq, struct task_struct *t)
+{
+       rcu_assign_pointer(rq->donor, t);
+}
+#else
 static inline void rq_set_donor(struct rq *rq, struct task_struct *t)
 {
        /* Do nothing */
 }
+#endif
 
 #ifdef CONFIG_SCHED_CORE
 static inline struct cpumask *sched_group_span(struct sched_group *sg);