sched: Add an initial sketch of the find_proxy_task() function

Add a find_proxy_task() function which doesn't do much.

When we select a blocked task to run, we will just deactivate it
and pick again. The exception being if it has become unblocked
after find_proxy_task() was called.

This allows us to validate keeping blocked tasks on the runqueue
and later deactivating them is working ok, stressing the failure
cases for when a proxy isn't found.

Greatly simplified from patch by:
  Peter Zijlstra (Intel) <peterz@infradead.org>
  Juri Lelli <juri.lelli@redhat.com>
  Valentin Schneider <valentin.schneider@arm.com>
  Connor O'Brien <connoro@google.com>

[jstultz: Split out from larger proxy patch and simplified
 for review and testing.]
Signed-off-by: John Stultz <jstultz@google.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: K Prateek Nayak <kprateek.nayak@amd.com>
Link: https://lkml.kernel.org/r/20250712033407.2383110-7-jstultz@google.com

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index dd9f5c08b56358901c7246b50153fd61e454d620..cb55d4247e65efd9e706edfb5a16640ba1ad2f41 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6522,11 +6522,13 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 /*
  * Helper function for __schedule()
  *
- * If a task does not have signals pending, deactivate it
- * Otherwise marks the task's __state as RUNNING
+ * Tries to deactivate the task, unless the should_block arg
+ * is false or if a signal is pending. In the case a signal
+ * is pending, marks the task's __state as RUNNING (and clear
+ * blocked_on).
  */
 static bool try_to_block_task(struct rq *rq, struct task_struct *p,
-                             unsigned long *task_state_p)
+                             unsigned long *task_state_p, bool should_block)
 {
        unsigned long task_state = *task_state_p;
        int flags = DEQUEUE_NOCLOCK;
@@ -6537,6 +6539,16 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
                return false;
        }
 
+       /*
+        * We check should_block after signal_pending because we
+        * will want to wake the task in that case. But if
+        * should_block is false, its likely due to the task being
+        * blocked on a mutex, and we want to keep it on the runqueue
+        * to be selectable for proxy-execution.
+        */
+       if (!should_block)
+               return false;
+
        p->sched_contributes_to_load =
                (task_state & TASK_UNINTERRUPTIBLE) &&
                !(task_state & TASK_NOLOAD) &&
@@ -6560,6 +6572,88 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
        return true;
 }
 
+#ifdef CONFIG_SCHED_PROXY_EXEC
+static inline void proxy_resched_idle(struct rq *rq)
+{
+       put_prev_set_next_task(rq, rq->donor, rq->idle);
+       rq_set_donor(rq, rq->idle);
+       set_tsk_need_resched(rq->idle);
+}
+
+static bool __proxy_deactivate(struct rq *rq, struct task_struct *donor)
+{
+       unsigned long state = READ_ONCE(donor->__state);
+
+       /* Don't deactivate if the state has been changed to TASK_RUNNING */
+       if (state == TASK_RUNNING)
+               return false;
+       /*
+        * Because we got donor from pick_next_task(), it is *crucial*
+        * that we call proxy_resched_idle() before we deactivate it.
+        * As once we deactivate donor, donor->on_rq is set to zero,
+        * which allows ttwu() to immediately try to wake the task on
+        * another rq. So we cannot use *any* references to donor
+        * after that point. So things like cfs_rq->curr or rq->donor
+        * need to be changed from next *before* we deactivate.
+        */
+       proxy_resched_idle(rq);
+       return try_to_block_task(rq, donor, &state, true);
+}
+
+static struct task_struct *proxy_deactivate(struct rq *rq, struct task_struct *donor)
+{
+       if (!__proxy_deactivate(rq, donor)) {
+               /*
+                * XXX: For now, if deactivation failed, set donor
+                * as unblocked, as we aren't doing proxy-migrations
+                * yet (more logic will be needed then).
+                */
+               donor->blocked_on = NULL;
+       }
+       return NULL;
+}
+
+/*
+ * Initial simple sketch that just deactivates the blocked task
+ * chosen by pick_next_task() so we can then pick something that
+ * isn't blocked.
+ */
+static struct task_struct *
+find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
+{
+       struct mutex *mutex;
+
+       mutex = donor->blocked_on;
+       /* Something changed in the chain, so pick again */
+       if (!mutex)
+               return NULL;
+       /*
+        * By taking mutex->wait_lock we hold off concurrent mutex_unlock()
+        * and ensure @owner sticks around.
+        */
+       guard(raw_spinlock)(&mutex->wait_lock);
+
+       /* Check again that donor is blocked with blocked_lock held */
+       if (!task_is_blocked(donor) || mutex != __get_task_blocked_on(donor)) {
+               /*
+                * Something changed in the blocked_on chain and
+                * we don't know if only at this level. So, let's
+                * just bail out completely and let __schedule()
+                * figure things out (pick_again loop).
+                */
+               return NULL; /* do pick_next_task() again */
+       }
+       return proxy_deactivate(rq, donor);
+}
+#else /* SCHED_PROXY_EXEC */
+static struct task_struct *
+find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
+{
+       WARN_ONCE(1, "This should never be called in the !SCHED_PROXY_EXEC case\n");
+       return donor;
+}
+#endif /* SCHED_PROXY_EXEC */
+
 /*
  * __schedule() is the main scheduler function.
  *
@@ -6672,12 +6766,25 @@ static void __sched notrace __schedule(int sched_mode)
                        goto picked;
                }
        } else if (!preempt && prev_state) {
-               try_to_block_task(rq, prev, &prev_state);
+               /*
+                * We pass task_is_blocked() as the should_block arg
+                * in order to keep mutex-blocked tasks on the runqueue
+                * for slection with proxy-exec (without proxy-exec
+                * task_is_blocked() will always be false).
+                */
+               try_to_block_task(rq, prev, &prev_state,
+                                 !task_is_blocked(prev));
                switch_count = &prev->nvcsw;
        }
 
-       next = pick_next_task(rq, prev, &rf);
+pick_again:
+       next = pick_next_task(rq, rq->donor, &rf);
        rq_set_donor(rq, next);
+       if (unlikely(task_is_blocked(next))) {
+               next = find_proxy_task(rq, next, &rf);
+               if (!next)
+                       goto pick_again;
+       }
 picked:
        clear_tsk_need_resched(prev);
        clear_preempt_need_resched();

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index e53d0b87f780dce28d2eac566270f2500b4036b0..d3f33d10c58c94d60f91533ed45d4a0708942bbb 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2262,6 +2262,14 @@ static inline int task_current_donor(struct rq *rq, struct task_struct *p)
        return rq->donor == p;
 }
 
+static inline bool task_is_blocked(struct task_struct *p)
+{
+       if (!sched_proxy_exec())
+               return false;
+
+       return !!p->blocked_on;
+}
+
 static inline int task_on_cpu(struct rq *rq, struct task_struct *p)
 {
        return p->on_cpu;
@@ -2459,7 +2467,7 @@ static inline void put_prev_set_next_task(struct rq *rq,
                                          struct task_struct *prev,
                                          struct task_struct *next)
 {
-       WARN_ON_ONCE(rq->curr != prev);
+       WARN_ON_ONCE(rq->donor != prev);
 
        __put_prev_set_next_dl_server(rq, prev, next);