sched: Teach dequeue_task() about special task states

Since special task states must not suffer spurious wakeups, and the
proposed delayed dequeue can cause exactly these (under some boundary
conditions), propagate this knowledge into dequeue_task() such that it
can do the right thing.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Valentin Schneider <vschneid@redhat.com>
Tested-by: Valentin Schneider <vschneid@redhat.com>
Link: https://lkml.kernel.org/r/20240727105030.110439521@infradead.org

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 80e639e231401c6fd6da5176db14e19bbd48c846..868b71b9f2e4907ac91364bb557dba15b67f9342 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6530,11 +6530,16 @@ static void __sched notrace __schedule(unsigned int sched_mode)
                if (signal_pending_state(prev_state, prev)) {
                        WRITE_ONCE(prev->__state, TASK_RUNNING);
                } else {
+                       int flags = DEQUEUE_NOCLOCK;
+
                        prev->sched_contributes_to_load =
                                (prev_state & TASK_UNINTERRUPTIBLE) &&
                                !(prev_state & TASK_NOLOAD) &&
                                !(prev_state & TASK_FROZEN);
 
+                       if (unlikely(is_special_task_state(prev_state)))
+                               flags |= DEQUEUE_SPECIAL;
+
                        /*
                         * __schedule()                 ttwu()
                         *   prev_state = prev->state;    if (p->on_rq && ...)
@@ -6546,7 +6551,7 @@ static void __sched notrace __schedule(unsigned int sched_mode)
                         *
                         * After this, schedule() must not care about p->state any more.
                         */
-                       block_task(rq, prev, DEQUEUE_NOCLOCK);
+                       block_task(rq, prev, flags);
                }
                switch_count = &prev->nvcsw;
        }

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index ffca9779519cf3588263ecb44aa53fa1547a8f75..263b4ded2b47d5dc13ee740f7f513feac3934ee8 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2248,10 +2248,11 @@ extern const u32                sched_prio_to_wmult[40];
  *
  */
 
-#define DEQUEUE_SLEEP          0x01
+#define DEQUEUE_SLEEP          0x01 /* Matches ENQUEUE_WAKEUP */
 #define DEQUEUE_SAVE           0x02 /* Matches ENQUEUE_RESTORE */
 #define DEQUEUE_MOVE           0x04 /* Matches ENQUEUE_MOVE */
 #define DEQUEUE_NOCLOCK                0x08 /* Matches ENQUEUE_NOCLOCK */
+#define DEQUEUE_SPECIAL                0x10
 #define DEQUEUE_MIGRATING      0x100 /* Matches ENQUEUE_MIGRATING */
 #define DEQUEUE_DELAYED                0x200 /* Matches ENQUEUE_DELAYED */