sched/fair: Fix NEXT_BUDDY

[ Upstream commit 493afbd187c4c9cc1642792c0d9ba400c3d6d90d ]

Adam reports that enabling NEXT_BUDDY insta triggers a WARN in
pick_next_entity().

Moving clear_buddies() up before the delayed dequeue bits ensures
no ->next buddy becomes delayed. Further ensure no new ->next buddy
ever starts as delayed.

Fixes: 152e11f6df29 ("sched/fair: Implement delayed dequeue")
Reported-by: Adam Li <adamli@os.amperecomputing.com>
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: Adam Li <adamli@os.amperecomputing.com>
Link: https://lkml.kernel.org/r/670a0d54-e398-4b1f-8a6e-90784e2fdf89@amd.com
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 782ce70ebd1b08592b11d6c01d733cc3d1fd2dd2..c467e389cd6fef0ce859700f3fdfcd12bee7853c 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5484,6 +5484,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
        bool sleep = flags & DEQUEUE_SLEEP;
 
        update_curr(cfs_rq);
+       clear_buddies(cfs_rq, se);
 
        if (flags & DEQUEUE_DELAYED) {
                SCHED_WARN_ON(!se->sched_delayed);
@@ -5500,8 +5501,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 
                if (sched_feat(DELAY_DEQUEUE) && delay &&
                    !entity_eligible(cfs_rq, se)) {
-                       if (cfs_rq->next == se)
-                               cfs_rq->next = NULL;
                        update_load_avg(cfs_rq, se, 0);
                        se->sched_delayed = 1;
                        return false;
@@ -5526,8 +5525,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 
        update_stats_dequeue_fair(cfs_rq, se, flags);
 
-       clear_buddies(cfs_rq, se);
-
        update_entity_lag(cfs_rq, se);
        if (sched_feat(PLACE_REL_DEADLINE) && !sleep) {
                se->deadline -= se->vruntime;
@@ -8786,7 +8783,7 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int
        if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
                return;
 
-       if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK)) {
+       if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK) && !pse->sched_delayed) {
                set_next_buddy(pse);
        }