sched/psi: Use task->psi_flags to clear in CPU migration

[ Upstream commit 52b33d87b9197c51e8ffdc61873739d90dd0a16f ]

The commit d583d360a620 ("psi: Fix psi state corruption when schedule()
races with cgroup move") fixed a race problem by making cgroup_move_task()
use task->psi_flags instead of looking at the scheduler state.

We can extend task->psi_flags usage to CPU migration, which should be
a minor optimization for performance and code simplicity.

Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Link: https://lore.kernel.org/r/20220926081931.45420-1-zhouchengming@bytedance.com
Stable-dep-of: a430d99e3490 ("sched/fair: Fix value reported by hot tasks pulled in /proc/schedstat")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 9b3cfe685cb452890839aeef1b1509849d162390..875f3d317b9c897ae62a9950d71d8f3ee73a3e05 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -874,9 +874,6 @@ struct task_struct {
        unsigned                        sched_reset_on_fork:1;
        unsigned                        sched_contributes_to_load:1;
        unsigned                        sched_migrated:1;
-#ifdef CONFIG_PSI
-       unsigned                        sched_psi_wake_requeue:1;
-#endif
 
        /* Force alignment to the next boundary: */
        unsigned                        :0;

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ed92b75f7e02461090622e6c3489487e4bf0f955..fee8e2a7c75303b5c441ef2135cb73f3413b5c1c 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1967,7 +1967,7 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
 
        if (!(flags & ENQUEUE_RESTORE)) {
                sched_info_enqueue(rq, p);
-               psi_enqueue(p, flags & ENQUEUE_WAKEUP);
+               psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED));
        }
 
        uclamp_rq_inc(rq, p);

diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
index 975703572bc0da7825dbba3364f2421edb7316d2..cee3100c74cd5b447b499146f13d52d5c48b78dc 100644 (file)
--- a/kernel/sched/stats.h
+++ b/kernel/sched/stats.h
@@ -91,11 +91,9 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup)
        if (p->in_memstall)
                set |= TSK_MEMSTALL_RUNNING;
 
-       if (!wakeup || p->sched_psi_wake_requeue) {
+       if (!wakeup) {
                if (p->in_memstall)
                        set |= TSK_MEMSTALL;
-               if (p->sched_psi_wake_requeue)
-                       p->sched_psi_wake_requeue = 0;
        } else {
                if (p->in_iowait)
                        clear |= TSK_IOWAIT;
@@ -106,8 +104,6 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup)
 
 static inline void psi_dequeue(struct task_struct *p, bool sleep)
 {
-       int clear = TSK_RUNNING;
-
        if (static_branch_likely(&psi_disabled))
                return;
 
@@ -120,10 +116,7 @@ static inline void psi_dequeue(struct task_struct *p, bool sleep)
        if (sleep)
                return;
 
-       if (p->in_memstall)
-               clear |= (TSK_MEMSTALL | TSK_MEMSTALL_RUNNING);
-
-       psi_task_change(p, clear, 0);
+       psi_task_change(p, p->psi_flags, 0);
 }
 
 static inline void psi_ttwu_dequeue(struct task_struct *p)
@@ -135,19 +128,12 @@ static inline void psi_ttwu_dequeue(struct task_struct *p)
         * deregister its sleep-persistent psi states from the old
         * queue, and let psi_enqueue() know it has to requeue.
         */
-       if (unlikely(p->in_iowait || p->in_memstall)) {
+       if (unlikely(p->psi_flags)) {
                struct rq_flags rf;
                struct rq *rq;
-               int clear = 0;
-
-               if (p->in_iowait)
-                       clear |= TSK_IOWAIT;
-               if (p->in_memstall)
-                       clear |= TSK_MEMSTALL;
 
                rq = __task_rq_lock(p, &rf);
-               psi_task_change(p, clear, 0);
-               p->sched_psi_wake_requeue = 1;
+               psi_task_change(p, p->psi_flags, 0);
                __task_rq_unlock(rq, &rf);
        }
 }