sched_ext: keep running prev when prev->scx.slice != 0

[ Upstream commit 30dd3b13f9de612ef7328ccffcf1a07d0d40ab51 ]

When %SCX_OPS_ENQ_LAST is set and prev->scx.slice != 0,
@prev will be dispacthed into the local DSQ in put_prev_task_scx().
However, pick_task_scx() is executed before put_prev_task_scx(),
so it will not pick @prev.
Set %SCX_RQ_BAL_KEEP in balance_one() to ensure that pick_task_scx()
can pick @prev.

Signed-off-by: Henry Huang <henry.hj@antgroup.com>
Acked-by: Andrea Righi <arighi@nvidia.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index f928a67a07d29aa1839dce9678223abf506f3830..4c4681cb9337b40d94ac6639852c648565a45815 100644 (file)
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -2630,6 +2630,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 {
        struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
        bool prev_on_scx = prev->sched_class == &ext_sched_class;
+       bool prev_on_rq = prev->scx.flags & SCX_TASK_QUEUED;
        int nr_loops = SCX_DSP_MAX_LOOPS;
 
        lockdep_assert_rq_held(rq);
@@ -2662,8 +2663,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
                 * See scx_ops_disable_workfn() for the explanation on the
                 * bypassing test.
                 */
-               if ((prev->scx.flags & SCX_TASK_QUEUED) &&
-                   prev->scx.slice && !scx_rq_bypassing(rq)) {
+               if (prev_on_rq && prev->scx.slice && !scx_rq_bypassing(rq)) {
                        rq->scx.flags |= SCX_RQ_BAL_KEEP;
                        goto has_tasks;
                }
@@ -2696,6 +2696,10 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 
                flush_dispatch_buf(rq);
 
+               if (prev_on_rq && prev->scx.slice) {
+                       rq->scx.flags |= SCX_RQ_BAL_KEEP;
+                       goto has_tasks;
+               }
                if (rq->scx.local_dsq.nr)
                        goto has_tasks;
                if (consume_global_dsq(rq))
@@ -2721,8 +2725,7 @@ no_tasks:
         * Didn't find another task to run. Keep running @prev unless
         * %SCX_OPS_ENQ_LAST is in effect.
         */
-       if ((prev->scx.flags & SCX_TASK_QUEUED) &&
-           (!static_branch_unlikely(&scx_ops_enq_last) ||
+       if (prev_on_rq && (!static_branch_unlikely(&scx_ops_enq_last) ||
             scx_rq_bypassing(rq))) {
                rq->scx.flags |= SCX_RQ_BAL_KEEP;
                goto has_tasks;