sched: Cancel the slice protection of the idle entity

[ Upstream commit f553741ac8c0e467a3b873e305f34b902e50b86d ]

A wakeup non-idle entity should preempt idle entity at any time,
but because of the slice protection of the idle entity, the non-idle
entity has to wait, so just cancel it.

This patch is aimed at minimizing the impact of SCHED_IDLE on
SCHED_NORMAL. For example, a task with SCHED_IDLE policy that sleeps for
1s and then runs for 3 ms, running cyclictest on the same cpu, has a
maximum latency of 3 ms, which is caused by the slice protection of the
idle entity. It is unreasonable. With this patch, the cyclictest latency
under the same conditions is basically the same on the cpu with idle
processes and on empty cpu.

[peterz: add helpers]
Fixes: 63304558ba5d ("sched/eevdf: Curb wakeup-preemption")
Signed-off-by: zihan zhou <15645113830zzh@gmail.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org>
Tested-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lkml.kernel.org/r/20250208080850.16300-1-15645113830zzh@gmail.com
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 58ba14ed8fbcb98ef1d2bb6779aae1a51c71e595..49e340f9fa71b453195394ac73b6e42d84f42e0e 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -885,6 +885,26 @@ struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
        return __node_2_se(left);
 }
 
+/*
+ * HACK, stash a copy of deadline at the point of pick in vlag,
+ * which isn't used until dequeue.
+ */
+static inline void set_protect_slice(struct sched_entity *se)
+{
+       se->vlag = se->deadline;
+}
+
+static inline bool protect_slice(struct sched_entity *se)
+{
+       return se->vlag == se->deadline;
+}
+
+static inline void cancel_protect_slice(struct sched_entity *se)
+{
+       if (protect_slice(se))
+               se->vlag = se->deadline + 1;
+}
+
 /*
  * Earliest Eligible Virtual Deadline First
  *
@@ -921,11 +941,7 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)
        if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr)))
                curr = NULL;
 
-       /*
-        * Once selected, run a task until it either becomes non-eligible or
-        * until it gets a new slice. See the HACK in set_next_entity().
-        */
-       if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline)
+       if (sched_feat(RUN_TO_PARITY) && curr && protect_slice(curr))
                return curr;
 
        /* Pick the leftmost entity if it's eligible */
@@ -5626,11 +5642,8 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
                update_stats_wait_end_fair(cfs_rq, se);
                __dequeue_entity(cfs_rq, se);
                update_load_avg(cfs_rq, se, UPDATE_TG);
-               /*
-                * HACK, stash a copy of deadline at the point of pick in vlag,
-                * which isn't used until dequeue.
-                */
-               se->vlag = se->deadline;
+
+               set_protect_slice(se);
        }
 
        update_stats_curr_start(cfs_rq, se);
@@ -8882,8 +8895,15 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int
         * Preempt an idle entity in favor of a non-idle entity (and don't preempt
         * in the inverse case).
         */
-       if (cse_is_idle && !pse_is_idle)
+       if (cse_is_idle && !pse_is_idle) {
+               /*
+                * When non-idle entity preempt an idle entity,
+                * don't give idle entity slice protection.
+                */
+               cancel_protect_slice(se);
                goto preempt;
+       }
+
        if (cse_is_idle != pse_is_idle)
                return;
 
@@ -8902,8 +8922,8 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int
         * Note that even if @p does not turn out to be the most eligible
         * task at this moment, current's slice protection will be lost.
         */
-       if (do_preempt_short(cfs_rq, pse, se) && se->vlag == se->deadline)
-               se->vlag = se->deadline + 1;
+       if (do_preempt_short(cfs_rq, pse, se))
+               cancel_protect_slice(se);
 
        /*
         * If @p has become the most eligible task, force preemption.