sched/eevdf: More PELT vs DELAYED_DEQUEUE

[ Upstream commit 76f2f783294d7d55c2564e2dfb0a7279ba0bc264 ]

Vincent and Dietmar noted that while
commit fc1892becd56 ("sched/eevdf: Fixup PELT vs DELAYED_DEQUEUE") fixes
the entity runnable stats, it does not adjust the cfs_rq runnable stats,
which are based off of h_nr_running.

Track h_nr_delayed such that we can discount those and adjust the
signal.

Fixes: fc1892becd56 ("sched/eevdf: Fixup PELT vs DELAYED_DEQUEUE")
Closes: https://lore.kernel.org/lkml/a9a45193-d0c6-4ba2-a822-464ad30b550e@arm.com/
Closes: https://lore.kernel.org/lkml/CAKfTPtCNUvWE_GX5LyvTF-WdxUT=ZgvZZv-4t=eWntg5uOFqiQ@mail.gmail.com/
[ Fixes checkpatch warnings and rebased ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reported-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Reported-by: Vincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: "Peter Zijlstra (Intel)" <peterz@infradead.org>
Signed-off-by: Vincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Tested-by: K Prateek Nayak <kprateek.nayak@amd.com>
Link: https://lore.kernel.org/r/20241202174606.4074512-3-vincent.guittot@linaro.org
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index f4035c7a0fa1df365222300f443f87a6fe02dd71..82b165bf48c42323dab35203c4cb439944f16072 100644 (file)
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -844,6 +844,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
        SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread));
        SEQ_printf(m, "  .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
        SEQ_printf(m, "  .%-30s: %d\n", "h_nr_running", cfs_rq->h_nr_running);
+       SEQ_printf(m, "  .%-30s: %d\n", "h_nr_delayed", cfs_rq->h_nr_delayed);
        SEQ_printf(m, "  .%-30s: %d\n", "idle_nr_running",
                        cfs_rq->idle_nr_running);
        SEQ_printf(m, "  .%-30s: %d\n", "idle_h_nr_running",

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c467e389cd6fef0ce859700f3fdfcd12bee7853c..93142f9077c766e07a8b1a94cfdae78c06e18f37 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5471,9 +5471,33 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
 
 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
 
-static inline void finish_delayed_dequeue_entity(struct sched_entity *se)
+static void set_delayed(struct sched_entity *se)
+{
+       se->sched_delayed = 1;
+       for_each_sched_entity(se) {
+               struct cfs_rq *cfs_rq = cfs_rq_of(se);
+
+               cfs_rq->h_nr_delayed++;
+               if (cfs_rq_throttled(cfs_rq))
+                       break;
+       }
+}
+
+static void clear_delayed(struct sched_entity *se)
 {
        se->sched_delayed = 0;
+       for_each_sched_entity(se) {
+               struct cfs_rq *cfs_rq = cfs_rq_of(se);
+
+               cfs_rq->h_nr_delayed--;
+               if (cfs_rq_throttled(cfs_rq))
+                       break;
+       }
+}
+
+static inline void finish_delayed_dequeue_entity(struct sched_entity *se)
+{
+       clear_delayed(se);
        if (sched_feat(DELAY_ZERO) && se->vlag > 0)
                se->vlag = 0;
 }
@@ -5502,7 +5526,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
                if (sched_feat(DELAY_DEQUEUE) && delay &&
                    !entity_eligible(cfs_rq, se)) {
                        update_load_avg(cfs_rq, se, 0);
-                       se->sched_delayed = 1;
+                       set_delayed(se);
                        return false;
                }
        }
@@ -5920,7 +5944,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
        struct rq *rq = rq_of(cfs_rq);
        struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
        struct sched_entity *se;
-       long task_delta, idle_task_delta, dequeue = 1;
+       long task_delta, idle_task_delta, delayed_delta, dequeue = 1;
        long rq_h_nr_running = rq->cfs.h_nr_running;
 
        raw_spin_lock(&cfs_b->lock);
@@ -5953,6 +5977,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
 
        task_delta = cfs_rq->h_nr_running;
        idle_task_delta = cfs_rq->idle_h_nr_running;
+       delayed_delta = cfs_rq->h_nr_delayed;
        for_each_sched_entity(se) {
                struct cfs_rq *qcfs_rq = cfs_rq_of(se);
                int flags;
@@ -5976,6 +6001,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
 
                qcfs_rq->h_nr_running -= task_delta;
                qcfs_rq->idle_h_nr_running -= idle_task_delta;
+               qcfs_rq->h_nr_delayed -= delayed_delta;
 
                if (qcfs_rq->load.weight) {
                        /* Avoid re-evaluating load for this entity: */
@@ -5998,6 +6024,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
 
                qcfs_rq->h_nr_running -= task_delta;
                qcfs_rq->idle_h_nr_running -= idle_task_delta;
+               qcfs_rq->h_nr_delayed -= delayed_delta;
        }
 
        /* At this point se is NULL and we are at root level*/
@@ -6023,7 +6050,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
        struct rq *rq = rq_of(cfs_rq);
        struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
        struct sched_entity *se;
-       long task_delta, idle_task_delta;
+       long task_delta, idle_task_delta, delayed_delta;
        long rq_h_nr_running = rq->cfs.h_nr_running;
 
        se = cfs_rq->tg->se[cpu_of(rq)];
@@ -6059,6 +6086,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
 
        task_delta = cfs_rq->h_nr_running;
        idle_task_delta = cfs_rq->idle_h_nr_running;
+       delayed_delta = cfs_rq->h_nr_delayed;
        for_each_sched_entity(se) {
                struct cfs_rq *qcfs_rq = cfs_rq_of(se);
 
@@ -6076,6 +6104,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
 
                qcfs_rq->h_nr_running += task_delta;
                qcfs_rq->idle_h_nr_running += idle_task_delta;
+               qcfs_rq->h_nr_delayed += delayed_delta;
 
                /* end evaluation on encountering a throttled cfs_rq */
                if (cfs_rq_throttled(qcfs_rq))
@@ -6093,6 +6122,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
 
                qcfs_rq->h_nr_running += task_delta;
                qcfs_rq->idle_h_nr_running += idle_task_delta;
+               qcfs_rq->h_nr_delayed += delayed_delta;
 
                /* end evaluation on encountering a throttled cfs_rq */
                if (cfs_rq_throttled(qcfs_rq))
@@ -6946,7 +6976,7 @@ requeue_delayed_entity(struct sched_entity *se)
        }
 
        update_load_avg(cfs_rq, se, 0);
-       se->sched_delayed = 0;
+       clear_delayed(se);
 }
 
 /*
@@ -6960,6 +6990,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
        struct cfs_rq *cfs_rq;
        struct sched_entity *se = &p->se;
        int idle_h_nr_running = task_has_idle_policy(p);
+       int h_nr_delayed = 0;
        int task_new = !(flags & ENQUEUE_WAKEUP);
        int rq_h_nr_running = rq->cfs.h_nr_running;
        u64 slice = 0;
@@ -6986,6 +7017,9 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
        if (p->in_iowait)
                cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
 
+       if (task_new)
+               h_nr_delayed = !!se->sched_delayed;
+
        for_each_sched_entity(se) {
                if (se->on_rq) {
                        if (se->sched_delayed)
@@ -7008,6 +7042,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 
                cfs_rq->h_nr_running++;
                cfs_rq->idle_h_nr_running += idle_h_nr_running;
+               cfs_rq->h_nr_delayed += h_nr_delayed;
 
                if (cfs_rq_is_idle(cfs_rq))
                        idle_h_nr_running = 1;
@@ -7031,6 +7066,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 
                cfs_rq->h_nr_running++;
                cfs_rq->idle_h_nr_running += idle_h_nr_running;
+               cfs_rq->h_nr_delayed += h_nr_delayed;
 
                if (cfs_rq_is_idle(cfs_rq))
                        idle_h_nr_running = 1;
@@ -7093,6 +7129,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
        struct task_struct *p = NULL;
        int idle_h_nr_running = 0;
        int h_nr_running = 0;
+       int h_nr_delayed = 0;
        struct cfs_rq *cfs_rq;
        u64 slice = 0;
 
@@ -7100,6 +7137,8 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
                p = task_of(se);
                h_nr_running = 1;
                idle_h_nr_running = task_has_idle_policy(p);
+               if (!task_sleep && !task_delayed)
+                       h_nr_delayed = !!se->sched_delayed;
        } else {
                cfs_rq = group_cfs_rq(se);
                slice = cfs_rq_min_slice(cfs_rq);
@@ -7117,6 +7156,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
 
                cfs_rq->h_nr_running -= h_nr_running;
                cfs_rq->idle_h_nr_running -= idle_h_nr_running;
+               cfs_rq->h_nr_delayed -= h_nr_delayed;
 
                if (cfs_rq_is_idle(cfs_rq))
                        idle_h_nr_running = h_nr_running;
@@ -7155,6 +7195,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
 
                cfs_rq->h_nr_running -= h_nr_running;
                cfs_rq->idle_h_nr_running -= idle_h_nr_running;
+               cfs_rq->h_nr_delayed -= h_nr_delayed;
 
                if (cfs_rq_is_idle(cfs_rq))
                        idle_h_nr_running = h_nr_running;

diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c
index a9c65d97b3cac61c6203b345ae47a791dc3dddc3..171a802420a10a5a2e55d41888569188220da7e3 100644 (file)
--- a/kernel/sched/pelt.c
+++ b/kernel/sched/pelt.c
@@ -321,7 +321,7 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq)
 {
        if (___update_load_sum(now, &cfs_rq->avg,
                                scale_load_down(cfs_rq->load.weight),
-                               cfs_rq->h_nr_running,
+                               cfs_rq->h_nr_running - cfs_rq->h_nr_delayed,
                                cfs_rq->curr != NULL)) {
 
                ___update_load_avg(&cfs_rq->avg, 1);

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index c03b3d7b320e9c4c87f68f7eeb53c9e2c5f419d4..c53696275ca10447d0804564784439b0955ebbb9 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -649,6 +649,7 @@ struct cfs_rq {
        unsigned int            h_nr_running;      /* SCHED_{NORMAL,BATCH,IDLE} */
        unsigned int            idle_nr_running;   /* SCHED_IDLE */
        unsigned int            idle_h_nr_running; /* SCHED_IDLE */
+       unsigned int            h_nr_delayed;
 
        s64                     avg_vruntime;
        u64                     avg_load;
@@ -898,8 +899,11 @@ struct dl_rq {
 
 static inline void se_update_runnable(struct sched_entity *se)
 {
-       if (!entity_is_task(se))
-               se->runnable_weight = se->my_q->h_nr_running;
+       if (!entity_is_task(se)) {
+               struct cfs_rq *cfs_rq = se->my_q;
+
+               se->runnable_weight = cfs_rq->h_nr_running - cfs_rq->h_nr_delayed;
+       }
 }
 
 static inline long se_runnable(struct sched_entity *se)