sched/fair: Cleanup fair_server

The throttle interaction made my brain hurt, make it consistently
about 0 transitions of h_nr_running.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index ee251ac50398c27d7549a253eb578cb2763e0a6a..795ceef5e7e146f99ea05f8d86ea9aeae0c7df9c 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5849,10 +5849,10 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
        /* At this point se is NULL and we are at root level*/
        sub_nr_running(rq, task_delta);
 
-done:
        /* Stop the fair server if throttling resulted in no runnable tasks */
        if (rq_h_nr_running && !rq->cfs.h_nr_running)
                dl_server_stop(&rq->fair_server);
+done:
        /*
         * Note: distribution will already see us throttled via the
         * throttled-list.  rq->lock protects completion.
@@ -5940,16 +5940,16 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
                        goto unthrottle_throttle;
        }
 
+       /* Start the fair server if un-throttling resulted in new runnable tasks */
+       if (!rq_h_nr_running && rq->cfs.h_nr_running)
+               dl_server_start(&rq->fair_server);
+
        /* At this point se is NULL and we are at root level*/
        add_nr_running(rq, task_delta);
 
 unthrottle_throttle:
        assert_list_leaf_cfs_rq(rq);
 
-       /* Start the fair server if un-throttling resulted in new runnable tasks */
-       if (!rq_h_nr_running && rq->cfs.h_nr_running)
-               dl_server_start(&rq->fair_server);
-
        /* Determine whether we need to wake up potentially idle CPU: */
        if (rq->curr == rq->idle && rq->cfs.nr_running)
                resched_curr(rq);
@@ -6771,6 +6771,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
        struct sched_entity *se = &p->se;
        int idle_h_nr_running = task_has_idle_policy(p);
        int task_new = !(flags & ENQUEUE_WAKEUP);
+       int rq_h_nr_running = rq->cfs.h_nr_running;
 
        /*
         * The code below (indirectly) updates schedutil which looks at
@@ -6780,13 +6781,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
         */
        util_est_enqueue(&rq->cfs, p);
 
-       if (!throttled_hierarchy(task_cfs_rq(p)) && !rq->cfs.h_nr_running) {
-               /* Account for idle runtime */
-               if (!rq->nr_running)
-                       dl_server_update_idle_time(rq, rq->curr);
-               dl_server_start(&rq->fair_server);
-       }
-
        /*
         * If in_iowait is set, the code below may not trigger any cpufreq
         * utilization updates, so do it here explicitly with the IOWAIT flag
@@ -6832,6 +6826,13 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
                        goto enqueue_throttle;
        }
 
+       if (!rq_h_nr_running && rq->cfs.h_nr_running) {
+               /* Account for idle runtime */
+               if (!rq->nr_running)
+                       dl_server_update_idle_time(rq, rq->curr);
+               dl_server_start(&rq->fair_server);
+       }
+
        /* At this point se is NULL and we are at root level*/
        add_nr_running(rq, 1);
 
@@ -6872,6 +6873,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
        int task_sleep = flags & DEQUEUE_SLEEP;
        int idle_h_nr_running = task_has_idle_policy(p);
        bool was_sched_idle = sched_idle_rq(rq);
+       int rq_h_nr_running = rq->cfs.h_nr_running;
 
        util_est_dequeue(&rq->cfs, p);
 
@@ -6926,14 +6928,14 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
        /* At this point se is NULL and we are at root level*/
        sub_nr_running(rq, 1);
 
+       if (rq_h_nr_running && !rq->cfs.h_nr_running)
+               dl_server_stop(&rq->fair_server);
+
        /* balance early to pull high priority tasks */
        if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
                rq->next_balance = jiffies;
 
 dequeue_throttle:
-       if (!throttled_hierarchy(task_cfs_rq(p)) && !rq->cfs.h_nr_running)
-               dl_server_stop(&rq->fair_server);
-
        util_est_update(&rq->cfs, p, task_sleep);
        hrtick_update(rq);
 }