sched/deadline: Generalize unique visiting of root domains

Bandwidth checks and updates that work on root domains currently employ
a cookie mechanism for efficiency. This mechanism is very much tied to
when root domains are first created and initialized.

Generalize the cookie mechanism so that it can be used also later at
runtime while updating root domains. Also, additionally guard it with
sched_domains_mutex, since domains need to be stable while updating them
(and it will be required for further dynamic changes).

Fixes: 53916d5fd3c0 ("sched/deadline: Check bandwidth overflow earlier for hotplug")
Reported-by: Jon Hunter <jonathanh@nvidia.com>
Signed-off-by: Juri Lelli <juri.lelli@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Valentin Schneider <vschneid@redhat.com>
Reviewed-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Tested-by: Waiman Long <longman@redhat.com>
Tested-by: Jon Hunter <jonathanh@nvidia.com>
Tested-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Link: https://lore.kernel.org/r/Z9MQaiXPvEeW_v7x@jlelli-thinkpadt14gen4.remote.csb

diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h
index 3a912ab42bb550c89547a2be26a95cd4d507f5fc..6ec578600b24ca4185180a1b4f3c4af33bb93e57 100644 (file)
--- a/include/linux/sched/deadline.h
+++ b/include/linux/sched/deadline.h
@@ -37,4 +37,7 @@ extern void dl_clear_root_domain(struct root_domain *rd);
 
 #endif /* CONFIG_SMP */
 
+extern u64 dl_cookie;
+extern bool dl_bw_visited(int cpu, u64 cookie);
+
 #endif /* _LINUX_SCHED_DEADLINE_H */

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 1a041c1fc0d1ec9c115c8bbcf9262c9d5866d68a..3e05032e9e0ebf4a6214d0a15aa5c3a1c2ba019e 100644 (file)
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -166,14 +166,14 @@ static inline unsigned long dl_bw_capacity(int i)
        }
 }
 
-static inline bool dl_bw_visited(int cpu, u64 gen)
+static inline bool dl_bw_visited(int cpu, u64 cookie)
 {
        struct root_domain *rd = cpu_rq(cpu)->rd;
 
-       if (rd->visit_gen == gen)
+       if (rd->visit_cookie == cookie)
                return true;
 
-       rd->visit_gen = gen;
+       rd->visit_cookie = cookie;
        return false;
 }
 
@@ -207,7 +207,7 @@ static inline unsigned long dl_bw_capacity(int i)
        return SCHED_CAPACITY_SCALE;
 }
 
-static inline bool dl_bw_visited(int cpu, u64 gen)
+static inline bool dl_bw_visited(int cpu, u64 cookie)
 {
        return false;
 }
@@ -3171,15 +3171,18 @@ DEFINE_SCHED_CLASS(dl) = {
 #endif
 };
 
-/* Used for dl_bw check and update, used under sched_rt_handler()::mutex */
-static u64 dl_generation;
+/*
+ * Used for dl_bw check and update, used under sched_rt_handler()::mutex and
+ * sched_domains_mutex.
+ */
+u64 dl_cookie;
 
 int sched_dl_global_validate(void)
 {
        u64 runtime = global_rt_runtime();
        u64 period = global_rt_period();
        u64 new_bw = to_ratio(period, runtime);
-       u64 gen = ++dl_generation;
+       u64 cookie = ++dl_cookie;
        struct dl_bw *dl_b;
        int cpu, cpus, ret = 0;
        unsigned long flags;
@@ -3192,7 +3195,7 @@ int sched_dl_global_validate(void)
        for_each_online_cpu(cpu) {
                rcu_read_lock_sched();
 
-               if (dl_bw_visited(cpu, gen))
+               if (dl_bw_visited(cpu, cookie))
                        goto next;
 
                dl_b = dl_bw_of(cpu);
@@ -3229,7 +3232,7 @@ static void init_dl_rq_bw_ratio(struct dl_rq *dl_rq)
 void sched_dl_do_global(void)
 {
        u64 new_bw = -1;
-       u64 gen = ++dl_generation;
+       u64 cookie = ++dl_cookie;
        struct dl_bw *dl_b;
        int cpu;
        unsigned long flags;
@@ -3240,7 +3243,7 @@ void sched_dl_do_global(void)
        for_each_possible_cpu(cpu) {
                rcu_read_lock_sched();
 
-               if (dl_bw_visited(cpu, gen)) {
+               if (dl_bw_visited(cpu, cookie)) {
                        rcu_read_unlock_sched();
                        continue;
                }

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 4b8e33c615b12c4cc2661ca2337b03333eec7076..8cebe71d2bb161c5ca3c350ae950136d4353a496 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -2910,6 +2910,7 @@ static int sched_rt_handler(const struct ctl_table *table, int write, void *buff
        int ret;
 
        mutex_lock(&mutex);
+       sched_domains_mutex_lock();
        old_period = sysctl_sched_rt_period;
        old_runtime = sysctl_sched_rt_runtime;
 
@@ -2936,6 +2937,7 @@ undo:
                sysctl_sched_rt_period = old_period;
                sysctl_sched_rt_runtime = old_runtime;
        }
+       sched_domains_mutex_unlock();
        mutex_unlock(&mutex);
 
        return ret;

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index e8915ad5652c79e0995ca43e7c9945995e515dde..5d853f91004ef1a0aade0d0597ef0f26d31f9025 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -998,7 +998,7 @@ struct root_domain {
         * Also, some corner cases, like 'wrap around' is dangerous, but given
         * that u64 is 'big enough'. So that shouldn't be a concern.
         */
-       u64 visit_gen;
+       u64 visit_cookie;
 
 #ifdef HAVE_RT_PUSH_IPI
        /*

diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 296ff2acfd321d18083e4470b4c6730cd764dc73..44093339761c91cdb27c0cf3e3fd044a85d680f3 100644 (file)
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -568,7 +568,7 @@ static int init_rootdomain(struct root_domain *rd)
        rd->rto_push_work = IRQ_WORK_INIT_HARD(rto_push_irq_work_func);
 #endif
 
-       rd->visit_gen = 0;
+       rd->visit_cookie = 0;
        init_dl_bw(&rd->dl_bw);
        if (cpudl_init(&rd->cpudl) != 0)
                goto free_rto_mask;