sched/fair: Remove task_group->se pointer array

Now that struct sched_entity is co-located with struct cfs_rq for non-root task
groups, the task_group->se pointer array is redundant. The associated
sched_entity can be loaded directly from the cfs_rq.

This patch performs the access conversion with the helpers:

 - is_root_task_group(tg): checks if a task group is the root task group. It
   compares the task group's address with the global root_task_group variable.

 - tg_se(tg, cpu): retrieves the cfs_rq and returns the address of the
   co-located se. This function checks if tg is the root task group to ensure
   behaving the same of previous tg->se[cpu]. Replaces all accesses that use
   the tg->se[cpu] pointer array with calls to the new tg_se(tg, cpu) accessor.

 - cfs_rq_se(cfs_rq): simplifies access paths like cfs_rq->tg->se[...] to use
   the co-located sched_entity. This function also checks if tg is the root
   task group to ensure same behavior.

Since tg_se is not in very hot code paths, and the branch is a register
comparison with an immediate value (`&root_task_group`), the performance impact
is expected to be negligible.

Signed-off-by: Zecheng Li <zecheng@google.com>
Signed-off-by: Zecheng Li <zli94@ncsu.edu>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: K Prateek Nayak <kprateek.nayak@amd.com>
Reviewed-by: Josh Don <joshdon@google.com>
Link: https://patch.msgid.link/20260522141623.600235-3-zli94@ncsu.edu

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2cfe8932d5dbe7df72ddc18e156a6afe043fdf35..39cea012c230ebca0f1170534d8dff686c11fdec 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -8923,7 +8923,7 @@ void __init sched_init(void)
        wait_bit_init();
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-       ptr += 2 * nr_cpu_ids * sizeof(void **);
+       ptr += nr_cpu_ids * sizeof(void **);
 #endif
 #ifdef CONFIG_RT_GROUP_SCHED
        ptr += 2 * nr_cpu_ids * sizeof(void **);
@@ -8932,9 +8932,6 @@ void __init sched_init(void)
                ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-               root_task_group.se = (struct sched_entity **)ptr;
-               ptr += nr_cpu_ids * sizeof(void **);
-
                root_task_group.cfs_rq = (struct cfs_rq **)ptr;
                ptr += nr_cpu_ids * sizeof(void **);
 
@@ -10016,7 +10013,7 @@ static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
                int i;
 
                for_each_possible_cpu(i) {
-                       stats = __schedstats_from_se(tg->se[i]);
+                       stats = __schedstats_from_se(tg_se(tg, i));
                        ws += schedstat_val(stats->wait_sum);
                }
 

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 5e09cf9fae3e0df8ad588025b315582b7de313c4..40584b27ea0ce37c27409d4c400545cd6f7a2977 100644 (file)
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -808,7 +808,7 @@ void dirty_sched_domain_sysctl(int cpu)
 #ifdef CONFIG_FAIR_GROUP_SCHED
 static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
 {
-       struct sched_entity *se = tg->se[cpu];
+       struct sched_entity *se = tg_se(tg, cpu);
 
 #define P(F)           SEQ_printf(m, "  .%-30s: %lld\n",       #F, (long long)F)
 #define P_SCHEDSTAT(F) SEQ_printf(m, "  .%-30s: %lld\n",       \

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e7d7d47ef7b2d1f7109b5517bb10d623a194c13f..447b0ac426d1438f521d3cc29156f4caddcba626 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6876,7 +6876,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
 {
        struct rq *rq = rq_of(cfs_rq);
        struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
-       struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
+       struct sched_entity *se = cfs_rq_se(cfs_rq);
 
        /*
         * It's possible we are called with runtime_remaining < 0 due to things
@@ -11102,7 +11102,6 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
 {
        struct cfs_rq *cfs_rq, *pos;
        bool decayed = false;
-       int cpu = cpu_of(rq);
 
        /*
         * Iterates the task_group tree in a bottom up fashion, see
@@ -11122,7 +11121,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
                }
 
                /* Propagate pending load changes to the parent, if any: */
-               se = cfs_rq->tg->se[cpu];
+               se = cfs_rq_se(cfs_rq);
                if (se && !skip_blocked_update(se))
                        update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
 
@@ -11148,8 +11147,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
  */
 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
 {
-       struct rq *rq = rq_of(cfs_rq);
-       struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
+       struct sched_entity *se = cfs_rq_se(cfs_rq);
        unsigned long now = jiffies;
        unsigned long load;
 
@@ -15086,7 +15084,6 @@ void free_fair_sched_group(struct task_group *tg)
        }
 
        kfree(tg->cfs_rq);
-       kfree(tg->se);
 }
 
 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
@@ -15099,9 +15096,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
        tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids);
        if (!tg->cfs_rq)
                goto err;
-       tg->se = kzalloc_objs(se, nr_cpu_ids);
-       if (!tg->se)
-               goto err;
 
        tg->shares = NICE_0_LOAD;
 
@@ -15116,7 +15110,7 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
                cfs_rq = &state->cfs_rq;
                se = &state->se;
                init_cfs_rq(cfs_rq);
-               init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
+               init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
                init_entity_runnable_average(se);
        }
 
@@ -15135,7 +15129,7 @@ void online_fair_sched_group(struct task_group *tg)
 
        for_each_possible_cpu(i) {
                rq = cpu_rq(i);
-               se = tg->se[i];
+               se = tg_se(tg, i);
                rq_lock_irq(rq, &rf);
                update_rq_clock(rq);
                attach_entity_cfs_rq(se);
@@ -15152,7 +15146,7 @@ void unregister_fair_sched_group(struct task_group *tg)
 
        for_each_possible_cpu(cpu) {
                struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
-               struct sched_entity *se = tg->se[cpu];
+               struct sched_entity *se = tg_se(tg, cpu);
                struct rq *rq = cpu_rq(cpu);
 
                if (se) {
@@ -15189,7 +15183,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
        init_cfs_rq_runtime(cfs_rq);
 
        tg->cfs_rq[cpu] = cfs_rq;
-       tg->se[cpu] = se;
 
        /* se could be NULL for root_task_group */
        if (!se)
@@ -15220,7 +15213,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
        /*
         * We can't change the weight of the root cgroup.
         */
-       if (!tg->se[0])
+       if (is_root_task_group(tg))
                return -EINVAL;
 
        shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
@@ -15231,7 +15224,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
        tg->shares = shares;
        for_each_possible_cpu(i) {
                struct rq *rq = cpu_rq(i);
-               struct sched_entity *se = tg->se[i];
+               struct sched_entity *se = tg_se(tg, i);
                struct rq_flags rf;
 
                /* Propagate contribution to hierarchy */
@@ -15282,7 +15275,7 @@ int sched_group_set_idle(struct task_group *tg, long idle)
 
        for_each_possible_cpu(i) {
                struct rq *rq = cpu_rq(i);
-               struct sched_entity *se = tg->se[i];
+               struct sched_entity *se = tg_se(tg, i);
                struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];
                bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
                long idle_task_delta;

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 585aba9f63b4119dec5dd29e25e518d178d54dba..823ba40cf0981a71d69ac55e5a7154ff96ce86c9 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -484,8 +484,6 @@ struct task_group {
 #endif
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-       /* schedulable entities of this group on each CPU */
-       struct sched_entity     **se;
        /* runqueue "owned" by this group on each CPU */
        struct cfs_rq           **cfs_rq;
        unsigned long           shares;
@@ -934,7 +932,8 @@ struct dl_rq {
 };
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-
+/* Check whether a task group is root tg */
+#define is_root_task_group(tg) ((tg) == &root_task_group)
 /* An entity is a task if it doesn't "own" a runqueue */
 #define entity_is_task(se)     (!se->my_q)
 
@@ -2304,6 +2303,28 @@ struct cfs_tg_state {
        struct sched_entity     se;
        struct sched_statistics stats;
 } __no_randomize_layout;
+
+static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
+{
+       struct cfs_tg_state *state;
+
+       if (is_root_task_group(tg))
+               return NULL;
+
+       state = container_of(tg->cfs_rq[cpu], struct cfs_tg_state, cfs_rq);
+       return &state->se;
+}
+
+static inline struct sched_entity *cfs_rq_se(struct cfs_rq *cfs_rq)
+{
+       struct cfs_tg_state *state;
+
+       if (is_root_task_group(cfs_rq->tg))
+               return NULL;
+
+       state = container_of(cfs_rq, struct cfs_tg_state, cfs_rq);
+       return &state->se;
+}
 #endif
 
 /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
@@ -2316,8 +2337,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 #ifdef CONFIG_FAIR_GROUP_SCHED
        set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
        p->se.cfs_rq = tg->cfs_rq[cpu];
-       p->se.parent = tg->se[cpu];
-       p->se.depth = tg->se[cpu] ? tg->se[cpu]->depth + 1 : 0;
+       p->se.parent = tg_se(tg, cpu);
+       p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED