sched/fair: Add related data structure for task based throttle

Add related data structures for this new throttle functionality.

Tesed-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Valentin Schneider <vschneid@redhat.com>
Signed-off-by: Aaron Lu <ziqianlu@bytedance.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Chengming Zhou <chengming.zhou@linux.dev>
Tested-by: Valentin Schneider <vschneid@redhat.com>
Tested-by: Matteo Martelli <matteo.martelli@codethink.co.uk>
Link: https://lore.kernel.org/r/20250829081120.806-2-ziqianlu@bytedance.com

diff --git a/include/linux/sched.h b/include/linux/sched.h
index f8188b8333503cb252366779b0117f0e74d11218..644a01bdae700d2526c9c91e7385be3523e81959 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -883,6 +883,11 @@ struct task_struct {
 
 #ifdef CONFIG_CGROUP_SCHED
        struct task_group               *sched_task_group;
+#ifdef CONFIG_CFS_BANDWIDTH
+       struct callback_head            sched_throttle_work;
+       struct list_head                throttle_node;
+       bool                            throttled;
+#endif
 #endif
 
 

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index be00629f0ba4cc5832189c0052b0b632deb4ea2e..feb750aae71b64639f890f00a91b496e7aa6059a 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4490,6 +4490,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
        p->se.cfs_rq                    = NULL;
+#ifdef CONFIG_CFS_BANDWIDTH
+       init_cfs_throttle_work(p);
+#endif
 #endif
 
 #ifdef CONFIG_SCHEDSTATS

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index b173a059315c2e076ef1d8347f8816d9c2c33937..8fff40fcbc425b5f2b4ca1ab57ddf3231c8249c5 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5748,6 +5748,18 @@ static inline int throttled_lb_pair(struct task_group *tg,
               throttled_hierarchy(dest_cfs_rq);
 }
 
+static void throttle_cfs_rq_work(struct callback_head *work)
+{
+}
+
+void init_cfs_throttle_work(struct task_struct *p)
+{
+       init_task_work(&p->sched_throttle_work, throttle_cfs_rq_work);
+       /* Protect against double add, see throttle_cfs_rq() and throttle_cfs_rq_work() */
+       p->sched_throttle_work.next = &p->sched_throttle_work;
+       INIT_LIST_HEAD(&p->throttle_node);
+}
+
 static int tg_unthrottle_up(struct task_group *tg, void *data)
 {
        struct rq *rq = data;
@@ -6472,6 +6484,7 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
        cfs_rq->runtime_enabled = 0;
        INIT_LIST_HEAD(&cfs_rq->throttled_list);
        INIT_LIST_HEAD(&cfs_rq->throttled_csd_list);
+       INIT_LIST_HEAD(&cfs_rq->throttled_limbo_list);
 }
 
 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index be9745d104f751d71787cb777c5235dce9f31aa1..a6493d255397f8533984319fe5030d73a063f397 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -739,6 +739,7 @@ struct cfs_rq {
        int                     throttle_count;
        struct list_head        throttled_list;
        struct list_head        throttled_csd_list;
+       struct list_head        throttled_limbo_list;
 #endif /* CONFIG_CFS_BANDWIDTH */
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 };
@@ -2658,6 +2659,8 @@ extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
 
 extern void init_dl_entity(struct sched_dl_entity *dl_se);
 
+extern void init_cfs_throttle_work(struct task_struct *p);
+
 #define BW_SHIFT               20
 #define BW_UNIT                        (1 << BW_SHIFT)
 #define RATIO_SHIFT            8