sched/core: Fix picking of tasks for core scheduling with DL server

* Use simple CFS pick_task for DL pick_task

  DL server's pick_task calls CFS's pick_next_task_fair(), this is wrong
  because core scheduling's pick_task only calls CFS's pick_task() for
  evaluation / checking of the CFS task (comparing across CPUs), not for
  actually affirmatively picking the next task. This causes RB tree
  corruption issues in CFS that were found by syzbot.

* Make pick_task_fair clear DL server

  A DL task pick might set ->dl_server, but it is possible the task will
  never run (say the other HT has a stop task). If the CFS task is picked
  in the future directly (say without DL server), ->dl_server will be
  set. So clear it in pick_task_fair().

This fixes the KASAN issue reported by syzbot in set_next_entity().

(DL refactoring suggestions by Vineeth Pillai).

Reported-by: Suleiman Souhlal <suleiman@google.com>
Signed-off-by: "Joel Fernandes (Google)" <joel@joelfernandes.org>
Signed-off-by: Daniel Bristot de Oliveira <bristot@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Vineeth Pillai <vineeth@bitbyteword.org>
Tested-by: Juri Lelli <juri.lelli@redhat.com>
Link: https://lore.kernel.org/r/b10489ab1f03d23e08e6097acea47442e7d6466f.1716811044.git.bristot@kernel.org

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 4edd7e2096fbfa49d97b5f836ed6f51caf31ba5a..2c1b4ee3234fd600a3315ea8e3aab7c66dde967e 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -686,7 +686,8 @@ struct sched_dl_entity {
         */
        struct rq                       *rq;
        dl_server_has_tasks_f           server_has_tasks;
-       dl_server_pick_f                server_pick;
+       dl_server_pick_f                server_pick_next;
+       dl_server_pick_f                server_pick_task;
 
 #ifdef CONFIG_RT_MUTEXES
        /*

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 747c0c51d753092385894438c2769e1a05ceb8a1..8571bc9b37cc0e82a567f1f4f8556aa0c7c44538 100644 (file)
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1664,11 +1664,13 @@ void dl_server_stop(struct sched_dl_entity *dl_se)
 
 void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,
                    dl_server_has_tasks_f has_tasks,
-                   dl_server_pick_f pick)
+                   dl_server_pick_f pick_next,
+                   dl_server_pick_f pick_task)
 {
        dl_se->rq = rq;
        dl_se->server_has_tasks = has_tasks;
-       dl_se->server_pick = pick;
+       dl_se->server_pick_next = pick_next;
+       dl_se->server_pick_task = pick_task;
 }
 
 void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq)
@@ -2399,7 +2401,12 @@ static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq)
        return __node_2_dle(left);
 }
 
-static struct task_struct *pick_task_dl(struct rq *rq)
+/*
+ * __pick_next_task_dl - Helper to pick the next -deadline task to run.
+ * @rq: The runqueue to pick the next task from.
+ * @peek: If true, just peek at the next task. Only relevant for dlserver.
+ */
+static struct task_struct *__pick_next_task_dl(struct rq *rq, bool peek)
 {
        struct sched_dl_entity *dl_se;
        struct dl_rq *dl_rq = &rq->dl;
@@ -2413,7 +2420,10 @@ again:
        WARN_ON_ONCE(!dl_se);
 
        if (dl_server(dl_se)) {
-               p = dl_se->server_pick(dl_se);
+               if (IS_ENABLED(CONFIG_SMP) && peek)
+                       p = dl_se->server_pick_task(dl_se);
+               else
+                       p = dl_se->server_pick_next(dl_se);
                if (!p) {
                        WARN_ON_ONCE(1);
                        dl_se->dl_yielded = 1;
@@ -2428,11 +2438,18 @@ again:
        return p;
 }
 
+#ifdef CONFIG_SMP
+static struct task_struct *pick_task_dl(struct rq *rq)
+{
+       return __pick_next_task_dl(rq, true);
+}
+#endif
+
 static struct task_struct *pick_next_task_dl(struct rq *rq)
 {
        struct task_struct *p;
 
-       p = pick_task_dl(rq);
+       p = __pick_next_task_dl(rq, false);
        if (!p)
                return p;
 

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 1ea5ec81431aedb961a78fd2e787e2f5a9fd8bac..ee251ac50398c27d7549a253eb578cb2763e0a6a 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -8479,6 +8479,14 @@ again:
                cfs_rq = group_cfs_rq(se);
        } while (cfs_rq);
 
+       /*
+        * This can be called from directly from CFS's ->pick_task() or indirectly
+        * from DL's ->pick_task when fair server is enabled. In the indirect case,
+        * DL will set ->dl_server just after this function is called, so its Ok to
+        * clear. In the direct case, we are picking directly so we must clear it.
+        */
+       task_of(se)->dl_server = NULL;
+
        return task_of(se);
 }
 #endif
@@ -8638,7 +8646,16 @@ static bool fair_server_has_tasks(struct sched_dl_entity *dl_se)
        return !!dl_se->rq->cfs.nr_running;
 }
 
-static struct task_struct *fair_server_pick(struct sched_dl_entity *dl_se)
+static struct task_struct *fair_server_pick_task(struct sched_dl_entity *dl_se)
+{
+#ifdef CONFIG_SMP
+       return pick_task_fair(dl_se->rq);
+#else
+       return NULL;
+#endif
+}
+
+static struct task_struct *fair_server_pick_next(struct sched_dl_entity *dl_se)
 {
        return pick_next_task_fair(dl_se->rq, NULL, NULL);
 }
@@ -8649,7 +8666,9 @@ void fair_server_init(struct rq *rq)
 
        init_dl_entity(dl_se);
 
-       dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick);
+       dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_next,
+                      fair_server_pick_task);
+
 }
 
 /*

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b777ac361543ec402c1d66c845ca45814431c7e3..f7e028b2e34e080100230568b29e8d9c6f8e879a 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -361,7 +361,8 @@ extern void dl_server_start(struct sched_dl_entity *dl_se);
 extern void dl_server_stop(struct sched_dl_entity *dl_se);
 extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,
                    dl_server_has_tasks_f has_tasks,
-                   dl_server_pick_f pick);
+                   dl_server_pick_f pick_next,
+                   dl_server_pick_f pick_task);
 
 extern void dl_server_update_idle_time(struct rq *rq,
                    struct task_struct *p);