sched/psi: Create the psimon kthread outside of cgroup_mutex

a5b98009f16d ("sched/psi: fix race between file release and pressure write")
made pressure_write() hold cgroup_mutex across psi_trigger_create(), which
forks the psimon kthread for the first rtpoll trigger. As kthread creation
depends on the whole fork path, the commit inadvertently created a lot of
unwanted locking dependencies from cgroup_mutex.

sched_ext got hit by one: its enable path blocks forks and then grabs
cgroup_mutex, so a pressure write racing a scheduler enable deadlocks, with
every other fork piling up behind.

Fix it by splitting trigger creation so that the worker is forked with
cgroup_mutex dropped and the kernfs active reference left broken. The latter
matters because rmdir and cgroup.pressure writes drain active references
under cgroup_mutex. Publishing the trigger last keeps error reporting
synchronous and preserves the of->priv lifetime rules.

The trigger registered in the first stage pins the group's rtpoll machinery
across the unlocked window, leaving only creation races to resolve. The
catch-up poll on installation covers scheduling attempts dropped while there
was no worker.

v2: Retagged sched/psi (was cgroup).

Fixes: a5b98009f16d ("sched/psi: fix race between file release and pressure write")
Cc: stable@vger.kernel.org
Cc: Edward Adam Davis <eadavis@qq.com>
Cc: Chen Ridong <chenridong@huaweicloud.com>
Reported-by: Matt Fleming <mfleming@cloudflare.com>
Closes: https://lore.kernel.org/all/20260710100441.2653477-1-matt@readmodwrite.com/
Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Tested-by: Matt Fleming <mfleming@cloudflare.com>
Acked-by: Suren Baghdasaryan <surenb@google.com>

diff --git a/include/linux/psi.h b/include/linux/psi.h
index e0745873e3f26cf4c471aac72c0e1ed373fbb662..7966e3ac03b99be2b73d4428233d3bcd690c6e28 100644 (file)
--- a/include/linux/psi.h
+++ b/include/linux/psi.h
@@ -25,7 +25,9 @@ void psi_memstall_leave(unsigned long *flags);
 int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res);
 struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
                                       enum psi_res res, struct file *file,
-                                      struct kernfs_open_file *of);
+                                      struct kernfs_open_file *of,
+                                      bool *need_rtpoll_worker);
+int psi_trigger_create_rtpoll_worker(struct psi_group *group);
 void psi_trigger_destroy(struct psi_trigger *t);
 
 __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file,

diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 38f8d9df8fbc4ac4f3002ecab731c759deab57f1..b5b461d4418b19a11254481ee69d729bdbcf16b1 100644 (file)
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -3996,6 +3996,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
        struct psi_trigger *new;
        struct cgroup *cgrp;
        struct psi_group *psi;
+       bool need_rtpoll_worker;
        ssize_t ret = 0;
 
        cgrp = cgroup_kn_lock_live(of->kn, false);
@@ -4015,12 +4016,32 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
        }
 
        psi = cgroup_psi(cgrp);
-       new = psi_trigger_create(psi, buf, res, of->file, of);
+       new = psi_trigger_create(psi, buf, res, of->file, of,
+                                &need_rtpoll_worker);
        if (IS_ERR(new)) {
                ret = PTR_ERR(new);
                goto out_unlock;
        }
 
+       /*
+        * The worker fork must run with neither cgroup_mutex nor the file's
+        * kernfs active reference held. The latter is broken since
+        * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so
+        * recheck before publishing @new.
+        */
+       if (need_rtpoll_worker) {
+               cgroup_unlock();
+               ret = psi_trigger_create_rtpoll_worker(psi);
+               cgroup_lock();
+
+               if (!ret && !of->priv)
+                       ret = -ENODEV;
+               if (ret) {
+                       psi_trigger_destroy(new);
+                       goto out_unlock;
+               }
+       }
+
        smp_store_release(&ctx->psi.trigger, new);
 
 out_unlock:

diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index d9c9d9480a45b735a996cc22f5d7a4bef6545a3c..565ec7b807432b7262d203bb66a926b1b1332b97 100644 (file)
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -1292,9 +1292,44 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)
        return 0;
 }
 
+/*
+ * Create @group's rtpoll worker after psi_trigger_create() reported the need
+ * for one. kthread creation depends on the whole fork path and we don't want
+ * all of that nested inside cgroup_mutex, so the caller must drop it and any
+ * other lock that forks can wait behind. If two callers race, the loser stops
+ * its never-woken kthread.
+ */
+int psi_trigger_create_rtpoll_worker(struct psi_group *group)
+{
+       struct task_struct *task;
+
+       task = kthread_create(psi_rtpoll_worker, group, "psimon");
+       if (IS_ERR(task))
+               return PTR_ERR(task);
+
+       scoped_guard(mutex, &group->rtpoll_trigger_lock) {
+               if (!rcu_access_pointer(group->rtpoll_task)) {
+                       atomic_set(&group->rtpoll_wakeup, 0);
+                       wake_up_process(task);
+                       rcu_assign_pointer(group->rtpoll_task, task);
+
+                       /*
+                        * Poll once to catch up on scheduling attempts dropped
+                        * while there was no rtpoll worker.
+                        */
+                       psi_schedule_rtpoll_work(group, 1, true);
+                       return 0;
+               }
+       }
+
+       kthread_stop(task);
+       return 0;
+}
+
 struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
                                       enum psi_res res, struct file *file,
-                                      struct kernfs_open_file *of)
+                                      struct kernfs_open_file *of,
+                                      bool *need_rtpoll_worker)
 {
        struct psi_trigger *t;
        enum psi_states state;
@@ -1302,6 +1337,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
        bool privileged;
        u32 window_us;
 
+       *need_rtpoll_worker = false;
+
        if (static_branch_likely(&psi_disabled))
                return ERR_PTR(-EOPNOTSUPP);
 
@@ -1362,26 +1399,14 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
        if (privileged) {
                mutex_lock(&group->rtpoll_trigger_lock);
 
-               if (!rcu_access_pointer(group->rtpoll_task)) {
-                       struct task_struct *task;
-
-                       task = kthread_create(psi_rtpoll_worker, group, "psimon");
-                       if (IS_ERR(task)) {
-                               kfree(t);
-                               mutex_unlock(&group->rtpoll_trigger_lock);
-                               return ERR_CAST(task);
-                       }
-                       atomic_set(&group->rtpoll_wakeup, 0);
-                       wake_up_process(task);
-                       rcu_assign_pointer(group->rtpoll_task, task);
-               }
-
                list_add(&t->node, &group->rtpoll_triggers);
                group->rtpoll_min_period = min(group->rtpoll_min_period,
                        div_u64(t->win.size, UPDATES_PER_WINDOW));
                group->rtpoll_nr_triggers[t->state]++;
                group->rtpoll_states |= (1 << t->state);
 
+               *need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task);
+
                mutex_unlock(&group->rtpoll_trigger_lock);
        } else {
                mutex_lock(&group->avgs_lock);
@@ -1541,6 +1566,8 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf,
        size_t buf_size;
        struct seq_file *seq;
        struct psi_trigger *new;
+       bool need_rtpoll_worker;
+       int ret;
 
        if (static_branch_likely(&psi_disabled))
                return -EOPNOTSUPP;
@@ -1565,12 +1592,22 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf,
                return -EBUSY;
        }
 
-       new = psi_trigger_create(&psi_system, buf, res, file, NULL);
+       new = psi_trigger_create(&psi_system, buf, res, file, NULL,
+                                &need_rtpoll_worker);
        if (IS_ERR(new)) {
                mutex_unlock(&seq->lock);
                return PTR_ERR(new);
        }
 
+       if (need_rtpoll_worker) {
+               ret = psi_trigger_create_rtpoll_worker(&psi_system);
+               if (ret) {
+                       psi_trigger_destroy(new);
+                       mutex_unlock(&seq->lock);
+                       return ret;
+               }
+       }
+
        smp_store_release(&seq->private, new);
        mutex_unlock(&seq->lock);