if (!tryget_task_struct(p))
continue;
+ /*
+ * Set %INIT_BEGIN under the iter's rq lock so that a concurrent
+ * sched_ext_dead() does not call ops.exit_task() on @p while
+ * ops.init_task() is running. If sched_ext_dead() runs before
+ * this store, it has already removed @p from scx_tasks and the
+ * iter won't visit @p; if it runs after, it observes
+ * %INIT_BEGIN and transitions to %DEAD without calling ops,
+ * leaving the post-init recheck below to unwind.
+ */
+ scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
scx_task_iter_unlock(&sti);
- ret = scx_init_task(sch, p, false);
+ ret = __scx_init_task(sch, p, false);
- rq = task_rq_lock(p, &rf);
+ scx_task_iter_relock(&sti, p);
- if (ret) {
+ if (unlikely(ret)) {
+ if (scx_get_task_state(p) != SCX_TASK_DEAD)
+ scx_set_task_state(p, SCX_TASK_NONE);
- task_rq_unlock(rq, p, &rf);
put_task_struct(p);
scx_task_iter_stop(&sti);
scx_error(sch, "ops.init_task() failed (%d) for %s[%d]",
goto err_disable_unlock_all;
}
- scx_set_task_sched(p, sch);
- scx_set_task_state(p, SCX_TASK_READY);
+ if (scx_get_task_state(p) == SCX_TASK_DEAD) {
+ /*
+ * sched_ext_dead() observed %INIT_BEGIN and set %DEAD.
+ * ops.exit_task() is owed to the sched __scx_init_task()
+ * ran against; call it now.
+ */
+ scx_sub_init_cancel_task(sch, p);
+ } else {
+ scx_set_task_state(p, SCX_TASK_INIT);
+ scx_set_task_sched(p, sch);
+ scx_set_task_state(p, SCX_TASK_READY);
+ }
- task_rq_unlock(rq, p, &rf);
+ /*
+ * Insert into the tid hash. scx_tasks_lock is held by the iter;
+ * list_empty() guards against sched_ext_dead() having taken @p
+ * off the list while init ran unlocked.
+ */
+ if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node))
+ scx_tid_hash_insert(p);
+
put_task_struct(p);
}
scx_task_iter_stop(&sti);