static struct rhashtable scx_sched_hash;
#endif
+/* see SCX_OPS_TID_TO_TASK */
+static const struct rhashtable_params scx_tid_hash_params = {
+ .key_len = sizeof_field(struct sched_ext_entity, tid),
+ .key_offset = offsetof(struct sched_ext_entity, tid),
+ .head_offset = offsetof(struct sched_ext_entity, tid_hash_node),
+ .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */
+};
+static struct rhashtable scx_tid_hash;
+
/*
* During exit, a task may schedule after losing its PIDs. When disabling the
* BPF scheduler, we need to be able to iterate tasks in every state to
static bool scx_init_task_enabled;
static bool scx_switching_all;
DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
+static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled);
+
+/*
+ * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler
+ * and the tid->task table is live. Wraps the static key so callers don't
+ * take the address, and hints "likely enabled" for the common case where
+ * the feature is in use.
+ */
+static inline bool scx_tid_to_task_enabled(void)
+{
+ return static_branch_likely(&__scx_tid_to_task_enabled);
+}
static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);
static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
+/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */
+static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1);
+
#ifdef CONFIG_EXT_SUB_SCHED
/*
* The sub sched being enabled. Used by scx_disable_and_exit_task() to exit
static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs);
+/*
+ * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of
+ * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without
+ * further synchronization. See scx_alloc_tid().
+ */
+struct scx_tid_alloc {
+ u64 next;
+ u64 end;
+};
+static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc);
+
/*
* Direct dispatch marker.
*
scx->slice = SCX_SLICE_DFL;
}
+/* See scx_tid_alloc / scx_tid_cursor. */
+static u64 scx_alloc_tid(void)
+{
+ struct scx_tid_alloc *ta;
+
+ guard(preempt)();
+ ta = this_cpu_ptr(&scx_tid_alloc);
+
+ if (unlikely(ta->next >= ta->end)) {
+ ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor);
+ ta->end = ta->next + SCX_TID_CHUNK;
+ }
+ return ta->next++;
+}
+
+static void scx_tid_hash_insert(struct task_struct *p)
+{
+ int ret;
+
+ lockdep_assert_held(&scx_tasks_lock);
+
+ ret = rhashtable_lookup_insert_fast(&scx_tid_hash,
+ &p->scx.tid_hash_node,
+ scx_tid_hash_params);
+ WARN_ON_ONCE(ret);
+}
+
void scx_pre_fork(struct task_struct *p)
{
/*
percpu_rwsem_assert_held(&scx_fork_rwsem);
+ p->scx.tid = scx_alloc_tid();
+
if (scx_init_task_enabled) {
#ifdef CONFIG_EXT_SUB_SCHED
struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched;
}
}
- raw_spin_lock_irq(&scx_tasks_lock);
- list_add_tail(&p->scx.tasks_node, &scx_tasks);
- raw_spin_unlock_irq(&scx_tasks_lock);
+ scoped_guard(raw_spinlock_irq, &scx_tasks_lock) {
+ list_add_tail(&p->scx.tasks_node, &scx_tasks);
+ if (scx_tid_to_task_enabled())
+ scx_tid_hash_insert(p);
+ }
percpu_up_read(&scx_fork_rwsem);
}
void sched_ext_dead(struct task_struct *p)
{
- unsigned long flags;
-
/*
* By the time control reaches here, @p has %TASK_DEAD set, switched out
* for the last time and then dropped the rq lock - task_dead_and_done()
* should be returning %true nullifying the straggling sched_class ops.
* Remove from scx_tasks and exit @p.
*/
- raw_spin_lock_irqsave(&scx_tasks_lock, flags);
- list_del_init(&p->scx.tasks_node);
- raw_spin_unlock_irqrestore(&scx_tasks_lock, flags);
+ scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) {
+ list_del_init(&p->scx.tasks_node);
+ if (scx_tid_to_task_enabled())
+ rhashtable_remove_fast(&scx_tid_hash,
+ &p->scx.tid_hash_node,
+ scx_tid_hash_params);
+ }
/*
* @p is off scx_tasks and wholly ours. scx_root_enable()'s READY ->
/* no task is on scx, turn off all the switches and flush in-progress calls */
static_branch_disable(&__scx_enabled);
+ if (sch->ops.flags & SCX_OPS_TID_TO_TASK)
+ static_branch_disable(&__scx_tid_to_task_enabled);
bitmap_zero(sch->has_op, SCX_OPI_END);
scx_idle_disable();
synchronize_rcu();
+ if (sch->ops.flags & SCX_OPS_TID_TO_TASK)
+ rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
scx_log_sched_disable(sch);
return -EINVAL;
}
+ /*
+ * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched
+ * may set it to declare a dependency; reject if the root hasn't
+ * enabled it.
+ */
+ if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) &&
+ !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) {
+ scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it");
+ return -EINVAL;
+ }
+
/*
* SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle
* selection policy to be enabled.
if (ret)
goto err_unlock;
+ if (ops->flags & SCX_OPS_TID_TO_TASK) {
+ ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params);
+ if (ret)
+ goto err_free_ksyncs;
+ }
+
#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED)
cgroup_get(cgrp);
#endif
sch = scx_alloc_and_add_sched(ops, cgrp, NULL);
if (IS_ERR(sch)) {
ret = PTR_ERR(sch);
- goto err_free_ksyncs;
+ goto err_free_tid_hash;
}
/*
WARN_ON_ONCE(scx_init_task_enabled);
scx_init_task_enabled = true;
+ /* flip under fork_rwsem; the iter below covers existing tasks */
+ if (ops->flags & SCX_OPS_TID_TO_TASK)
+ static_branch_enable(&__scx_tid_to_task_enabled);
+
/*
* Enable ops for every task. Fork is excluded by scx_fork_rwsem
* preventing new tasks from being added. No need to exclude tasks
scx_set_task_sched(p, sch);
scx_set_task_state(p, SCX_TASK_READY);
+ /*
+ * Insert into the tid hash under scx_tasks_lock so we can't
+ * race sched_ext_dead() and leave a stale entry for an already
+ * exited task.
+ */
+ if (scx_tid_to_task_enabled()) {
+ guard(raw_spinlock_irq)(&scx_tasks_lock);
+ if (!list_empty(&p->scx.tasks_node))
+ scx_tid_hash_insert(p);
+ }
+
put_task_struct(p);
}
scx_task_iter_stop(&sti);
cmd->ret = 0;
return;
+err_free_tid_hash:
+ if (ops->flags & SCX_OPS_TID_TO_TASK)
+ rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
err_free_ksyncs:
free_kick_syncs();
err_unlock:
return rcu_dereference(cpu_rq(cpu)->curr);
}
+/**
+ * scx_bpf_tid_to_task - Look up a task by its scx tid
+ * @tid: task ID previously read from p->scx.tid
+ *
+ * Returns the task with the given tid, or NULL if no such task exists. The
+ * returned pointer is valid until the end of the current RCU read section
+ * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root
+ * scheduler; otherwise an error is raised and NULL returned.
+ */
+__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid)
+{
+ struct sched_ext_entity *scx;
+
+ if (!scx_tid_to_task_enabled()) {
+ struct scx_sched *sch = rcu_dereference(scx_root);
+
+ if (sch)
+ scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK");
+ return NULL;
+ }
+
+ scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params);
+ if (!scx)
+ return NULL;
+
+ return container_of(scx, struct task_struct, scx);
+}
+
/**
* scx_bpf_now - Returns a high-performance monotonically non-decreasing
* clock for the current CPU. The clock returned is in nanoseconds.
BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL)
BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
+BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED)
BTF_ID_FLAGS(func, scx_bpf_now)
BTF_ID_FLAGS(func, scx_bpf_events)
#ifdef CONFIG_CGROUP_SCHED
SCX_DSP_MAX_LOOPS = 32,
SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ,
+ /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */
+ SCX_TID_CHUNK = 1024,
+
SCX_EXIT_BT_LEN = 64,
SCX_EXIT_MSG_LEN = 1024,
SCX_EXIT_DUMP_DFL_LEN = 32768,
* To mask this problem, by default, unhashed tasks are automatically
* dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't
* depend on pid lookups and wants to handle these tasks directly, the
- * following flag can be used.
+ * following flag can be used. With %SCX_OPS_TID_TO_TASK,
+ * scx_bpf_tid_to_task() can find exiting tasks reliably.
*/
SCX_OPS_ENQ_EXITING = 1LLU << 2,
*/
SCX_OPS_ALWAYS_ENQ_IMMED = 1LLU << 7,
+ /*
+ * Maintain a mapping from p->scx.tid to task_struct so the BPF
+ * scheduler can recover task pointers from stored tids via
+ * scx_bpf_tid_to_task().
+ *
+ * Only the root scheduler turns this on. A sub-sched may set the flag
+ * to declare a dependency on the lookup; if the root scheduler hasn't
+ * enabled it, attaching the sub-sched is rejected.
+ */
+ SCX_OPS_TID_TO_TASK = 1LLU << 8,
+
SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE |
SCX_OPS_ENQ_LAST |
SCX_OPS_ENQ_EXITING |
SCX_OPS_ALLOW_QUEUED_WAKEUP |
SCX_OPS_SWITCH_PARTIAL |
SCX_OPS_BUILTIN_IDLE_PER_NODE |
- SCX_OPS_ALWAYS_ENQ_IMMED,
+ SCX_OPS_ALWAYS_ENQ_IMMED |
+ SCX_OPS_TID_TO_TASK,
/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */
__SCX_OPS_INTERNAL_MASK = 0xffLLU << 56,
projects / thirdparty / linux.git / commitdiff
