#define CREATE_TRACE_POINTS
#include <trace/events/sched_ext.h>
-static void process_ddsp_deferred_locals(struct rq *rq);
+static void run_deferred(struct rq *rq);
static bool task_dead_and_done(struct task_struct *p);
-static u32 reenq_local(struct scx_sched *sch, struct rq *rq);
static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind,
return -EPROTO;
}
-static void run_deferred(struct rq *rq)
-{
- process_ddsp_deferred_locals(rq);
-
- if (!llist_empty(&rq->scx.deferred_reenq_locals)) {
- struct llist_node *llist =
- llist_del_all(&rq->scx.deferred_reenq_locals);
- struct scx_sched_pcpu *pos, *next;
-
- llist_for_each_entry_safe(pos, next, llist,
- deferred_reenq_locals_node) {
- init_llist_node(&pos->deferred_reenq_locals_node);
- reenq_local(pos->sch, rq);
- }
- }
-}
-
static void deferred_bal_cb_workfn(struct rq *rq)
{
run_deferred(rq);
return true;
}
-static void process_ddsp_deferred_locals(struct rq *rq)
-{
- struct task_struct *p;
-
- lockdep_assert_rq_held(rq);
-
- /*
- * Now that @rq can be unlocked, execute the deferred enqueueing of
- * tasks directly dispatched to the local DSQs of other CPUs. See
- * direct_dispatch(). Keep popping from the head instead of using
- * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
- * temporarily.
- */
- while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
- struct task_struct, scx.dsq_list.node))) {
- struct scx_sched *sch = scx_task_sched(p);
- struct scx_dispatch_q *dsq;
-
- list_del_init(&p->scx.dsq_list.node);
-
- dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
- if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
- dispatch_to_local_dsq(sch, rq, dsq, p,
- p->scx.ddsp_enq_flags);
- }
-}
-
static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
{
struct scx_sched *sch = scx_task_sched(p);
rq->scx.flags &= ~SCX_RQ_ONLINE;
}
-
static bool check_rq_for_timeouts(struct rq *rq)
{
struct scx_sched *sch;
return 0;
}
+static void process_ddsp_deferred_locals(struct rq *rq)
+{
+ struct task_struct *p;
+
+ lockdep_assert_rq_held(rq);
+
+ /*
+ * Now that @rq can be unlocked, execute the deferred enqueueing of
+ * tasks directly dispatched to the local DSQs of other CPUs. See
+ * direct_dispatch(). Keep popping from the head instead of using
+ * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
+ * temporarily.
+ */
+ while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
+ struct task_struct, scx.dsq_list.node))) {
+ struct scx_sched *sch = scx_task_sched(p);
+ struct scx_dispatch_q *dsq;
+
+ list_del_init(&p->scx.dsq_list.node);
+
+ dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
+ if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
+ dispatch_to_local_dsq(sch, rq, dsq, p,
+ p->scx.ddsp_enq_flags);
+ }
+}
+
+static u32 reenq_local(struct scx_sched *sch, struct rq *rq)
+{
+ LIST_HEAD(tasks);
+ u32 nr_enqueued = 0;
+ struct task_struct *p, *n;
+
+ lockdep_assert_rq_held(rq);
+
+ /*
+ * The BPF scheduler may choose to dispatch tasks back to
+ * @rq->scx.local_dsq. Move all candidate tasks off to a private list
+ * first to avoid processing the same tasks repeatedly.
+ */
+ list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
+ scx.dsq_list.node) {
+ struct scx_sched *task_sch = scx_task_sched(p);
+
+ /*
+ * If @p is being migrated, @p's current CPU may not agree with
+ * its allowed CPUs and the migration_cpu_stop is about to
+ * deactivate and re-activate @p anyway. Skip re-enqueueing.
+ *
+ * While racing sched property changes may also dequeue and
+ * re-enqueue a migrating task while its current CPU and allowed
+ * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
+ * the current local DSQ for running tasks and thus are not
+ * visible to the BPF scheduler.
+ */
+ if (p->migration_pending)
+ continue;
+
+ if (!scx_is_descendant(task_sch, sch))
+ continue;
+
+ dispatch_dequeue(rq, p);
+ list_add_tail(&p->scx.dsq_list.node, &tasks);
+ }
+
+ list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
+ list_del_init(&p->scx.dsq_list.node);
+ do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+ nr_enqueued++;
+ }
+
+ return nr_enqueued;
+}
+
+static void run_deferred(struct rq *rq)
+{
+ process_ddsp_deferred_locals(rq);
+
+ if (!llist_empty(&rq->scx.deferred_reenq_locals)) {
+ struct llist_node *llist =
+ llist_del_all(&rq->scx.deferred_reenq_locals);
+ struct scx_sched_pcpu *pos, *next;
+
+ llist_for_each_entry_safe(pos, next, llist,
+ deferred_reenq_locals_node) {
+ init_llist_node(&pos->deferred_reenq_locals_node);
+ reenq_local(pos->sch, rq);
+ }
+ }
+}
+
#ifdef CONFIG_NO_HZ_FULL
bool scx_can_stop_tick(struct rq *rq)
{
.set = &scx_kfunc_ids_dispatch,
};
-static u32 reenq_local(struct scx_sched *sch, struct rq *rq)
-{
- LIST_HEAD(tasks);
- u32 nr_enqueued = 0;
- struct task_struct *p, *n;
-
- lockdep_assert_rq_held(rq);
-
- /*
- * The BPF scheduler may choose to dispatch tasks back to
- * @rq->scx.local_dsq. Move all candidate tasks off to a private list
- * first to avoid processing the same tasks repeatedly.
- */
- list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
- scx.dsq_list.node) {
- struct scx_sched *task_sch = scx_task_sched(p);
-
- /*
- * If @p is being migrated, @p's current CPU may not agree with
- * its allowed CPUs and the migration_cpu_stop is about to
- * deactivate and re-activate @p anyway. Skip re-enqueueing.
- *
- * While racing sched property changes may also dequeue and
- * re-enqueue a migrating task while its current CPU and allowed
- * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
- * the current local DSQ for running tasks and thus are not
- * visible to the BPF scheduler.
- */
- if (p->migration_pending)
- continue;
-
- if (!scx_is_descendant(task_sch, sch))
- continue;
-
- dispatch_dequeue(rq, p);
- list_add_tail(&p->scx.dsq_list.node, &tasks);
- }
-
- list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
- list_del_init(&p->scx.dsq_list.node);
- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
- nr_enqueued++;
- }
-
- return nr_enqueued;
-}
-
__bpf_kfunc_start_defs();
/**
projects / thirdparty / kernel / linux.git / commitdiff
