sched_ext: Add SCX_TASK_REENQ_REASON flags

SCX_ENQ_REENQ indicates that a task is being re-enqueued but doesn't tell the
BPF scheduler why. Add SCX_TASK_REENQ_REASON flags using bits 12-13 of
p->scx.flags to communicate the reason during ops.enqueue():

- NONE: Not being reenqueued
- KFUNC: Reenqueued by scx_bpf_dsq_reenq() and friends

More reasons will be added.

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>

diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index e822b374b17f822a9bfcf146c90146f72a98449d..60a4f65d0174acf6447a5e4c9dbd9bf76a258b61 100644 (file)
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -118,6 +118,21 @@ enum scx_ent_flags {
        SCX_TASK_READY          = 2 << SCX_TASK_STATE_SHIFT,
        SCX_TASK_ENABLED        = 3 << SCX_TASK_STATE_SHIFT,
 
+       /*
+        * Bits 12 and 13 are used to carry reenqueue reason. In addition to
+        * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for
+        * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues.
+        *
+        * NONE         not being reenqueued
+        * KFUNC        reenqueued by scx_bpf_dsq_reenq() and friends
+        */
+       SCX_TASK_REENQ_REASON_SHIFT = 12,
+       SCX_TASK_REENQ_REASON_BITS = 2,
+       SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT,
+
+       SCX_TASK_REENQ_NONE     = 0 << SCX_TASK_REENQ_REASON_SHIFT,
+       SCX_TASK_REENQ_KFUNC    = 1 << SCX_TASK_REENQ_REASON_SHIFT,
+
        /* iteration cursor, not a task */
        SCX_TASK_CURSOR         = 1 << 31,
 };

diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index f55e1603fc8cea147ae14c655e88a0aead383eb9..d5849ed4cd3e597c0a62c73a1e277d7dc7f77824 100644 (file)
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -3728,8 +3728,10 @@ static void process_ddsp_deferred_locals(struct rq *rq)
        }
 }
 
-static bool task_should_reenq(struct task_struct *p, u64 reenq_flags)
+static bool task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason)
 {
+       *reason = SCX_TASK_REENQ_KFUNC;
+
        if (reenq_flags & SCX_REENQ_ANY)
                return true;
        return false;
@@ -3751,6 +3753,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
        list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
                                 scx.dsq_list.node) {
                struct scx_sched *task_sch = scx_task_sched(p);
+               u32 reason;
 
                /*
                 * If @p is being migrated, @p's current CPU may not agree with
@@ -3769,16 +3772,24 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
                if (!scx_is_descendant(task_sch, sch))
                        continue;
 
-               if (!task_should_reenq(p, reenq_flags))
+               if (!task_should_reenq(p, reenq_flags, &reason))
                        continue;
 
                dispatch_dequeue(rq, p);
+
+               if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
+                       p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+               p->scx.flags |= reason;
+
                list_add_tail(&p->scx.dsq_list.node, &tasks);
        }
 
        list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
                list_del_init(&p->scx.dsq_list.node);
+
                do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+
+               p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
                nr_enqueued++;
        }
 
@@ -3832,12 +3843,13 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
 
        while (likely(!READ_ONCE(sch->bypass_depth))) {
                struct rq *task_rq;
+               u32 reason;
 
                p = nldsq_cursor_next_task(&cursor, dsq);
                if (!p)
                        break;
 
-               if (!task_should_reenq(p, reenq_flags))
+               if (!task_should_reenq(p, reenq_flags, &reason))
                        continue;
 
                task_rq = task_rq(p);
@@ -3860,8 +3872,15 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
                /* @p is on @dsq, its rq and @dsq are locked */
                dispatch_dequeue_locked(p, dsq);
                raw_spin_unlock(&dsq->lock);
+
+               if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
+                       p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+               p->scx.flags |= reason;
+
                do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
 
+               p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+
                if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) {
                        raw_spin_rq_unlock(locked_rq);
                        locked_rq = NULL;

diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h
index d9eda2e8701c2b717e10a787790d8dd16527b038..f8df730445156eb8e3d5dedb4e685e086efea6c4 100644 (file)
--- a/kernel/sched/ext_internal.h
+++ b/kernel/sched/ext_internal.h
@@ -1080,13 +1080,9 @@ enum scx_enq_flags {
        SCX_ENQ_PREEMPT         = 1LLU << 32,
 
        /*
-        * The task being enqueued was previously enqueued on the current CPU's
-        * %SCX_DSQ_LOCAL, but was removed from it in a call to the
-        * scx_bpf_reenqueue_local() kfunc. If scx_bpf_reenqueue_local() was
-        * invoked in a ->cpu_release() callback, and the task is again
-        * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the
-        * task will not be scheduled on the CPU until at least the next invocation
-        * of the ->cpu_acquire() callback.
+        * The task being enqueued was previously enqueued on a DSQ, but was
+        * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find
+        * out why a given task is being reenqueued.
         */
        SCX_ENQ_REENQ           = 1LLU << 40,