sched_ext: Separate bypass dispatch enabling from bypass depth tracking

The bypass_depth field tracks nesting of bypass operations but is also used
to determine whether the bypass dispatch path should be active. With
hierarchical scheduling, child schedulers may need to activate their parent's
bypass dispatch path without affecting the parent's bypass_depth, requiring
separation of these concerns.

Add bypass_dsp_enable_depth and bypass_dsp_claim to independently control
bypass dispatch path activation. The new enable_bypass_dsp() and
disable_bypass_dsp() functions manage this state with proper claim semantics
to prevent races. The bypass dispatch path now only activates when
bypass_dsp_enabled() returns true, which checks the new enable_depth counter.

The disable operation is carefully ordered after all tasks are moved out of
bypass DSQs to ensure they are drained before the dispatch path is disabled.
During scheduler teardown, disable_bypass_dsp() is called explicitly to ensure
cleanup even if bypass mode was never entered normally.

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>

diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index 413d0558372f04f3a22bf0515cf2749fa276035d..ea5179df3de484277db89fa08d28484e3146d0eb 100644 (file)
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -357,6 +357,26 @@ static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu)
        return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq;
 }
 
+/**
+ * bypass_dsp_enabled - Check if bypass dispatch path is enabled
+ * @sch: scheduler to check
+ *
+ * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled
+ * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors
+ * are bypassing. In the former case, the ancestor is not itself bypassing but
+ * its bypass DSQs will be populated with bypassed tasks from descendants. Thus,
+ * the ancestor's bypass dispatch path must be active even though its own
+ * bypass_depth remains zero.
+ *
+ * This function checks bypass_dsp_enable_depth which is managed separately from
+ * bypass_depth to enable this decoupling. See enable_bypass_dsp() and
+ * disable_bypass_dsp().
+ */
+static bool bypass_dsp_enabled(struct scx_sched *sch)
+{
+       return unlikely(atomic_read(&sch->bypass_dsp_enable_depth));
+}
+
 /*
  * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX
  * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate
@@ -2400,7 +2420,7 @@ static bool scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
        if (consume_global_dsq(sch, rq))
                return true;
 
-       if (scx_bypassing(sch, cpu))
+       if (bypass_dsp_enabled(sch) && scx_bypassing(sch, cpu))
                return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu));
 
        if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
@@ -4397,7 +4417,7 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer)
        int node;
        u32 intv_us;
 
-       if (!READ_ONCE(sch->bypass_depth))
+       if (!bypass_dsp_enabled(sch))
                return;
 
        for_each_node_with_cpus(node)
@@ -4438,6 +4458,42 @@ static bool dec_bypass_depth(struct scx_sched *sch)
        return true;
 }
 
+static void enable_bypass_dsp(struct scx_sched *sch)
+{
+       u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us);
+       s32 ret;
+
+       /*
+        * @sch->bypass_depth transitioning from 0 to 1 triggers enabling.
+        * Shouldn't stagger.
+        */
+       if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim)))
+               return;
+
+       /*
+        * The LB timer will stop running if bypass_arm_depth is 0. Increment
+        * before starting the LB timer.
+        */
+       ret = atomic_inc_return(&sch->bypass_dsp_enable_depth);
+       WARN_ON_ONCE(ret <= 0);
+
+       if (intv_us && !timer_pending(&sch->bypass_lb_timer))
+               mod_timer(&sch->bypass_lb_timer,
+                         jiffies + usecs_to_jiffies(intv_us));
+}
+
+/* may be called without holding scx_bypass_lock */
+static void disable_bypass_dsp(struct scx_sched *sch)
+{
+       s32 ret;
+
+       if (!test_and_clear_bit(0, &sch->bypass_dsp_claim))
+               return;
+
+       ret = atomic_dec_return(&sch->bypass_dsp_enable_depth);
+       WARN_ON_ONCE(ret < 0);
+}
+
 /**
  * scx_bypass - [Un]bypass scx_ops and guarantee forward progress
  * @sch: sched to bypass
@@ -4479,17 +4535,10 @@ static void scx_bypass(struct scx_sched *sch, bool bypass)
        raw_spin_lock_irqsave(&scx_bypass_lock, flags);
 
        if (bypass) {
-               u32 intv_us;
-
                if (!inc_bypass_depth(sch))
                        goto unlock;
 
-               intv_us = READ_ONCE(scx_bypass_lb_intv_us);
-               if (intv_us && !timer_pending(&sch->bypass_lb_timer)) {
-                       sch->bypass_lb_timer.expires =
-                               jiffies + usecs_to_jiffies(intv_us);
-                       add_timer_global(&sch->bypass_lb_timer);
-               }
+               enable_bypass_dsp(sch);
        } else {
                if (!dec_bypass_depth(sch))
                        goto unlock;
@@ -4571,6 +4620,9 @@ static void scx_bypass(struct scx_sched *sch, bool bypass)
                raw_spin_rq_unlock(rq);
        }
 
+       /* disarming must come after moving all tasks out of the bypass DSQs */
+       if (!bypass)
+               disable_bypass_dsp(sch);
 unlock:
        raw_spin_unlock_irqrestore(&scx_bypass_lock, flags);
 }
@@ -4672,6 +4724,8 @@ static void scx_sub_disable(struct scx_sched *sch)
        scx_cgroup_unlock();
        percpu_up_write(&scx_fork_rwsem);
 
+       disable_bypass_dsp(sch);
+
        raw_spin_lock_irq(&scx_sched_lock);
        list_del_init(&sch->sibling);
        list_del_rcu(&sch->all);

diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h
index 0e080611971110afd21448e49980803917be6989..575c26f9a3d7a53cb0d52d9452c75d1b1ed82878 100644 (file)
--- a/kernel/sched/ext_internal.h
+++ b/kernel/sched/ext_internal.h
@@ -961,6 +961,11 @@ struct scx_sched {
        u64                     slice_dfl;
        u64                     bypass_timestamp;
        s32                     bypass_depth;
+
+       /* bypass dispatch path enable state, see bypass_dsp_enabled() */
+       unsigned long           bypass_dsp_claim;
+       atomic_t                bypass_dsp_enable_depth;
+
        bool                    aborting;
        s32                     level;