static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
struct task_struct *next)
{
- struct scx_sched *sch = scx_root;
+ struct scx_sched *sch = scx_task_sched(p);
- /* see kick_cpus_irq_workfn() */
+ /* see kick_sync_wait_bal_cb() */
smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
update_curr_scx(rq);
BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n));
BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n));
BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n));
+ BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n));
+ raw_spin_lock_init(&rq->scx.deferred_reenq_lock);
+ INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals);
+ INIT_LIST_HEAD(&rq->scx.deferred_reenq_users);
rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn);
rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn);
cpumask_var_t cpus_to_kick_if_idle;
cpumask_var_t cpus_to_preempt;
cpumask_var_t cpus_to_wait;
+ cpumask_var_t cpus_to_sync;
+ bool kick_sync_pending;
unsigned long kick_sync;
- local_t reenq_local_deferred;
+
+ struct task_struct *sub_dispatch_prev;
+
+ raw_spinlock_t deferred_reenq_lock;
+ u64 deferred_reenq_locals_seq;
+ struct list_head deferred_reenq_locals; /* scheds requesting reenq of local DSQ */
+ struct list_head deferred_reenq_users; /* user DSQs requesting reenq */
struct balance_callback deferred_bal_cb;
+ struct balance_callback kick_sync_bal_cb;
struct irq_work deferred_irq_work;
struct irq_work kick_cpus_irq_work;
- struct scx_dispatch_q bypass_dsq;
};
#endif /* CONFIG_SCHED_CLASS_EXT */