rcu/nocb: Avoid synchronous wakeup in __call_rcu_nocb_wake()

When callbacks are in full flow, the common case is waiting for a
grace period, and this grace period will normally take a few jiffies to
complete.  It therefore isn't all that helpful for __call_rcu_nocb_wake()
to do a synchronous wakeup in this case.  This commit therefore turns this
into a timer-based deferred wakeup of the no-CBs grace-period kthread.

Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 25a53742ca68e076f7678d5964dc6eaa8a67a039..4b59ef1cbc8bfe8ddc957957cb67ea8802d115a2 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1900,22 +1900,13 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
        } else if (len > rdp->qlen_last_fqs_check + qhimark) {
                /* ... or if many callbacks queued. */
                rdp->qlen_last_fqs_check = len;
-               if (!rdp->nocb_cb_sleep &&
-                   rcu_segcblist_ready_cbs(&rdp->cblist)) {
-                       // Already going full tilt, so don't try to rewake.
-                       rcu_nocb_unlock_irqrestore(rdp, flags);
-               } else {
+               if (rdp->nocb_cb_sleep ||
+                   !rcu_segcblist_ready_cbs(&rdp->cblist)) {
                        rcu_advance_cbs_nowake(rdp->mynode, rdp);
-                       if (!irqs_disabled_flags(flags)) {
-                               wake_nocb_gp(rdp, false, flags);
-                               trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
-                                                   TPS("WakeOvf"));
-                       } else {
-                               wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
-                                                  TPS("WakeOvfIsDeferred"));
-                               rcu_nocb_unlock_irqrestore(rdp, flags);
-                       }
+                       wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
+                                          TPS("WakeOvfIsDeferred"));
                }
+               rcu_nocb_unlock_irqrestore(rdp, flags);
        } else {
                trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));
                rcu_nocb_unlock_irqrestore(rdp, flags);