1 // SPDX-License-Identifier: GPL-2.0+
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
5 * Copyright IBM Corporation, 2008
7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfullife.com>
9 * Paul E. McKenney <paulmck@linux.ibm.com>
11 * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
12 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
14 * For detailed explanation of Read-Copy Update mechanism see -
18 #define pr_fmt(fmt) "rcu: " fmt
20 #include <linux/types.h>
21 #include <linux/kernel.h>
22 #include <linux/init.h>
23 #include <linux/spinlock.h>
24 #include <linux/smp.h>
25 #include <linux/rcupdate_wait.h>
26 #include <linux/interrupt.h>
27 #include <linux/sched.h>
28 #include <linux/sched/debug.h>
29 #include <linux/nmi.h>
30 #include <linux/atomic.h>
31 #include <linux/bitops.h>
32 #include <linux/export.h>
33 #include <linux/completion.h>
34 #include <linux/moduleparam.h>
35 #include <linux/percpu.h>
36 #include <linux/notifier.h>
37 #include <linux/cpu.h>
38 #include <linux/mutex.h>
39 #include <linux/time.h>
40 #include <linux/kernel_stat.h>
41 #include <linux/wait.h>
42 #include <linux/kthread.h>
43 #include <uapi/linux/sched/types.h>
44 #include <linux/prefetch.h>
45 #include <linux/delay.h>
46 #include <linux/random.h>
47 #include <linux/trace_events.h>
48 #include <linux/suspend.h>
49 #include <linux/ftrace.h>
50 #include <linux/tick.h>
51 #include <linux/sysrq.h>
52 #include <linux/kprobes.h>
53 #include <linux/gfp.h>
54 #include <linux/oom.h>
55 #include <linux/smpboot.h>
56 #include <linux/jiffies.h>
57 #include <linux/slab.h>
58 #include <linux/sched/isolation.h>
59 #include <linux/sched/clock.h>
60 #include "../time/tick-internal.h"
65 #ifdef MODULE_PARAM_PREFIX
66 #undef MODULE_PARAM_PREFIX
68 #define MODULE_PARAM_PREFIX "rcutree."
70 /* Data structures. */
73 * Steal a bit from the bottom of ->dynticks for idle entry/exit
74 * control. Initially this is for TLB flushing.
76 #define RCU_DYNTICK_CTRL_MASK 0x1
77 #define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
78 #ifndef rcu_eqs_special_exit
79 #define rcu_eqs_special_exit() do { } while (0)
82 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data
, rcu_data
) = {
83 .dynticks_nesting
= 1,
84 .dynticks_nmi_nesting
= DYNTICK_IRQ_NONIDLE
,
85 .dynticks
= ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR
),
87 static struct rcu_state rcu_state
= {
88 .level
= { &rcu_state
.node
[0] },
89 .gp_state
= RCU_GP_IDLE
,
90 .gp_seq
= (0UL - 300UL) << RCU_SEQ_CTR_SHIFT
,
91 .barrier_mutex
= __MUTEX_INITIALIZER(rcu_state
.barrier_mutex
),
94 .exp_mutex
= __MUTEX_INITIALIZER(rcu_state
.exp_mutex
),
95 .exp_wake_mutex
= __MUTEX_INITIALIZER(rcu_state
.exp_wake_mutex
),
96 .ofl_lock
= __RAW_SPIN_LOCK_UNLOCKED(rcu_state
.ofl_lock
),
99 /* Dump rcu_node combining tree at boot to verify correct setup. */
100 static bool dump_tree
;
101 module_param(dump_tree
, bool, 0444);
102 /* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */
103 static bool use_softirq
= 1;
104 module_param(use_softirq
, bool, 0444);
105 /* Control rcu_node-tree auto-balancing at boot time. */
106 static bool rcu_fanout_exact
;
107 module_param(rcu_fanout_exact
, bool, 0444);
108 /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */
109 static int rcu_fanout_leaf
= RCU_FANOUT_LEAF
;
110 module_param(rcu_fanout_leaf
, int, 0444);
111 int rcu_num_lvls __read_mostly
= RCU_NUM_LVLS
;
112 /* Number of rcu_nodes at specified level. */
113 int num_rcu_lvl
[] = NUM_RCU_LVL_INIT
;
114 int rcu_num_nodes __read_mostly
= NUM_RCU_NODES
; /* Total # rcu_nodes in use. */
117 * The rcu_scheduler_active variable is initialized to the value
118 * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the
119 * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE,
120 * RCU can assume that there is but one task, allowing RCU to (for example)
121 * optimize synchronize_rcu() to a simple barrier(). When this variable
122 * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required
123 * to detect real grace periods. This variable is also used to suppress
124 * boot-time false positives from lockdep-RCU error checking. Finally, it
125 * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU
126 * is fully initialized, including all of its kthreads having been spawned.
128 int rcu_scheduler_active __read_mostly
;
129 EXPORT_SYMBOL_GPL(rcu_scheduler_active
);
132 * The rcu_scheduler_fully_active variable transitions from zero to one
133 * during the early_initcall() processing, which is after the scheduler
134 * is capable of creating new tasks. So RCU processing (for example,
135 * creating tasks for RCU priority boosting) must be delayed until after
136 * rcu_scheduler_fully_active transitions from zero to one. We also
137 * currently delay invocation of any RCU callbacks until after this point.
139 * It might later prove better for people registering RCU callbacks during
140 * early boot to take responsibility for these callbacks, but one step at
143 static int rcu_scheduler_fully_active __read_mostly
;
145 static void rcu_report_qs_rnp(unsigned long mask
, struct rcu_node
*rnp
,
146 unsigned long gps
, unsigned long flags
);
147 static void rcu_init_new_rnp(struct rcu_node
*rnp_leaf
);
148 static void rcu_cleanup_dead_rnp(struct rcu_node
*rnp_leaf
);
149 static void rcu_boost_kthread_setaffinity(struct rcu_node
*rnp
, int outgoingcpu
);
150 static void invoke_rcu_core(void);
151 static void rcu_report_exp_rdp(struct rcu_data
*rdp
);
152 static void sync_sched_exp_online_cleanup(int cpu
);
153 static void check_cb_ovld_locked(struct rcu_data
*rdp
, struct rcu_node
*rnp
);
155 /* rcuc/rcub kthread realtime priority */
156 static int kthread_prio
= IS_ENABLED(CONFIG_RCU_BOOST
) ? 1 : 0;
157 module_param(kthread_prio
, int, 0444);
159 /* Delay in jiffies for grace-period initialization delays, debug only. */
161 static int gp_preinit_delay
;
162 module_param(gp_preinit_delay
, int, 0444);
163 static int gp_init_delay
;
164 module_param(gp_init_delay
, int, 0444);
165 static int gp_cleanup_delay
;
166 module_param(gp_cleanup_delay
, int, 0444);
168 /* Retrieve RCU kthreads priority for rcutorture */
169 int rcu_get_gp_kthreads_prio(void)
173 EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio
);
176 * Number of grace periods between delays, normalized by the duration of
177 * the delay. The longer the delay, the more the grace periods between
178 * each delay. The reason for this normalization is that it means that,
179 * for non-zero delays, the overall slowdown of grace periods is constant
180 * regardless of the duration of the delay. This arrangement balances
181 * the need for long delays to increase some race probabilities with the
182 * need for fast grace periods to increase other race probabilities.
184 #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */
187 * Compute the mask of online CPUs for the specified rcu_node structure.
188 * This will not be stable unless the rcu_node structure's ->lock is
189 * held, but the bit corresponding to the current CPU will be stable
192 static unsigned long rcu_rnp_online_cpus(struct rcu_node
*rnp
)
194 return READ_ONCE(rnp
->qsmaskinitnext
);
198 * Return true if an RCU grace period is in progress. The READ_ONCE()s
199 * permit this function to be invoked without holding the root rcu_node
200 * structure's ->lock, but of course results can be subject to change.
202 static int rcu_gp_in_progress(void)
204 return rcu_seq_state(rcu_seq_current(&rcu_state
.gp_seq
));
208 * Return the number of callbacks queued on the specified CPU.
209 * Handles both the nocbs and normal cases.
211 static long rcu_get_n_cbs_cpu(int cpu
)
213 struct rcu_data
*rdp
= per_cpu_ptr(&rcu_data
, cpu
);
215 if (rcu_segcblist_is_enabled(&rdp
->cblist
))
216 return rcu_segcblist_n_cbs(&rdp
->cblist
);
220 void rcu_softirq_qs(void)
223 rcu_preempt_deferred_qs(current
);
227 * Record entry into an extended quiescent state. This is only to be
228 * called when not already in an extended quiescent state.
230 static void rcu_dynticks_eqs_enter(void)
232 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
236 * CPUs seeing atomic_add_return() must see prior RCU read-side
237 * critical sections, and we also must force ordering with the
240 seq
= atomic_add_return(RCU_DYNTICK_CTRL_CTR
, &rdp
->dynticks
);
241 /* Better be in an extended quiescent state! */
242 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
243 (seq
& RCU_DYNTICK_CTRL_CTR
));
244 /* Better not have special action (TLB flush) pending! */
245 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
246 (seq
& RCU_DYNTICK_CTRL_MASK
));
250 * Record exit from an extended quiescent state. This is only to be
251 * called from an extended quiescent state.
253 static void rcu_dynticks_eqs_exit(void)
255 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
259 * CPUs seeing atomic_add_return() must see prior idle sojourns,
260 * and we also must force ordering with the next RCU read-side
263 seq
= atomic_add_return(RCU_DYNTICK_CTRL_CTR
, &rdp
->dynticks
);
264 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
265 !(seq
& RCU_DYNTICK_CTRL_CTR
));
266 if (seq
& RCU_DYNTICK_CTRL_MASK
) {
267 atomic_andnot(RCU_DYNTICK_CTRL_MASK
, &rdp
->dynticks
);
268 smp_mb__after_atomic(); /* _exit after clearing mask. */
269 /* Prefer duplicate flushes to losing a flush. */
270 rcu_eqs_special_exit();
275 * Reset the current CPU's ->dynticks counter to indicate that the
276 * newly onlined CPU is no longer in an extended quiescent state.
277 * This will either leave the counter unchanged, or increment it
278 * to the next non-quiescent value.
280 * The non-atomic test/increment sequence works because the upper bits
281 * of the ->dynticks counter are manipulated only by the corresponding CPU,
282 * or when the corresponding CPU is offline.
284 static void rcu_dynticks_eqs_online(void)
286 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
288 if (atomic_read(&rdp
->dynticks
) & RCU_DYNTICK_CTRL_CTR
)
290 atomic_add(RCU_DYNTICK_CTRL_CTR
, &rdp
->dynticks
);
294 * Is the current CPU in an extended quiescent state?
296 * No ordering, as we are sampling CPU-local information.
298 static bool rcu_dynticks_curr_cpu_in_eqs(void)
300 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
302 return !(atomic_read(&rdp
->dynticks
) & RCU_DYNTICK_CTRL_CTR
);
306 * Snapshot the ->dynticks counter with full ordering so as to allow
307 * stable comparison of this counter with past and future snapshots.
309 static int rcu_dynticks_snap(struct rcu_data
*rdp
)
311 int snap
= atomic_add_return(0, &rdp
->dynticks
);
313 return snap
& ~RCU_DYNTICK_CTRL_MASK
;
317 * Return true if the snapshot returned from rcu_dynticks_snap()
318 * indicates that RCU is in an extended quiescent state.
320 static bool rcu_dynticks_in_eqs(int snap
)
322 return !(snap
& RCU_DYNTICK_CTRL_CTR
);
326 * Return true if the CPU corresponding to the specified rcu_data
327 * structure has spent some time in an extended quiescent state since
328 * rcu_dynticks_snap() returned the specified snapshot.
330 static bool rcu_dynticks_in_eqs_since(struct rcu_data
*rdp
, int snap
)
332 return snap
!= rcu_dynticks_snap(rdp
);
336 * Set the special (bottom) bit of the specified CPU so that it
337 * will take special action (such as flushing its TLB) on the
338 * next exit from an extended quiescent state. Returns true if
339 * the bit was successfully set, or false if the CPU was not in
340 * an extended quiescent state.
342 bool rcu_eqs_special_set(int cpu
)
347 struct rcu_data
*rdp
= &per_cpu(rcu_data
, cpu
);
349 new_old
= atomic_read(&rdp
->dynticks
);
352 if (old
& RCU_DYNTICK_CTRL_CTR
)
354 new = old
| RCU_DYNTICK_CTRL_MASK
;
355 new_old
= atomic_cmpxchg(&rdp
->dynticks
, old
, new);
356 } while (new_old
!= old
);
361 * Let the RCU core know that this CPU has gone through the scheduler,
362 * which is a quiescent state. This is called when the need for a
363 * quiescent state is urgent, so we burn an atomic operation and full
364 * memory barriers to let the RCU core know about it, regardless of what
365 * this CPU might (or might not) do in the near future.
367 * We inform the RCU core by emulating a zero-duration dyntick-idle period.
369 * The caller must have disabled interrupts and must not be idle.
371 void rcu_momentary_dyntick_idle(void)
375 raw_cpu_write(rcu_data
.rcu_need_heavy_qs
, false);
376 special
= atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR
,
377 &this_cpu_ptr(&rcu_data
)->dynticks
);
378 /* It is illegal to call this from idle state. */
379 WARN_ON_ONCE(!(special
& RCU_DYNTICK_CTRL_CTR
));
380 rcu_preempt_deferred_qs(current
);
382 EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle
);
385 * rcu_is_cpu_rrupt_from_idle - see if interrupted from idle
387 * If the current CPU is idle and running at a first-level (not nested)
388 * interrupt from idle, return true. The caller must have at least
389 * disabled preemption.
391 static int rcu_is_cpu_rrupt_from_idle(void)
393 /* Called only from within the scheduling-clock interrupt */
394 lockdep_assert_in_irq();
396 /* Check for counter underflows */
397 RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data
.dynticks_nesting
) < 0,
398 "RCU dynticks_nesting counter underflow!");
399 RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data
.dynticks_nmi_nesting
) <= 0,
400 "RCU dynticks_nmi_nesting counter underflow/zero!");
402 /* Are we at first interrupt nesting level? */
403 if (__this_cpu_read(rcu_data
.dynticks_nmi_nesting
) != 1)
406 /* Does CPU appear to be idle from an RCU standpoint? */
407 return __this_cpu_read(rcu_data
.dynticks_nesting
) == 0;
410 #define DEFAULT_RCU_BLIMIT 10 /* Maximum callbacks per rcu_do_batch ... */
411 #define DEFAULT_MAX_RCU_BLIMIT 10000 /* ... even during callback flood. */
412 static long blimit
= DEFAULT_RCU_BLIMIT
;
413 #define DEFAULT_RCU_QHIMARK 10000 /* If this many pending, ignore blimit. */
414 static long qhimark
= DEFAULT_RCU_QHIMARK
;
415 #define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */
416 static long qlowmark
= DEFAULT_RCU_QLOMARK
;
417 #define DEFAULT_RCU_QOVLD_MULT 2
418 #define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
419 static long qovld
= DEFAULT_RCU_QOVLD
; /* If this many pending, hammer QS. */
420 static long qovld_calc
= -1; /* No pre-initialization lock acquisitions! */
422 module_param(blimit
, long, 0444);
423 module_param(qhimark
, long, 0444);
424 module_param(qlowmark
, long, 0444);
425 module_param(qovld
, long, 0444);
427 static ulong jiffies_till_first_fqs
= ULONG_MAX
;
428 static ulong jiffies_till_next_fqs
= ULONG_MAX
;
429 static bool rcu_kick_kthreads
;
430 static int rcu_divisor
= 7;
431 module_param(rcu_divisor
, int, 0644);
433 /* Force an exit from rcu_do_batch() after 3 milliseconds. */
434 static long rcu_resched_ns
= 3 * NSEC_PER_MSEC
;
435 module_param(rcu_resched_ns
, long, 0644);
438 * How long the grace period must be before we start recruiting
439 * quiescent-state help from rcu_note_context_switch().
441 static ulong jiffies_till_sched_qs
= ULONG_MAX
;
442 module_param(jiffies_till_sched_qs
, ulong
, 0444);
443 static ulong jiffies_to_sched_qs
; /* See adjust_jiffies_till_sched_qs(). */
444 module_param(jiffies_to_sched_qs
, ulong
, 0444); /* Display only! */
447 * Make sure that we give the grace-period kthread time to detect any
448 * idle CPUs before taking active measures to force quiescent states.
449 * However, don't go below 100 milliseconds, adjusted upwards for really
452 static void adjust_jiffies_till_sched_qs(void)
456 /* If jiffies_till_sched_qs was specified, respect the request. */
457 if (jiffies_till_sched_qs
!= ULONG_MAX
) {
458 WRITE_ONCE(jiffies_to_sched_qs
, jiffies_till_sched_qs
);
461 /* Otherwise, set to third fqs scan, but bound below on large system. */
462 j
= READ_ONCE(jiffies_till_first_fqs
) +
463 2 * READ_ONCE(jiffies_till_next_fqs
);
464 if (j
< HZ
/ 10 + nr_cpu_ids
/ RCU_JIFFIES_FQS_DIV
)
465 j
= HZ
/ 10 + nr_cpu_ids
/ RCU_JIFFIES_FQS_DIV
;
466 pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j
);
467 WRITE_ONCE(jiffies_to_sched_qs
, j
);
470 static int param_set_first_fqs_jiffies(const char *val
, const struct kernel_param
*kp
)
473 int ret
= kstrtoul(val
, 0, &j
);
476 WRITE_ONCE(*(ulong
*)kp
->arg
, (j
> HZ
) ? HZ
: j
);
477 adjust_jiffies_till_sched_qs();
482 static int param_set_next_fqs_jiffies(const char *val
, const struct kernel_param
*kp
)
485 int ret
= kstrtoul(val
, 0, &j
);
488 WRITE_ONCE(*(ulong
*)kp
->arg
, (j
> HZ
) ? HZ
: (j
?: 1));
489 adjust_jiffies_till_sched_qs();
494 static struct kernel_param_ops first_fqs_jiffies_ops
= {
495 .set
= param_set_first_fqs_jiffies
,
496 .get
= param_get_ulong
,
499 static struct kernel_param_ops next_fqs_jiffies_ops
= {
500 .set
= param_set_next_fqs_jiffies
,
501 .get
= param_get_ulong
,
504 module_param_cb(jiffies_till_first_fqs
, &first_fqs_jiffies_ops
, &jiffies_till_first_fqs
, 0644);
505 module_param_cb(jiffies_till_next_fqs
, &next_fqs_jiffies_ops
, &jiffies_till_next_fqs
, 0644);
506 module_param(rcu_kick_kthreads
, bool, 0644);
508 static void force_qs_rnp(int (*f
)(struct rcu_data
*rdp
));
509 static int rcu_pending(int user
);
512 * Return the number of RCU GPs completed thus far for debug & stats.
514 unsigned long rcu_get_gp_seq(void)
516 return READ_ONCE(rcu_state
.gp_seq
);
518 EXPORT_SYMBOL_GPL(rcu_get_gp_seq
);
521 * Return the number of RCU expedited batches completed thus far for
522 * debug & stats. Odd numbers mean that a batch is in progress, even
523 * numbers mean idle. The value returned will thus be roughly double
524 * the cumulative batches since boot.
526 unsigned long rcu_exp_batches_completed(void)
528 return rcu_state
.expedited_sequence
;
530 EXPORT_SYMBOL_GPL(rcu_exp_batches_completed
);
533 * Return the root node of the rcu_state structure.
535 static struct rcu_node
*rcu_get_root(void)
537 return &rcu_state
.node
[0];
541 * Send along grace-period-related data for rcutorture diagnostics.
543 void rcutorture_get_gp_data(enum rcutorture_type test_type
, int *flags
,
544 unsigned long *gp_seq
)
548 *flags
= READ_ONCE(rcu_state
.gp_flags
);
549 *gp_seq
= rcu_seq_current(&rcu_state
.gp_seq
);
555 EXPORT_SYMBOL_GPL(rcutorture_get_gp_data
);
558 * Enter an RCU extended quiescent state, which can be either the
559 * idle loop or adaptive-tickless usermode execution.
561 * We crowbar the ->dynticks_nmi_nesting field to zero to allow for
562 * the possibility of usermode upcalls having messed up our count
563 * of interrupt nesting level during the prior busy period.
565 static void rcu_eqs_enter(bool user
)
567 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
569 WARN_ON_ONCE(rdp
->dynticks_nmi_nesting
!= DYNTICK_IRQ_NONIDLE
);
570 WRITE_ONCE(rdp
->dynticks_nmi_nesting
, 0);
571 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
572 rdp
->dynticks_nesting
== 0);
573 if (rdp
->dynticks_nesting
!= 1) {
574 rdp
->dynticks_nesting
--;
578 lockdep_assert_irqs_disabled();
579 trace_rcu_dyntick(TPS("Start"), rdp
->dynticks_nesting
, 0, atomic_read(&rdp
->dynticks
));
580 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) && !user
&& !is_idle_task(current
));
581 rdp
= this_cpu_ptr(&rcu_data
);
582 do_nocb_deferred_wakeup(rdp
);
583 rcu_prepare_for_idle();
584 rcu_preempt_deferred_qs(current
);
585 WRITE_ONCE(rdp
->dynticks_nesting
, 0); /* Avoid irq-access tearing. */
586 rcu_dynticks_eqs_enter();
587 rcu_dynticks_task_enter();
591 * rcu_idle_enter - inform RCU that current CPU is entering idle
593 * Enter idle mode, in other words, -leave- the mode in which RCU
594 * read-side critical sections can occur. (Though RCU read-side
595 * critical sections can occur in irq handlers in idle, a possibility
596 * handled by irq_enter() and irq_exit().)
598 * If you add or remove a call to rcu_idle_enter(), be sure to test with
599 * CONFIG_RCU_EQS_DEBUG=y.
601 void rcu_idle_enter(void)
603 lockdep_assert_irqs_disabled();
604 rcu_eqs_enter(false);
607 #ifdef CONFIG_NO_HZ_FULL
609 * rcu_user_enter - inform RCU that we are resuming userspace.
611 * Enter RCU idle mode right before resuming userspace. No use of RCU
612 * is permitted between this call and rcu_user_exit(). This way the
613 * CPU doesn't need to maintain the tick for RCU maintenance purposes
614 * when the CPU runs in userspace.
616 * If you add or remove a call to rcu_user_enter(), be sure to test with
617 * CONFIG_RCU_EQS_DEBUG=y.
619 void rcu_user_enter(void)
621 lockdep_assert_irqs_disabled();
624 #endif /* CONFIG_NO_HZ_FULL */
627 * If we are returning from the outermost NMI handler that interrupted an
628 * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting
629 * to let the RCU grace-period handling know that the CPU is back to
632 * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
633 * with CONFIG_RCU_EQS_DEBUG=y.
635 static __always_inline
void rcu_nmi_exit_common(bool irq
)
637 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
640 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
641 * (We are exiting an NMI handler, so RCU better be paying attention
644 WARN_ON_ONCE(rdp
->dynticks_nmi_nesting
<= 0);
645 WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
648 * If the nesting level is not 1, the CPU wasn't RCU-idle, so
649 * leave it in non-RCU-idle state.
651 if (rdp
->dynticks_nmi_nesting
!= 1) {
652 trace_rcu_dyntick(TPS("--="), rdp
->dynticks_nmi_nesting
, rdp
->dynticks_nmi_nesting
- 2,
653 atomic_read(&rdp
->dynticks
));
654 WRITE_ONCE(rdp
->dynticks_nmi_nesting
, /* No store tearing. */
655 rdp
->dynticks_nmi_nesting
- 2);
659 /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
660 trace_rcu_dyntick(TPS("Startirq"), rdp
->dynticks_nmi_nesting
, 0, atomic_read(&rdp
->dynticks
));
661 WRITE_ONCE(rdp
->dynticks_nmi_nesting
, 0); /* Avoid store tearing. */
664 rcu_prepare_for_idle();
666 rcu_dynticks_eqs_enter();
669 rcu_dynticks_task_enter();
673 * rcu_nmi_exit - inform RCU of exit from NMI context
675 * If you add or remove a call to rcu_nmi_exit(), be sure to test
676 * with CONFIG_RCU_EQS_DEBUG=y.
678 void rcu_nmi_exit(void)
680 rcu_nmi_exit_common(false);
684 * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
686 * Exit from an interrupt handler, which might possibly result in entering
687 * idle mode, in other words, leaving the mode in which read-side critical
688 * sections can occur. The caller must have disabled interrupts.
690 * This code assumes that the idle loop never does anything that might
691 * result in unbalanced calls to irq_enter() and irq_exit(). If your
692 * architecture's idle loop violates this assumption, RCU will give you what
693 * you deserve, good and hard. But very infrequently and irreproducibly.
695 * Use things like work queues to work around this limitation.
697 * You have been warned.
699 * If you add or remove a call to rcu_irq_exit(), be sure to test with
700 * CONFIG_RCU_EQS_DEBUG=y.
702 void rcu_irq_exit(void)
704 lockdep_assert_irqs_disabled();
705 rcu_nmi_exit_common(true);
709 * Wrapper for rcu_irq_exit() where interrupts are enabled.
711 * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test
712 * with CONFIG_RCU_EQS_DEBUG=y.
714 void rcu_irq_exit_irqson(void)
718 local_irq_save(flags
);
720 local_irq_restore(flags
);
724 * Exit an RCU extended quiescent state, which can be either the
725 * idle loop or adaptive-tickless usermode execution.
727 * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to
728 * allow for the possibility of usermode upcalls messing up our count of
729 * interrupt nesting level during the busy period that is just now starting.
731 static void rcu_eqs_exit(bool user
)
733 struct rcu_data
*rdp
;
736 lockdep_assert_irqs_disabled();
737 rdp
= this_cpu_ptr(&rcu_data
);
738 oldval
= rdp
->dynticks_nesting
;
739 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) && oldval
< 0);
741 rdp
->dynticks_nesting
++;
744 rcu_dynticks_task_exit();
745 rcu_dynticks_eqs_exit();
746 rcu_cleanup_after_idle();
747 trace_rcu_dyntick(TPS("End"), rdp
->dynticks_nesting
, 1, atomic_read(&rdp
->dynticks
));
748 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) && !user
&& !is_idle_task(current
));
749 WRITE_ONCE(rdp
->dynticks_nesting
, 1);
750 WARN_ON_ONCE(rdp
->dynticks_nmi_nesting
);
751 WRITE_ONCE(rdp
->dynticks_nmi_nesting
, DYNTICK_IRQ_NONIDLE
);
755 * rcu_idle_exit - inform RCU that current CPU is leaving idle
757 * Exit idle mode, in other words, -enter- the mode in which RCU
758 * read-side critical sections can occur.
760 * If you add or remove a call to rcu_idle_exit(), be sure to test with
761 * CONFIG_RCU_EQS_DEBUG=y.
763 void rcu_idle_exit(void)
767 local_irq_save(flags
);
769 local_irq_restore(flags
);
772 #ifdef CONFIG_NO_HZ_FULL
774 * rcu_user_exit - inform RCU that we are exiting userspace.
776 * Exit RCU idle mode while entering the kernel because it can
777 * run a RCU read side critical section anytime.
779 * If you add or remove a call to rcu_user_exit(), be sure to test with
780 * CONFIG_RCU_EQS_DEBUG=y.
782 void rcu_user_exit(void)
786 #endif /* CONFIG_NO_HZ_FULL */
789 * rcu_nmi_enter_common - inform RCU of entry to NMI context
790 * @irq: Is this call from rcu_irq_enter?
792 * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and
793 * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know
794 * that the CPU is active. This implementation permits nested NMIs, as
795 * long as the nesting level does not overflow an int. (You will probably
796 * run out of stack space first.)
798 * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
799 * with CONFIG_RCU_EQS_DEBUG=y.
801 static __always_inline
void rcu_nmi_enter_common(bool irq
)
804 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
806 /* Complain about underflow. */
807 WARN_ON_ONCE(rdp
->dynticks_nmi_nesting
< 0);
810 * If idle from RCU viewpoint, atomically increment ->dynticks
811 * to mark non-idle and increment ->dynticks_nmi_nesting by one.
812 * Otherwise, increment ->dynticks_nmi_nesting by two. This means
813 * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
814 * to be in the outermost NMI handler that interrupted an RCU-idle
815 * period (observation due to Andy Lutomirski).
817 if (rcu_dynticks_curr_cpu_in_eqs()) {
820 rcu_dynticks_task_exit();
822 rcu_dynticks_eqs_exit();
825 rcu_cleanup_after_idle();
828 } else if (irq
&& tick_nohz_full_cpu(rdp
->cpu
) &&
829 rdp
->dynticks_nmi_nesting
== DYNTICK_IRQ_NONIDLE
&&
830 READ_ONCE(rdp
->rcu_urgent_qs
) &&
831 !READ_ONCE(rdp
->rcu_forced_tick
)) {
832 raw_spin_lock_rcu_node(rdp
->mynode
);
833 // Recheck under lock.
834 if (rdp
->rcu_urgent_qs
&& !rdp
->rcu_forced_tick
) {
835 WRITE_ONCE(rdp
->rcu_forced_tick
, true);
836 tick_dep_set_cpu(rdp
->cpu
, TICK_DEP_BIT_RCU
);
838 raw_spin_unlock_rcu_node(rdp
->mynode
);
840 trace_rcu_dyntick(incby
== 1 ? TPS("Endirq") : TPS("++="),
841 rdp
->dynticks_nmi_nesting
,
842 rdp
->dynticks_nmi_nesting
+ incby
, atomic_read(&rdp
->dynticks
));
843 WRITE_ONCE(rdp
->dynticks_nmi_nesting
, /* Prevent store tearing. */
844 rdp
->dynticks_nmi_nesting
+ incby
);
849 * rcu_nmi_enter - inform RCU of entry to NMI context
851 void rcu_nmi_enter(void)
853 rcu_nmi_enter_common(false);
855 NOKPROBE_SYMBOL(rcu_nmi_enter
);
858 * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
860 * Enter an interrupt handler, which might possibly result in exiting
861 * idle mode, in other words, entering the mode in which read-side critical
862 * sections can occur. The caller must have disabled interrupts.
864 * Note that the Linux kernel is fully capable of entering an interrupt
865 * handler that it never exits, for example when doing upcalls to user mode!
866 * This code assumes that the idle loop never does upcalls to user mode.
867 * If your architecture's idle loop does do upcalls to user mode (or does
868 * anything else that results in unbalanced calls to the irq_enter() and
869 * irq_exit() functions), RCU will give you what you deserve, good and hard.
870 * But very infrequently and irreproducibly.
872 * Use things like work queues to work around this limitation.
874 * You have been warned.
876 * If you add or remove a call to rcu_irq_enter(), be sure to test with
877 * CONFIG_RCU_EQS_DEBUG=y.
879 void rcu_irq_enter(void)
881 lockdep_assert_irqs_disabled();
882 rcu_nmi_enter_common(true);
886 * Wrapper for rcu_irq_enter() where interrupts are enabled.
888 * If you add or remove a call to rcu_irq_enter_irqson(), be sure to test
889 * with CONFIG_RCU_EQS_DEBUG=y.
891 void rcu_irq_enter_irqson(void)
895 local_irq_save(flags
);
897 local_irq_restore(flags
);
901 * If any sort of urgency was applied to the current CPU (for example,
902 * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order
903 * to get to a quiescent state, disable it.
905 static void rcu_disable_urgency_upon_qs(struct rcu_data
*rdp
)
907 raw_lockdep_assert_held_rcu_node(rdp
->mynode
);
908 WRITE_ONCE(rdp
->rcu_urgent_qs
, false);
909 WRITE_ONCE(rdp
->rcu_need_heavy_qs
, false);
910 if (tick_nohz_full_cpu(rdp
->cpu
) && rdp
->rcu_forced_tick
) {
911 tick_dep_clear_cpu(rdp
->cpu
, TICK_DEP_BIT_RCU
);
912 WRITE_ONCE(rdp
->rcu_forced_tick
, false);
917 * rcu_is_watching - see if RCU thinks that the current CPU is not idle
919 * Return true if RCU is watching the running CPU, which means that this
920 * CPU can safely enter RCU read-side critical sections. In other words,
921 * if the current CPU is not in its idle loop or is in an interrupt or
922 * NMI handler, return true.
924 bool notrace
rcu_is_watching(void)
928 preempt_disable_notrace();
929 ret
= !rcu_dynticks_curr_cpu_in_eqs();
930 preempt_enable_notrace();
933 EXPORT_SYMBOL_GPL(rcu_is_watching
);
936 * If a holdout task is actually running, request an urgent quiescent
937 * state from its CPU. This is unsynchronized, so migrations can cause
938 * the request to go to the wrong CPU. Which is OK, all that will happen
939 * is that the CPU's next context switch will be a bit slower and next
940 * time around this task will generate another request.
942 void rcu_request_urgent_qs_task(struct task_struct
*t
)
949 return; /* This task is not running on that CPU. */
950 smp_store_release(per_cpu_ptr(&rcu_data
.rcu_urgent_qs
, cpu
), true);
953 #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
956 * Is the current CPU online as far as RCU is concerned?
958 * Disable preemption to avoid false positives that could otherwise
959 * happen due to the current CPU number being sampled, this task being
960 * preempted, its old CPU being taken offline, resuming on some other CPU,
961 * then determining that its old CPU is now offline.
963 * Disable checking if in an NMI handler because we cannot safely
964 * report errors from NMI handlers anyway. In addition, it is OK to use
965 * RCU on an offline processor during initial boot, hence the check for
966 * rcu_scheduler_fully_active.
968 bool rcu_lockdep_current_cpu_online(void)
970 struct rcu_data
*rdp
;
971 struct rcu_node
*rnp
;
974 if (in_nmi() || !rcu_scheduler_fully_active
)
977 rdp
= this_cpu_ptr(&rcu_data
);
979 if (rdp
->grpmask
& rcu_rnp_online_cpus(rnp
))
984 EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online
);
986 #endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
989 * We are reporting a quiescent state on behalf of some other CPU, so
990 * it is our responsibility to check for and handle potential overflow
991 * of the rcu_node ->gp_seq counter with respect to the rcu_data counters.
992 * After all, the CPU might be in deep idle state, and thus executing no
995 static void rcu_gpnum_ovf(struct rcu_node
*rnp
, struct rcu_data
*rdp
)
997 raw_lockdep_assert_held_rcu_node(rnp
);
998 if (ULONG_CMP_LT(rcu_seq_current(&rdp
->gp_seq
) + ULONG_MAX
/ 4,
1000 WRITE_ONCE(rdp
->gpwrap
, true);
1001 if (ULONG_CMP_LT(rdp
->rcu_iw_gp_seq
+ ULONG_MAX
/ 4, rnp
->gp_seq
))
1002 rdp
->rcu_iw_gp_seq
= rnp
->gp_seq
+ ULONG_MAX
/ 4;
1006 * Snapshot the specified CPU's dynticks counter so that we can later
1007 * credit them with an implicit quiescent state. Return 1 if this CPU
1008 * is in dynticks idle mode, which is an extended quiescent state.
1010 static int dyntick_save_progress_counter(struct rcu_data
*rdp
)
1012 rdp
->dynticks_snap
= rcu_dynticks_snap(rdp
);
1013 if (rcu_dynticks_in_eqs(rdp
->dynticks_snap
)) {
1014 trace_rcu_fqs(rcu_state
.name
, rdp
->gp_seq
, rdp
->cpu
, TPS("dti"));
1015 rcu_gpnum_ovf(rdp
->mynode
, rdp
);
1022 * Return true if the specified CPU has passed through a quiescent
1023 * state by virtue of being in or having passed through an dynticks
1024 * idle state since the last call to dyntick_save_progress_counter()
1025 * for this same CPU, or by virtue of having been offline.
1027 static int rcu_implicit_dynticks_qs(struct rcu_data
*rdp
)
1032 struct rcu_node
*rnp
= rdp
->mynode
;
1035 * If the CPU passed through or entered a dynticks idle phase with
1036 * no active irq/NMI handlers, then we can safely pretend that the CPU
1037 * already acknowledged the request to pass through a quiescent
1038 * state. Either way, that CPU cannot possibly be in an RCU
1039 * read-side critical section that started before the beginning
1040 * of the current RCU grace period.
1042 if (rcu_dynticks_in_eqs_since(rdp
, rdp
->dynticks_snap
)) {
1043 trace_rcu_fqs(rcu_state
.name
, rdp
->gp_seq
, rdp
->cpu
, TPS("dti"));
1044 rcu_gpnum_ovf(rnp
, rdp
);
1048 /* If waiting too long on an offline CPU, complain. */
1049 if (!(rdp
->grpmask
& rcu_rnp_online_cpus(rnp
)) &&
1050 time_after(jiffies
, rcu_state
.gp_start
+ HZ
)) {
1052 struct rcu_node
*rnp1
;
1054 WARN_ON(1); /* Offline CPUs are supposed to report QS! */
1055 pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
1056 __func__
, rnp
->grplo
, rnp
->grphi
, rnp
->level
,
1057 (long)rnp
->gp_seq
, (long)rnp
->completedqs
);
1058 for (rnp1
= rnp
; rnp1
; rnp1
= rnp1
->parent
)
1059 pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx ->rcu_gp_init_mask %#lx\n",
1060 __func__
, rnp1
->grplo
, rnp1
->grphi
, rnp1
->qsmask
, rnp1
->qsmaskinit
, rnp1
->qsmaskinitnext
, rnp1
->rcu_gp_init_mask
);
1061 onl
= !!(rdp
->grpmask
& rcu_rnp_online_cpus(rnp
));
1062 pr_info("%s %d: %c online: %ld(%d) offline: %ld(%d)\n",
1063 __func__
, rdp
->cpu
, ".o"[onl
],
1064 (long)rdp
->rcu_onl_gp_seq
, rdp
->rcu_onl_gp_flags
,
1065 (long)rdp
->rcu_ofl_gp_seq
, rdp
->rcu_ofl_gp_flags
);
1066 return 1; /* Break things loose after complaining. */
1070 * A CPU running for an extended time within the kernel can
1071 * delay RCU grace periods: (1) At age jiffies_to_sched_qs,
1072 * set .rcu_urgent_qs, (2) At age 2*jiffies_to_sched_qs, set
1073 * both .rcu_need_heavy_qs and .rcu_urgent_qs. Note that the
1074 * unsynchronized assignments to the per-CPU rcu_need_heavy_qs
1075 * variable are safe because the assignments are repeated if this
1076 * CPU failed to pass through a quiescent state. This code
1077 * also checks .jiffies_resched in case jiffies_to_sched_qs
1080 jtsq
= READ_ONCE(jiffies_to_sched_qs
);
1081 ruqp
= per_cpu_ptr(&rcu_data
.rcu_urgent_qs
, rdp
->cpu
);
1082 rnhqp
= &per_cpu(rcu_data
.rcu_need_heavy_qs
, rdp
->cpu
);
1083 if (!READ_ONCE(*rnhqp
) &&
1084 (time_after(jiffies
, rcu_state
.gp_start
+ jtsq
* 2) ||
1085 time_after(jiffies
, rcu_state
.jiffies_resched
) ||
1086 rcu_state
.cbovld
)) {
1087 WRITE_ONCE(*rnhqp
, true);
1088 /* Store rcu_need_heavy_qs before rcu_urgent_qs. */
1089 smp_store_release(ruqp
, true);
1090 } else if (time_after(jiffies
, rcu_state
.gp_start
+ jtsq
)) {
1091 WRITE_ONCE(*ruqp
, true);
1095 * NO_HZ_FULL CPUs can run in-kernel without rcu_sched_clock_irq!
1096 * The above code handles this, but only for straight cond_resched().
1097 * And some in-kernel loops check need_resched() before calling
1098 * cond_resched(), which defeats the above code for CPUs that are
1099 * running in-kernel with scheduling-clock interrupts disabled.
1100 * So hit them over the head with the resched_cpu() hammer!
1102 if (tick_nohz_full_cpu(rdp
->cpu
) &&
1103 (time_after(jiffies
, READ_ONCE(rdp
->last_fqs_resched
) + jtsq
* 3) ||
1104 rcu_state
.cbovld
)) {
1105 WRITE_ONCE(*ruqp
, true);
1106 resched_cpu(rdp
->cpu
);
1107 WRITE_ONCE(rdp
->last_fqs_resched
, jiffies
);
1111 * If more than halfway to RCU CPU stall-warning time, invoke
1112 * resched_cpu() more frequently to try to loosen things up a bit.
1113 * Also check to see if the CPU is getting hammered with interrupts,
1114 * but only once per grace period, just to keep the IPIs down to
1117 if (time_after(jiffies
, rcu_state
.jiffies_resched
)) {
1118 if (time_after(jiffies
,
1119 READ_ONCE(rdp
->last_fqs_resched
) + jtsq
)) {
1120 resched_cpu(rdp
->cpu
);
1121 WRITE_ONCE(rdp
->last_fqs_resched
, jiffies
);
1123 if (IS_ENABLED(CONFIG_IRQ_WORK
) &&
1124 !rdp
->rcu_iw_pending
&& rdp
->rcu_iw_gp_seq
!= rnp
->gp_seq
&&
1125 (rnp
->ffmask
& rdp
->grpmask
)) {
1126 init_irq_work(&rdp
->rcu_iw
, rcu_iw_handler
);
1127 atomic_set(&rdp
->rcu_iw
.flags
, IRQ_WORK_HARD_IRQ
);
1128 rdp
->rcu_iw_pending
= true;
1129 rdp
->rcu_iw_gp_seq
= rnp
->gp_seq
;
1130 irq_work_queue_on(&rdp
->rcu_iw
, rdp
->cpu
);
1137 /* Trace-event wrapper function for trace_rcu_future_grace_period. */
1138 static void trace_rcu_this_gp(struct rcu_node
*rnp
, struct rcu_data
*rdp
,
1139 unsigned long gp_seq_req
, const char *s
)
1141 trace_rcu_future_grace_period(rcu_state
.name
, READ_ONCE(rnp
->gp_seq
),
1142 gp_seq_req
, rnp
->level
,
1143 rnp
->grplo
, rnp
->grphi
, s
);
1147 * rcu_start_this_gp - Request the start of a particular grace period
1148 * @rnp_start: The leaf node of the CPU from which to start.
1149 * @rdp: The rcu_data corresponding to the CPU from which to start.
1150 * @gp_seq_req: The gp_seq of the grace period to start.
1152 * Start the specified grace period, as needed to handle newly arrived
1153 * callbacks. The required future grace periods are recorded in each
1154 * rcu_node structure's ->gp_seq_needed field. Returns true if there
1155 * is reason to awaken the grace-period kthread.
1157 * The caller must hold the specified rcu_node structure's ->lock, which
1158 * is why the caller is responsible for waking the grace-period kthread.
1160 * Returns true if the GP thread needs to be awakened else false.
1162 static bool rcu_start_this_gp(struct rcu_node
*rnp_start
, struct rcu_data
*rdp
,
1163 unsigned long gp_seq_req
)
1166 struct rcu_node
*rnp
;
1169 * Use funnel locking to either acquire the root rcu_node
1170 * structure's lock or bail out if the need for this grace period
1171 * has already been recorded -- or if that grace period has in
1172 * fact already started. If there is already a grace period in
1173 * progress in a non-leaf node, no recording is needed because the
1174 * end of the grace period will scan the leaf rcu_node structures.
1175 * Note that rnp_start->lock must not be released.
1177 raw_lockdep_assert_held_rcu_node(rnp_start
);
1178 trace_rcu_this_gp(rnp_start
, rdp
, gp_seq_req
, TPS("Startleaf"));
1179 for (rnp
= rnp_start
; 1; rnp
= rnp
->parent
) {
1180 if (rnp
!= rnp_start
)
1181 raw_spin_lock_rcu_node(rnp
);
1182 if (ULONG_CMP_GE(rnp
->gp_seq_needed
, gp_seq_req
) ||
1183 rcu_seq_started(&rnp
->gp_seq
, gp_seq_req
) ||
1184 (rnp
!= rnp_start
&&
1185 rcu_seq_state(rcu_seq_current(&rnp
->gp_seq
)))) {
1186 trace_rcu_this_gp(rnp
, rdp
, gp_seq_req
,
1190 WRITE_ONCE(rnp
->gp_seq_needed
, gp_seq_req
);
1191 if (rcu_seq_state(rcu_seq_current(&rnp
->gp_seq
))) {
1193 * We just marked the leaf or internal node, and a
1194 * grace period is in progress, which means that
1195 * rcu_gp_cleanup() will see the marking. Bail to
1196 * reduce contention.
1198 trace_rcu_this_gp(rnp_start
, rdp
, gp_seq_req
,
1199 TPS("Startedleaf"));
1202 if (rnp
!= rnp_start
&& rnp
->parent
!= NULL
)
1203 raw_spin_unlock_rcu_node(rnp
);
1205 break; /* At root, and perhaps also leaf. */
1208 /* If GP already in progress, just leave, otherwise start one. */
1209 if (rcu_gp_in_progress()) {
1210 trace_rcu_this_gp(rnp
, rdp
, gp_seq_req
, TPS("Startedleafroot"));
1213 trace_rcu_this_gp(rnp
, rdp
, gp_seq_req
, TPS("Startedroot"));
1214 WRITE_ONCE(rcu_state
.gp_flags
, rcu_state
.gp_flags
| RCU_GP_FLAG_INIT
);
1215 WRITE_ONCE(rcu_state
.gp_req_activity
, jiffies
);
1216 if (!READ_ONCE(rcu_state
.gp_kthread
)) {
1217 trace_rcu_this_gp(rnp
, rdp
, gp_seq_req
, TPS("NoGPkthread"));
1220 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
, TPS("newreq"));
1221 ret
= true; /* Caller must wake GP kthread. */
1223 /* Push furthest requested GP to leaf node and rcu_data structure. */
1224 if (ULONG_CMP_LT(gp_seq_req
, rnp
->gp_seq_needed
)) {
1225 WRITE_ONCE(rnp_start
->gp_seq_needed
, rnp
->gp_seq_needed
);
1226 WRITE_ONCE(rdp
->gp_seq_needed
, rnp
->gp_seq_needed
);
1228 if (rnp
!= rnp_start
)
1229 raw_spin_unlock_rcu_node(rnp
);
1234 * Clean up any old requests for the just-ended grace period. Also return
1235 * whether any additional grace periods have been requested.
1237 static bool rcu_future_gp_cleanup(struct rcu_node
*rnp
)
1240 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
1242 needmore
= ULONG_CMP_LT(rnp
->gp_seq
, rnp
->gp_seq_needed
);
1244 rnp
->gp_seq_needed
= rnp
->gp_seq
; /* Avoid counter wrap. */
1245 trace_rcu_this_gp(rnp
, rdp
, rnp
->gp_seq
,
1246 needmore
? TPS("CleanupMore") : TPS("Cleanup"));
1251 * Awaken the grace-period kthread. Don't do a self-awaken (unless in an
1252 * interrupt or softirq handler, in which case we just might immediately
1253 * sleep upon return, resulting in a grace-period hang), and don't bother
1254 * awakening when there is nothing for the grace-period kthread to do
1255 * (as in several CPUs raced to awaken, we lost), and finally don't try
1256 * to awaken a kthread that has not yet been created. If all those checks
1257 * are passed, track some debug information and awaken.
1259 * So why do the self-wakeup when in an interrupt or softirq handler
1260 * in the grace-period kthread's context? Because the kthread might have
1261 * been interrupted just as it was going to sleep, and just after the final
1262 * pre-sleep check of the awaken condition. In this case, a wakeup really
1263 * is required, and is therefore supplied.
1265 static void rcu_gp_kthread_wake(void)
1267 struct task_struct
*t
= READ_ONCE(rcu_state
.gp_kthread
);
1269 if ((current
== t
&& !in_irq() && !in_serving_softirq()) ||
1270 !READ_ONCE(rcu_state
.gp_flags
) || !t
)
1272 WRITE_ONCE(rcu_state
.gp_wake_time
, jiffies
);
1273 WRITE_ONCE(rcu_state
.gp_wake_seq
, READ_ONCE(rcu_state
.gp_seq
));
1274 swake_up_one(&rcu_state
.gp_wq
);
1278 * If there is room, assign a ->gp_seq number to any callbacks on this
1279 * CPU that have not already been assigned. Also accelerate any callbacks
1280 * that were previously assigned a ->gp_seq number that has since proven
1281 * to be too conservative, which can happen if callbacks get assigned a
1282 * ->gp_seq number while RCU is idle, but with reference to a non-root
1283 * rcu_node structure. This function is idempotent, so it does not hurt
1284 * to call it repeatedly. Returns an flag saying that we should awaken
1285 * the RCU grace-period kthread.
1287 * The caller must hold rnp->lock with interrupts disabled.
1289 static bool rcu_accelerate_cbs(struct rcu_node
*rnp
, struct rcu_data
*rdp
)
1291 unsigned long gp_seq_req
;
1294 rcu_lockdep_assert_cblist_protected(rdp
);
1295 raw_lockdep_assert_held_rcu_node(rnp
);
1297 /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
1298 if (!rcu_segcblist_pend_cbs(&rdp
->cblist
))
1302 * Callbacks are often registered with incomplete grace-period
1303 * information. Something about the fact that getting exact
1304 * information requires acquiring a global lock... RCU therefore
1305 * makes a conservative estimate of the grace period number at which
1306 * a given callback will become ready to invoke. The following
1307 * code checks this estimate and improves it when possible, thus
1308 * accelerating callback invocation to an earlier grace-period
1311 gp_seq_req
= rcu_seq_snap(&rcu_state
.gp_seq
);
1312 if (rcu_segcblist_accelerate(&rdp
->cblist
, gp_seq_req
))
1313 ret
= rcu_start_this_gp(rnp
, rdp
, gp_seq_req
);
1315 /* Trace depending on how much we were able to accelerate. */
1316 if (rcu_segcblist_restempty(&rdp
->cblist
, RCU_WAIT_TAIL
))
1317 trace_rcu_grace_period(rcu_state
.name
, rdp
->gp_seq
, TPS("AccWaitCB"));
1319 trace_rcu_grace_period(rcu_state
.name
, rdp
->gp_seq
, TPS("AccReadyCB"));
1324 * Similar to rcu_accelerate_cbs(), but does not require that the leaf
1325 * rcu_node structure's ->lock be held. It consults the cached value
1326 * of ->gp_seq_needed in the rcu_data structure, and if that indicates
1327 * that a new grace-period request be made, invokes rcu_accelerate_cbs()
1328 * while holding the leaf rcu_node structure's ->lock.
1330 static void rcu_accelerate_cbs_unlocked(struct rcu_node
*rnp
,
1331 struct rcu_data
*rdp
)
1336 rcu_lockdep_assert_cblist_protected(rdp
);
1337 c
= rcu_seq_snap(&rcu_state
.gp_seq
);
1338 if (!READ_ONCE(rdp
->gpwrap
) && ULONG_CMP_GE(rdp
->gp_seq_needed
, c
)) {
1339 /* Old request still live, so mark recent callbacks. */
1340 (void)rcu_segcblist_accelerate(&rdp
->cblist
, c
);
1343 raw_spin_lock_rcu_node(rnp
); /* irqs already disabled. */
1344 needwake
= rcu_accelerate_cbs(rnp
, rdp
);
1345 raw_spin_unlock_rcu_node(rnp
); /* irqs remain disabled. */
1347 rcu_gp_kthread_wake();
1351 * Move any callbacks whose grace period has completed to the
1352 * RCU_DONE_TAIL sublist, then compact the remaining sublists and
1353 * assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL
1354 * sublist. This function is idempotent, so it does not hurt to
1355 * invoke it repeatedly. As long as it is not invoked -too- often...
1356 * Returns true if the RCU grace-period kthread needs to be awakened.
1358 * The caller must hold rnp->lock with interrupts disabled.
1360 static bool rcu_advance_cbs(struct rcu_node
*rnp
, struct rcu_data
*rdp
)
1362 rcu_lockdep_assert_cblist_protected(rdp
);
1363 raw_lockdep_assert_held_rcu_node(rnp
);
1365 /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
1366 if (!rcu_segcblist_pend_cbs(&rdp
->cblist
))
1370 * Find all callbacks whose ->gp_seq numbers indicate that they
1371 * are ready to invoke, and put them into the RCU_DONE_TAIL sublist.
1373 rcu_segcblist_advance(&rdp
->cblist
, rnp
->gp_seq
);
1375 /* Classify any remaining callbacks. */
1376 return rcu_accelerate_cbs(rnp
, rdp
);
1380 * Move and classify callbacks, but only if doing so won't require
1381 * that the RCU grace-period kthread be awakened.
1383 static void __maybe_unused
rcu_advance_cbs_nowake(struct rcu_node
*rnp
,
1384 struct rcu_data
*rdp
)
1386 rcu_lockdep_assert_cblist_protected(rdp
);
1387 if (!rcu_seq_state(rcu_seq_current(&rnp
->gp_seq
)) ||
1388 !raw_spin_trylock_rcu_node(rnp
))
1390 WARN_ON_ONCE(rcu_advance_cbs(rnp
, rdp
));
1391 raw_spin_unlock_rcu_node(rnp
);
1395 * Update CPU-local rcu_data state to record the beginnings and ends of
1396 * grace periods. The caller must hold the ->lock of the leaf rcu_node
1397 * structure corresponding to the current CPU, and must have irqs disabled.
1398 * Returns true if the grace-period kthread needs to be awakened.
1400 static bool __note_gp_changes(struct rcu_node
*rnp
, struct rcu_data
*rdp
)
1404 const bool offloaded
= IS_ENABLED(CONFIG_RCU_NOCB_CPU
) &&
1405 rcu_segcblist_is_offloaded(&rdp
->cblist
);
1407 raw_lockdep_assert_held_rcu_node(rnp
);
1409 if (rdp
->gp_seq
== rnp
->gp_seq
)
1410 return false; /* Nothing to do. */
1412 /* Handle the ends of any preceding grace periods first. */
1413 if (rcu_seq_completed_gp(rdp
->gp_seq
, rnp
->gp_seq
) ||
1414 unlikely(READ_ONCE(rdp
->gpwrap
))) {
1416 ret
= rcu_advance_cbs(rnp
, rdp
); /* Advance CBs. */
1417 rdp
->core_needs_qs
= false;
1418 trace_rcu_grace_period(rcu_state
.name
, rdp
->gp_seq
, TPS("cpuend"));
1421 ret
= rcu_accelerate_cbs(rnp
, rdp
); /* Recent CBs. */
1422 if (rdp
->core_needs_qs
)
1423 rdp
->core_needs_qs
= !!(rnp
->qsmask
& rdp
->grpmask
);
1426 /* Now handle the beginnings of any new-to-this-CPU grace periods. */
1427 if (rcu_seq_new_gp(rdp
->gp_seq
, rnp
->gp_seq
) ||
1428 unlikely(READ_ONCE(rdp
->gpwrap
))) {
1430 * If the current grace period is waiting for this CPU,
1431 * set up to detect a quiescent state, otherwise don't
1432 * go looking for one.
1434 trace_rcu_grace_period(rcu_state
.name
, rnp
->gp_seq
, TPS("cpustart"));
1435 need_qs
= !!(rnp
->qsmask
& rdp
->grpmask
);
1436 rdp
->cpu_no_qs
.b
.norm
= need_qs
;
1437 rdp
->core_needs_qs
= need_qs
;
1438 zero_cpu_stall_ticks(rdp
);
1440 rdp
->gp_seq
= rnp
->gp_seq
; /* Remember new grace-period state. */
1441 if (ULONG_CMP_LT(rdp
->gp_seq_needed
, rnp
->gp_seq_needed
) || rdp
->gpwrap
)
1442 WRITE_ONCE(rdp
->gp_seq_needed
, rnp
->gp_seq_needed
);
1443 WRITE_ONCE(rdp
->gpwrap
, false);
1444 rcu_gpnum_ovf(rnp
, rdp
);
1448 static void note_gp_changes(struct rcu_data
*rdp
)
1450 unsigned long flags
;
1452 struct rcu_node
*rnp
;
1454 local_irq_save(flags
);
1456 if ((rdp
->gp_seq
== rcu_seq_current(&rnp
->gp_seq
) &&
1457 !unlikely(READ_ONCE(rdp
->gpwrap
))) || /* w/out lock. */
1458 !raw_spin_trylock_rcu_node(rnp
)) { /* irqs already off, so later. */
1459 local_irq_restore(flags
);
1462 needwake
= __note_gp_changes(rnp
, rdp
);
1463 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1465 rcu_gp_kthread_wake();
1468 static void rcu_gp_slow(int delay
)
1471 !(rcu_seq_ctr(rcu_state
.gp_seq
) %
1472 (rcu_num_nodes
* PER_RCU_NODE_PERIOD
* delay
)))
1473 schedule_timeout_uninterruptible(delay
);
1477 * Initialize a new grace period. Return false if no grace period required.
1479 static bool rcu_gp_init(void)
1481 unsigned long flags
;
1482 unsigned long oldmask
;
1484 struct rcu_data
*rdp
;
1485 struct rcu_node
*rnp
= rcu_get_root();
1487 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
1488 raw_spin_lock_irq_rcu_node(rnp
);
1489 if (!READ_ONCE(rcu_state
.gp_flags
)) {
1490 /* Spurious wakeup, tell caller to go back to sleep. */
1491 raw_spin_unlock_irq_rcu_node(rnp
);
1494 WRITE_ONCE(rcu_state
.gp_flags
, 0); /* Clear all flags: New GP. */
1496 if (WARN_ON_ONCE(rcu_gp_in_progress())) {
1498 * Grace period already in progress, don't start another.
1499 * Not supposed to be able to happen.
1501 raw_spin_unlock_irq_rcu_node(rnp
);
1505 /* Advance to a new grace period and initialize state. */
1506 record_gp_stall_check_time();
1507 /* Record GP times before starting GP, hence rcu_seq_start(). */
1508 rcu_seq_start(&rcu_state
.gp_seq
);
1509 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
, TPS("start"));
1510 raw_spin_unlock_irq_rcu_node(rnp
);
1513 * Apply per-leaf buffered online and offline operations to the
1514 * rcu_node tree. Note that this new grace period need not wait
1515 * for subsequent online CPUs, and that quiescent-state forcing
1516 * will handle subsequent offline CPUs.
1518 rcu_state
.gp_state
= RCU_GP_ONOFF
;
1519 rcu_for_each_leaf_node(rnp
) {
1520 raw_spin_lock(&rcu_state
.ofl_lock
);
1521 raw_spin_lock_irq_rcu_node(rnp
);
1522 if (rnp
->qsmaskinit
== rnp
->qsmaskinitnext
&&
1523 !rnp
->wait_blkd_tasks
) {
1524 /* Nothing to do on this leaf rcu_node structure. */
1525 raw_spin_unlock_irq_rcu_node(rnp
);
1526 raw_spin_unlock(&rcu_state
.ofl_lock
);
1530 /* Record old state, apply changes to ->qsmaskinit field. */
1531 oldmask
= rnp
->qsmaskinit
;
1532 rnp
->qsmaskinit
= rnp
->qsmaskinitnext
;
1534 /* If zero-ness of ->qsmaskinit changed, propagate up tree. */
1535 if (!oldmask
!= !rnp
->qsmaskinit
) {
1536 if (!oldmask
) { /* First online CPU for rcu_node. */
1537 if (!rnp
->wait_blkd_tasks
) /* Ever offline? */
1538 rcu_init_new_rnp(rnp
);
1539 } else if (rcu_preempt_has_tasks(rnp
)) {
1540 rnp
->wait_blkd_tasks
= true; /* blocked tasks */
1541 } else { /* Last offline CPU and can propagate. */
1542 rcu_cleanup_dead_rnp(rnp
);
1547 * If all waited-on tasks from prior grace period are
1548 * done, and if all this rcu_node structure's CPUs are
1549 * still offline, propagate up the rcu_node tree and
1550 * clear ->wait_blkd_tasks. Otherwise, if one of this
1551 * rcu_node structure's CPUs has since come back online,
1552 * simply clear ->wait_blkd_tasks.
1554 if (rnp
->wait_blkd_tasks
&&
1555 (!rcu_preempt_has_tasks(rnp
) || rnp
->qsmaskinit
)) {
1556 rnp
->wait_blkd_tasks
= false;
1557 if (!rnp
->qsmaskinit
)
1558 rcu_cleanup_dead_rnp(rnp
);
1561 raw_spin_unlock_irq_rcu_node(rnp
);
1562 raw_spin_unlock(&rcu_state
.ofl_lock
);
1564 rcu_gp_slow(gp_preinit_delay
); /* Races with CPU hotplug. */
1567 * Set the quiescent-state-needed bits in all the rcu_node
1568 * structures for all currently online CPUs in breadth-first
1569 * order, starting from the root rcu_node structure, relying on the
1570 * layout of the tree within the rcu_state.node[] array. Note that
1571 * other CPUs will access only the leaves of the hierarchy, thus
1572 * seeing that no grace period is in progress, at least until the
1573 * corresponding leaf node has been initialized.
1575 * The grace period cannot complete until the initialization
1576 * process finishes, because this kthread handles both.
1578 rcu_state
.gp_state
= RCU_GP_INIT
;
1579 rcu_for_each_node_breadth_first(rnp
) {
1580 rcu_gp_slow(gp_init_delay
);
1581 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
1582 rdp
= this_cpu_ptr(&rcu_data
);
1583 rcu_preempt_check_blocked_tasks(rnp
);
1584 rnp
->qsmask
= rnp
->qsmaskinit
;
1585 WRITE_ONCE(rnp
->gp_seq
, rcu_state
.gp_seq
);
1586 if (rnp
== rdp
->mynode
)
1587 (void)__note_gp_changes(rnp
, rdp
);
1588 rcu_preempt_boost_start_gp(rnp
);
1589 trace_rcu_grace_period_init(rcu_state
.name
, rnp
->gp_seq
,
1590 rnp
->level
, rnp
->grplo
,
1591 rnp
->grphi
, rnp
->qsmask
);
1592 /* Quiescent states for tasks on any now-offline CPUs. */
1593 mask
= rnp
->qsmask
& ~rnp
->qsmaskinitnext
;
1594 rnp
->rcu_gp_init_mask
= mask
;
1595 if ((mask
|| rnp
->wait_blkd_tasks
) && rcu_is_leaf_node(rnp
))
1596 rcu_report_qs_rnp(mask
, rnp
, rnp
->gp_seq
, flags
);
1598 raw_spin_unlock_irq_rcu_node(rnp
);
1599 cond_resched_tasks_rcu_qs();
1600 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
1607 * Helper function for swait_event_idle_exclusive() wakeup at force-quiescent-state
1610 static bool rcu_gp_fqs_check_wake(int *gfp
)
1612 struct rcu_node
*rnp
= rcu_get_root();
1614 /* Someone like call_rcu() requested a force-quiescent-state scan. */
1615 *gfp
= READ_ONCE(rcu_state
.gp_flags
);
1616 if (*gfp
& RCU_GP_FLAG_FQS
)
1619 /* The current grace period has completed. */
1620 if (!READ_ONCE(rnp
->qsmask
) && !rcu_preempt_blocked_readers_cgp(rnp
))
1627 * Do one round of quiescent-state forcing.
1629 static void rcu_gp_fqs(bool first_time
)
1631 struct rcu_node
*rnp
= rcu_get_root();
1633 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
1634 rcu_state
.n_force_qs
++;
1636 /* Collect dyntick-idle snapshots. */
1637 force_qs_rnp(dyntick_save_progress_counter
);
1639 /* Handle dyntick-idle and offline CPUs. */
1640 force_qs_rnp(rcu_implicit_dynticks_qs
);
1642 /* Clear flag to prevent immediate re-entry. */
1643 if (READ_ONCE(rcu_state
.gp_flags
) & RCU_GP_FLAG_FQS
) {
1644 raw_spin_lock_irq_rcu_node(rnp
);
1645 WRITE_ONCE(rcu_state
.gp_flags
,
1646 READ_ONCE(rcu_state
.gp_flags
) & ~RCU_GP_FLAG_FQS
);
1647 raw_spin_unlock_irq_rcu_node(rnp
);
1652 * Loop doing repeated quiescent-state forcing until the grace period ends.
1654 static void rcu_gp_fqs_loop(void)
1660 struct rcu_node
*rnp
= rcu_get_root();
1662 first_gp_fqs
= true;
1663 j
= READ_ONCE(jiffies_till_first_fqs
);
1667 rcu_state
.jiffies_force_qs
= jiffies
+ j
;
1668 WRITE_ONCE(rcu_state
.jiffies_kick_kthreads
,
1669 jiffies
+ (j
? 3 * j
: 2));
1671 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
,
1673 rcu_state
.gp_state
= RCU_GP_WAIT_FQS
;
1674 ret
= swait_event_idle_timeout_exclusive(
1675 rcu_state
.gp_wq
, rcu_gp_fqs_check_wake(&gf
), j
);
1676 rcu_state
.gp_state
= RCU_GP_DOING_FQS
;
1677 /* Locking provides needed memory barriers. */
1678 /* If grace period done, leave loop. */
1679 if (!READ_ONCE(rnp
->qsmask
) &&
1680 !rcu_preempt_blocked_readers_cgp(rnp
))
1682 /* If time for quiescent-state forcing, do it. */
1683 if (ULONG_CMP_GE(jiffies
, rcu_state
.jiffies_force_qs
) ||
1684 (gf
& RCU_GP_FLAG_FQS
)) {
1685 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
,
1687 rcu_gp_fqs(first_gp_fqs
);
1688 first_gp_fqs
= false;
1689 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
,
1691 cond_resched_tasks_rcu_qs();
1692 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
1693 ret
= 0; /* Force full wait till next FQS. */
1694 j
= READ_ONCE(jiffies_till_next_fqs
);
1696 /* Deal with stray signal. */
1697 cond_resched_tasks_rcu_qs();
1698 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
1699 WARN_ON(signal_pending(current
));
1700 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
,
1702 ret
= 1; /* Keep old FQS timing. */
1704 if (time_after(jiffies
, rcu_state
.jiffies_force_qs
))
1707 j
= rcu_state
.jiffies_force_qs
- j
;
1713 * Clean up after the old grace period.
1715 static void rcu_gp_cleanup(void)
1718 bool needgp
= false;
1719 unsigned long gp_duration
;
1720 unsigned long new_gp_seq
;
1722 struct rcu_data
*rdp
;
1723 struct rcu_node
*rnp
= rcu_get_root();
1724 struct swait_queue_head
*sq
;
1726 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
1727 raw_spin_lock_irq_rcu_node(rnp
);
1728 rcu_state
.gp_end
= jiffies
;
1729 gp_duration
= rcu_state
.gp_end
- rcu_state
.gp_start
;
1730 if (gp_duration
> rcu_state
.gp_max
)
1731 rcu_state
.gp_max
= gp_duration
;
1734 * We know the grace period is complete, but to everyone else
1735 * it appears to still be ongoing. But it is also the case
1736 * that to everyone else it looks like there is nothing that
1737 * they can do to advance the grace period. It is therefore
1738 * safe for us to drop the lock in order to mark the grace
1739 * period as completed in all of the rcu_node structures.
1741 raw_spin_unlock_irq_rcu_node(rnp
);
1744 * Propagate new ->gp_seq value to rcu_node structures so that
1745 * other CPUs don't have to wait until the start of the next grace
1746 * period to process their callbacks. This also avoids some nasty
1747 * RCU grace-period initialization races by forcing the end of
1748 * the current grace period to be completely recorded in all of
1749 * the rcu_node structures before the beginning of the next grace
1750 * period is recorded in any of the rcu_node structures.
1752 new_gp_seq
= rcu_state
.gp_seq
;
1753 rcu_seq_end(&new_gp_seq
);
1754 rcu_for_each_node_breadth_first(rnp
) {
1755 raw_spin_lock_irq_rcu_node(rnp
);
1756 if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp
)))
1757 dump_blkd_tasks(rnp
, 10);
1758 WARN_ON_ONCE(rnp
->qsmask
);
1759 WRITE_ONCE(rnp
->gp_seq
, new_gp_seq
);
1760 rdp
= this_cpu_ptr(&rcu_data
);
1761 if (rnp
== rdp
->mynode
)
1762 needgp
= __note_gp_changes(rnp
, rdp
) || needgp
;
1763 /* smp_mb() provided by prior unlock-lock pair. */
1764 needgp
= rcu_future_gp_cleanup(rnp
) || needgp
;
1765 // Reset overload indication for CPUs no longer overloaded
1766 if (rcu_is_leaf_node(rnp
))
1767 for_each_leaf_node_cpu_mask(rnp
, cpu
, rnp
->cbovldmask
) {
1768 rdp
= per_cpu_ptr(&rcu_data
, cpu
);
1769 check_cb_ovld_locked(rdp
, rnp
);
1771 sq
= rcu_nocb_gp_get(rnp
);
1772 raw_spin_unlock_irq_rcu_node(rnp
);
1773 rcu_nocb_gp_cleanup(sq
);
1774 cond_resched_tasks_rcu_qs();
1775 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
1776 rcu_gp_slow(gp_cleanup_delay
);
1778 rnp
= rcu_get_root();
1779 raw_spin_lock_irq_rcu_node(rnp
); /* GP before ->gp_seq update. */
1781 /* Declare grace period done, trace first to use old GP number. */
1782 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
, TPS("end"));
1783 rcu_seq_end(&rcu_state
.gp_seq
);
1784 rcu_state
.gp_state
= RCU_GP_IDLE
;
1785 /* Check for GP requests since above loop. */
1786 rdp
= this_cpu_ptr(&rcu_data
);
1787 if (!needgp
&& ULONG_CMP_LT(rnp
->gp_seq
, rnp
->gp_seq_needed
)) {
1788 trace_rcu_this_gp(rnp
, rdp
, rnp
->gp_seq_needed
,
1789 TPS("CleanupMore"));
1792 /* Advance CBs to reduce false positives below. */
1793 offloaded
= IS_ENABLED(CONFIG_RCU_NOCB_CPU
) &&
1794 rcu_segcblist_is_offloaded(&rdp
->cblist
);
1795 if ((offloaded
|| !rcu_accelerate_cbs(rnp
, rdp
)) && needgp
) {
1796 WRITE_ONCE(rcu_state
.gp_flags
, RCU_GP_FLAG_INIT
);
1797 WRITE_ONCE(rcu_state
.gp_req_activity
, jiffies
);
1798 trace_rcu_grace_period(rcu_state
.name
,
1802 WRITE_ONCE(rcu_state
.gp_flags
,
1803 rcu_state
.gp_flags
& RCU_GP_FLAG_INIT
);
1805 raw_spin_unlock_irq_rcu_node(rnp
);
1809 * Body of kthread that handles grace periods.
1811 static int __noreturn
rcu_gp_kthread(void *unused
)
1813 rcu_bind_gp_kthread();
1816 /* Handle grace-period start. */
1818 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
,
1820 rcu_state
.gp_state
= RCU_GP_WAIT_GPS
;
1821 swait_event_idle_exclusive(rcu_state
.gp_wq
,
1822 READ_ONCE(rcu_state
.gp_flags
) &
1824 rcu_state
.gp_state
= RCU_GP_DONE_GPS
;
1825 /* Locking provides needed memory barrier. */
1828 cond_resched_tasks_rcu_qs();
1829 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
1830 WARN_ON(signal_pending(current
));
1831 trace_rcu_grace_period(rcu_state
.name
, rcu_state
.gp_seq
,
1835 /* Handle quiescent-state forcing. */
1838 /* Handle grace-period end. */
1839 rcu_state
.gp_state
= RCU_GP_CLEANUP
;
1841 rcu_state
.gp_state
= RCU_GP_CLEANED
;
1846 * Report a full set of quiescent states to the rcu_state data structure.
1847 * Invoke rcu_gp_kthread_wake() to awaken the grace-period kthread if
1848 * another grace period is required. Whether we wake the grace-period
1849 * kthread or it awakens itself for the next round of quiescent-state
1850 * forcing, that kthread will clean up after the just-completed grace
1851 * period. Note that the caller must hold rnp->lock, which is released
1854 static void rcu_report_qs_rsp(unsigned long flags
)
1855 __releases(rcu_get_root()->lock
)
1857 raw_lockdep_assert_held_rcu_node(rcu_get_root());
1858 WARN_ON_ONCE(!rcu_gp_in_progress());
1859 WRITE_ONCE(rcu_state
.gp_flags
,
1860 READ_ONCE(rcu_state
.gp_flags
) | RCU_GP_FLAG_FQS
);
1861 raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(), flags
);
1862 rcu_gp_kthread_wake();
1866 * Similar to rcu_report_qs_rdp(), for which it is a helper function.
1867 * Allows quiescent states for a group of CPUs to be reported at one go
1868 * to the specified rcu_node structure, though all the CPUs in the group
1869 * must be represented by the same rcu_node structure (which need not be a
1870 * leaf rcu_node structure, though it often will be). The gps parameter
1871 * is the grace-period snapshot, which means that the quiescent states
1872 * are valid only if rnp->gp_seq is equal to gps. That structure's lock
1873 * must be held upon entry, and it is released before return.
1875 * As a special case, if mask is zero, the bit-already-cleared check is
1876 * disabled. This allows propagating quiescent state due to resumed tasks
1877 * during grace-period initialization.
1879 static void rcu_report_qs_rnp(unsigned long mask
, struct rcu_node
*rnp
,
1880 unsigned long gps
, unsigned long flags
)
1881 __releases(rnp
->lock
)
1883 unsigned long oldmask
= 0;
1884 struct rcu_node
*rnp_c
;
1886 raw_lockdep_assert_held_rcu_node(rnp
);
1888 /* Walk up the rcu_node hierarchy. */
1890 if ((!(rnp
->qsmask
& mask
) && mask
) || rnp
->gp_seq
!= gps
) {
1893 * Our bit has already been cleared, or the
1894 * relevant grace period is already over, so done.
1896 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1899 WARN_ON_ONCE(oldmask
); /* Any child must be all zeroed! */
1900 WARN_ON_ONCE(!rcu_is_leaf_node(rnp
) &&
1901 rcu_preempt_blocked_readers_cgp(rnp
));
1902 WRITE_ONCE(rnp
->qsmask
, rnp
->qsmask
& ~mask
);
1903 trace_rcu_quiescent_state_report(rcu_state
.name
, rnp
->gp_seq
,
1904 mask
, rnp
->qsmask
, rnp
->level
,
1905 rnp
->grplo
, rnp
->grphi
,
1907 if (rnp
->qsmask
!= 0 || rcu_preempt_blocked_readers_cgp(rnp
)) {
1909 /* Other bits still set at this level, so done. */
1910 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1913 rnp
->completedqs
= rnp
->gp_seq
;
1914 mask
= rnp
->grpmask
;
1915 if (rnp
->parent
== NULL
) {
1917 /* No more levels. Exit loop holding root lock. */
1921 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1924 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
1925 oldmask
= READ_ONCE(rnp_c
->qsmask
);
1929 * Get here if we are the last CPU to pass through a quiescent
1930 * state for this grace period. Invoke rcu_report_qs_rsp()
1931 * to clean up and start the next grace period if one is needed.
1933 rcu_report_qs_rsp(flags
); /* releases rnp->lock. */
1937 * Record a quiescent state for all tasks that were previously queued
1938 * on the specified rcu_node structure and that were blocking the current
1939 * RCU grace period. The caller must hold the corresponding rnp->lock with
1940 * irqs disabled, and this lock is released upon return, but irqs remain
1943 static void __maybe_unused
1944 rcu_report_unblock_qs_rnp(struct rcu_node
*rnp
, unsigned long flags
)
1945 __releases(rnp
->lock
)
1949 struct rcu_node
*rnp_p
;
1951 raw_lockdep_assert_held_rcu_node(rnp
);
1952 if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RCU
)) ||
1953 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp
)) ||
1955 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1956 return; /* Still need more quiescent states! */
1959 rnp
->completedqs
= rnp
->gp_seq
;
1960 rnp_p
= rnp
->parent
;
1961 if (rnp_p
== NULL
) {
1963 * Only one rcu_node structure in the tree, so don't
1964 * try to report up to its nonexistent parent!
1966 rcu_report_qs_rsp(flags
);
1970 /* Report up the rest of the hierarchy, tracking current ->gp_seq. */
1972 mask
= rnp
->grpmask
;
1973 raw_spin_unlock_rcu_node(rnp
); /* irqs remain disabled. */
1974 raw_spin_lock_rcu_node(rnp_p
); /* irqs already disabled. */
1975 rcu_report_qs_rnp(mask
, rnp_p
, gps
, flags
);
1979 * Record a quiescent state for the specified CPU to that CPU's rcu_data
1980 * structure. This must be called from the specified CPU.
1983 rcu_report_qs_rdp(int cpu
, struct rcu_data
*rdp
)
1985 unsigned long flags
;
1987 bool needwake
= false;
1988 const bool offloaded
= IS_ENABLED(CONFIG_RCU_NOCB_CPU
) &&
1989 rcu_segcblist_is_offloaded(&rdp
->cblist
);
1990 struct rcu_node
*rnp
;
1993 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
1994 if (rdp
->cpu_no_qs
.b
.norm
|| rdp
->gp_seq
!= rnp
->gp_seq
||
1998 * The grace period in which this quiescent state was
1999 * recorded has ended, so don't report it upwards.
2000 * We will instead need a new quiescent state that lies
2001 * within the current grace period.
2003 rdp
->cpu_no_qs
.b
.norm
= true; /* need qs for new gp. */
2004 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2007 mask
= rdp
->grpmask
;
2008 if (rdp
->cpu
== smp_processor_id())
2009 rdp
->core_needs_qs
= false;
2010 if ((rnp
->qsmask
& mask
) == 0) {
2011 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2014 * This GP can't end until cpu checks in, so all of our
2015 * callbacks can be processed during the next GP.
2018 needwake
= rcu_accelerate_cbs(rnp
, rdp
);
2020 rcu_disable_urgency_upon_qs(rdp
);
2021 rcu_report_qs_rnp(mask
, rnp
, rnp
->gp_seq
, flags
);
2022 /* ^^^ Released rnp->lock */
2024 rcu_gp_kthread_wake();
2029 * Check to see if there is a new grace period of which this CPU
2030 * is not yet aware, and if so, set up local rcu_data state for it.
2031 * Otherwise, see if this CPU has just passed through its first
2032 * quiescent state for this grace period, and record that fact if so.
2035 rcu_check_quiescent_state(struct rcu_data
*rdp
)
2037 /* Check for grace-period ends and beginnings. */
2038 note_gp_changes(rdp
);
2041 * Does this CPU still need to do its part for current grace period?
2042 * If no, return and let the other CPUs do their part as well.
2044 if (!rdp
->core_needs_qs
)
2048 * Was there a quiescent state since the beginning of the grace
2049 * period? If no, then exit and wait for the next call.
2051 if (rdp
->cpu_no_qs
.b
.norm
)
2055 * Tell RCU we are done (but rcu_report_qs_rdp() will be the
2058 rcu_report_qs_rdp(rdp
->cpu
, rdp
);
2062 * Near the end of the offline process. Trace the fact that this CPU
2065 int rcutree_dying_cpu(unsigned int cpu
)
2068 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
2069 struct rcu_node
*rnp
= rdp
->mynode
;
2071 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU
))
2074 blkd
= !!(rnp
->qsmask
& rdp
->grpmask
);
2075 trace_rcu_grace_period(rcu_state
.name
, READ_ONCE(rnp
->gp_seq
),
2076 blkd
? TPS("cpuofl") : TPS("cpuofl-bgp"));
2081 * All CPUs for the specified rcu_node structure have gone offline,
2082 * and all tasks that were preempted within an RCU read-side critical
2083 * section while running on one of those CPUs have since exited their RCU
2084 * read-side critical section. Some other CPU is reporting this fact with
2085 * the specified rcu_node structure's ->lock held and interrupts disabled.
2086 * This function therefore goes up the tree of rcu_node structures,
2087 * clearing the corresponding bits in the ->qsmaskinit fields. Note that
2088 * the leaf rcu_node structure's ->qsmaskinit field has already been
2091 * This function does check that the specified rcu_node structure has
2092 * all CPUs offline and no blocked tasks, so it is OK to invoke it
2093 * prematurely. That said, invoking it after the fact will cost you
2094 * a needless lock acquisition. So once it has done its work, don't
2097 static void rcu_cleanup_dead_rnp(struct rcu_node
*rnp_leaf
)
2100 struct rcu_node
*rnp
= rnp_leaf
;
2102 raw_lockdep_assert_held_rcu_node(rnp_leaf
);
2103 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU
) ||
2104 WARN_ON_ONCE(rnp_leaf
->qsmaskinit
) ||
2105 WARN_ON_ONCE(rcu_preempt_has_tasks(rnp_leaf
)))
2108 mask
= rnp
->grpmask
;
2112 raw_spin_lock_rcu_node(rnp
); /* irqs already disabled. */
2113 rnp
->qsmaskinit
&= ~mask
;
2114 /* Between grace periods, so better already be zero! */
2115 WARN_ON_ONCE(rnp
->qsmask
);
2116 if (rnp
->qsmaskinit
) {
2117 raw_spin_unlock_rcu_node(rnp
);
2118 /* irqs remain disabled. */
2121 raw_spin_unlock_rcu_node(rnp
); /* irqs remain disabled. */
2126 * The CPU has been completely removed, and some other CPU is reporting
2127 * this fact from process context. Do the remainder of the cleanup.
2128 * There can only be one CPU hotplug operation at a time, so no need for
2131 int rcutree_dead_cpu(unsigned int cpu
)
2133 struct rcu_data
*rdp
= per_cpu_ptr(&rcu_data
, cpu
);
2134 struct rcu_node
*rnp
= rdp
->mynode
; /* Outgoing CPU's rdp & rnp. */
2136 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU
))
2139 /* Adjust any no-longer-needed kthreads. */
2140 rcu_boost_kthread_setaffinity(rnp
, -1);
2141 /* Do any needed no-CB deferred wakeups from this CPU. */
2142 do_nocb_deferred_wakeup(per_cpu_ptr(&rcu_data
, cpu
));
2144 // Stop-machine done, so allow nohz_full to disable tick.
2145 tick_dep_clear(TICK_DEP_BIT_RCU
);
2150 * Invoke any RCU callbacks that have made it to the end of their grace
2151 * period. Thottle as specified by rdp->blimit.
2153 static void rcu_do_batch(struct rcu_data
*rdp
)
2155 unsigned long flags
;
2156 const bool offloaded
= IS_ENABLED(CONFIG_RCU_NOCB_CPU
) &&
2157 rcu_segcblist_is_offloaded(&rdp
->cblist
);
2158 struct rcu_head
*rhp
;
2159 struct rcu_cblist rcl
= RCU_CBLIST_INITIALIZER(rcl
);
2161 long pending
, tlimit
= 0;
2163 /* If no callbacks are ready, just return. */
2164 if (!rcu_segcblist_ready_cbs(&rdp
->cblist
)) {
2165 trace_rcu_batch_start(rcu_state
.name
,
2166 rcu_segcblist_n_cbs(&rdp
->cblist
), 0);
2167 trace_rcu_batch_end(rcu_state
.name
, 0,
2168 !rcu_segcblist_empty(&rdp
->cblist
),
2169 need_resched(), is_idle_task(current
),
2170 rcu_is_callbacks_kthread());
2175 * Extract the list of ready callbacks, disabling to prevent
2176 * races with call_rcu() from interrupt handlers. Leave the
2177 * callback counts, as rcu_barrier() needs to be conservative.
2179 local_irq_save(flags
);
2181 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
2182 pending
= rcu_segcblist_n_cbs(&rdp
->cblist
);
2183 bl
= max(rdp
->blimit
, pending
>> rcu_divisor
);
2184 if (unlikely(bl
> 100))
2185 tlimit
= local_clock() + rcu_resched_ns
;
2186 trace_rcu_batch_start(rcu_state
.name
,
2187 rcu_segcblist_n_cbs(&rdp
->cblist
), bl
);
2188 rcu_segcblist_extract_done_cbs(&rdp
->cblist
, &rcl
);
2190 rdp
->qlen_last_fqs_check
= rcu_segcblist_n_cbs(&rdp
->cblist
);
2191 rcu_nocb_unlock_irqrestore(rdp
, flags
);
2193 /* Invoke callbacks. */
2194 tick_dep_set_task(current
, TICK_DEP_BIT_RCU
);
2195 rhp
= rcu_cblist_dequeue(&rcl
);
2196 for (; rhp
; rhp
= rcu_cblist_dequeue(&rcl
)) {
2199 debug_rcu_head_unqueue(rhp
);
2201 rcu_lock_acquire(&rcu_callback_map
);
2202 trace_rcu_invoke_callback(rcu_state
.name
, rhp
);
2205 WRITE_ONCE(rhp
->func
, (rcu_callback_t
)0L);
2208 rcu_lock_release(&rcu_callback_map
);
2211 * Stop only if limit reached and CPU has something to do.
2212 * Note: The rcl structure counts down from zero.
2214 if (-rcl
.len
>= bl
&& !offloaded
&&
2216 (!is_idle_task(current
) && !rcu_is_callbacks_kthread())))
2218 if (unlikely(tlimit
)) {
2219 /* only call local_clock() every 32 callbacks */
2220 if (likely((-rcl
.len
& 31) || local_clock() < tlimit
))
2222 /* Exceeded the time limit, so leave. */
2226 WARN_ON_ONCE(in_serving_softirq());
2228 lockdep_assert_irqs_enabled();
2229 cond_resched_tasks_rcu_qs();
2230 lockdep_assert_irqs_enabled();
2235 local_irq_save(flags
);
2238 trace_rcu_batch_end(rcu_state
.name
, count
, !!rcl
.head
, need_resched(),
2239 is_idle_task(current
), rcu_is_callbacks_kthread());
2241 /* Update counts and requeue any remaining callbacks. */
2242 rcu_segcblist_insert_done_cbs(&rdp
->cblist
, &rcl
);
2243 smp_mb(); /* List handling before counting for rcu_barrier(). */
2244 rcu_segcblist_insert_count(&rdp
->cblist
, &rcl
);
2246 /* Reinstate batch limit if we have worked down the excess. */
2247 count
= rcu_segcblist_n_cbs(&rdp
->cblist
);
2248 if (rdp
->blimit
>= DEFAULT_MAX_RCU_BLIMIT
&& count
<= qlowmark
)
2249 rdp
->blimit
= blimit
;
2251 /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
2252 if (count
== 0 && rdp
->qlen_last_fqs_check
!= 0) {
2253 rdp
->qlen_last_fqs_check
= 0;
2254 rdp
->n_force_qs_snap
= rcu_state
.n_force_qs
;
2255 } else if (count
< rdp
->qlen_last_fqs_check
- qhimark
)
2256 rdp
->qlen_last_fqs_check
= count
;
2259 * The following usually indicates a double call_rcu(). To track
2260 * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.
2262 WARN_ON_ONCE(count
== 0 && !rcu_segcblist_empty(&rdp
->cblist
));
2263 WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU
) &&
2264 count
!= 0 && rcu_segcblist_empty(&rdp
->cblist
));
2266 rcu_nocb_unlock_irqrestore(rdp
, flags
);
2268 /* Re-invoke RCU core processing if there are callbacks remaining. */
2269 if (!offloaded
&& rcu_segcblist_ready_cbs(&rdp
->cblist
))
2271 tick_dep_clear_task(current
, TICK_DEP_BIT_RCU
);
2275 * This function is invoked from each scheduling-clock interrupt,
2276 * and checks to see if this CPU is in a non-context-switch quiescent
2277 * state, for example, user mode or idle loop. It also schedules RCU
2278 * core processing. If the current grace period has gone on too long,
2279 * it will ask the scheduler to manufacture a context switch for the sole
2280 * purpose of providing a providing the needed quiescent state.
2282 void rcu_sched_clock_irq(int user
)
2284 trace_rcu_utilization(TPS("Start scheduler-tick"));
2285 raw_cpu_inc(rcu_data
.ticks_this_gp
);
2286 /* The load-acquire pairs with the store-release setting to true. */
2287 if (smp_load_acquire(this_cpu_ptr(&rcu_data
.rcu_urgent_qs
))) {
2288 /* Idle and userspace execution already are quiescent states. */
2289 if (!rcu_is_cpu_rrupt_from_idle() && !user
) {
2290 set_tsk_need_resched(current
);
2291 set_preempt_need_resched();
2293 __this_cpu_write(rcu_data
.rcu_urgent_qs
, false);
2295 rcu_flavor_sched_clock_irq(user
);
2296 if (rcu_pending(user
))
2299 trace_rcu_utilization(TPS("End scheduler-tick"));
2303 * Scan the leaf rcu_node structures. For each structure on which all
2304 * CPUs have reported a quiescent state and on which there are tasks
2305 * blocking the current grace period, initiate RCU priority boosting.
2306 * Otherwise, invoke the specified function to check dyntick state for
2307 * each CPU that has not yet reported a quiescent state.
2309 static void force_qs_rnp(int (*f
)(struct rcu_data
*rdp
))
2312 unsigned long flags
;
2314 struct rcu_data
*rdp
;
2315 struct rcu_node
*rnp
;
2317 rcu_state
.cbovld
= rcu_state
.cbovldnext
;
2318 rcu_state
.cbovldnext
= false;
2319 rcu_for_each_leaf_node(rnp
) {
2320 cond_resched_tasks_rcu_qs();
2322 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
2323 rcu_state
.cbovldnext
|= !!rnp
->cbovldmask
;
2324 if (rnp
->qsmask
== 0) {
2325 if (!IS_ENABLED(CONFIG_PREEMPT_RCU
) ||
2326 rcu_preempt_blocked_readers_cgp(rnp
)) {
2328 * No point in scanning bits because they
2329 * are all zero. But we might need to
2330 * priority-boost blocked readers.
2332 rcu_initiate_boost(rnp
, flags
);
2333 /* rcu_initiate_boost() releases rnp->lock */
2336 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2339 for_each_leaf_node_cpu_mask(rnp
, cpu
, rnp
->qsmask
) {
2340 rdp
= per_cpu_ptr(&rcu_data
, cpu
);
2342 mask
|= rdp
->grpmask
;
2343 rcu_disable_urgency_upon_qs(rdp
);
2347 /* Idle/offline CPUs, report (releases rnp->lock). */
2348 rcu_report_qs_rnp(mask
, rnp
, rnp
->gp_seq
, flags
);
2350 /* Nothing to do here, so just drop the lock. */
2351 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2357 * Force quiescent states on reluctant CPUs, and also detect which
2358 * CPUs are in dyntick-idle mode.
2360 void rcu_force_quiescent_state(void)
2362 unsigned long flags
;
2364 struct rcu_node
*rnp
;
2365 struct rcu_node
*rnp_old
= NULL
;
2367 /* Funnel through hierarchy to reduce memory contention. */
2368 rnp
= __this_cpu_read(rcu_data
.mynode
);
2369 for (; rnp
!= NULL
; rnp
= rnp
->parent
) {
2370 ret
= (READ_ONCE(rcu_state
.gp_flags
) & RCU_GP_FLAG_FQS
) ||
2371 !raw_spin_trylock(&rnp
->fqslock
);
2372 if (rnp_old
!= NULL
)
2373 raw_spin_unlock(&rnp_old
->fqslock
);
2378 /* rnp_old == rcu_get_root(), rnp == NULL. */
2380 /* Reached the root of the rcu_node tree, acquire lock. */
2381 raw_spin_lock_irqsave_rcu_node(rnp_old
, flags
);
2382 raw_spin_unlock(&rnp_old
->fqslock
);
2383 if (READ_ONCE(rcu_state
.gp_flags
) & RCU_GP_FLAG_FQS
) {
2384 raw_spin_unlock_irqrestore_rcu_node(rnp_old
, flags
);
2385 return; /* Someone beat us to it. */
2387 WRITE_ONCE(rcu_state
.gp_flags
,
2388 READ_ONCE(rcu_state
.gp_flags
) | RCU_GP_FLAG_FQS
);
2389 raw_spin_unlock_irqrestore_rcu_node(rnp_old
, flags
);
2390 rcu_gp_kthread_wake();
2392 EXPORT_SYMBOL_GPL(rcu_force_quiescent_state
);
2394 /* Perform RCU core processing work for the current CPU. */
2395 static __latent_entropy
void rcu_core(void)
2397 unsigned long flags
;
2398 struct rcu_data
*rdp
= raw_cpu_ptr(&rcu_data
);
2399 struct rcu_node
*rnp
= rdp
->mynode
;
2400 const bool offloaded
= IS_ENABLED(CONFIG_RCU_NOCB_CPU
) &&
2401 rcu_segcblist_is_offloaded(&rdp
->cblist
);
2403 if (cpu_is_offline(smp_processor_id()))
2405 trace_rcu_utilization(TPS("Start RCU core"));
2406 WARN_ON_ONCE(!rdp
->beenonline
);
2408 /* Report any deferred quiescent states if preemption enabled. */
2409 if (!(preempt_count() & PREEMPT_MASK
)) {
2410 rcu_preempt_deferred_qs(current
);
2411 } else if (rcu_preempt_need_deferred_qs(current
)) {
2412 set_tsk_need_resched(current
);
2413 set_preempt_need_resched();
2416 /* Update RCU state based on any recent quiescent states. */
2417 rcu_check_quiescent_state(rdp
);
2419 /* No grace period and unregistered callbacks? */
2420 if (!rcu_gp_in_progress() &&
2421 rcu_segcblist_is_enabled(&rdp
->cblist
) && !offloaded
) {
2422 local_irq_save(flags
);
2423 if (!rcu_segcblist_restempty(&rdp
->cblist
, RCU_NEXT_READY_TAIL
))
2424 rcu_accelerate_cbs_unlocked(rnp
, rdp
);
2425 local_irq_restore(flags
);
2428 rcu_check_gp_start_stall(rnp
, rdp
, rcu_jiffies_till_stall_check());
2430 /* If there are callbacks ready, invoke them. */
2431 if (!offloaded
&& rcu_segcblist_ready_cbs(&rdp
->cblist
) &&
2432 likely(READ_ONCE(rcu_scheduler_fully_active
)))
2435 /* Do any needed deferred wakeups of rcuo kthreads. */
2436 do_nocb_deferred_wakeup(rdp
);
2437 trace_rcu_utilization(TPS("End RCU core"));
2440 static void rcu_core_si(struct softirq_action
*h
)
2445 static void rcu_wake_cond(struct task_struct
*t
, int status
)
2448 * If the thread is yielding, only wake it when this
2449 * is invoked from idle
2451 if (t
&& (status
!= RCU_KTHREAD_YIELDING
|| is_idle_task(current
)))
2455 static void invoke_rcu_core_kthread(void)
2457 struct task_struct
*t
;
2458 unsigned long flags
;
2460 local_irq_save(flags
);
2461 __this_cpu_write(rcu_data
.rcu_cpu_has_work
, 1);
2462 t
= __this_cpu_read(rcu_data
.rcu_cpu_kthread_task
);
2463 if (t
!= NULL
&& t
!= current
)
2464 rcu_wake_cond(t
, __this_cpu_read(rcu_data
.rcu_cpu_kthread_status
));
2465 local_irq_restore(flags
);
2469 * Wake up this CPU's rcuc kthread to do RCU core processing.
2471 static void invoke_rcu_core(void)
2473 if (!cpu_online(smp_processor_id()))
2476 raise_softirq(RCU_SOFTIRQ
);
2478 invoke_rcu_core_kthread();
2481 static void rcu_cpu_kthread_park(unsigned int cpu
)
2483 per_cpu(rcu_data
.rcu_cpu_kthread_status
, cpu
) = RCU_KTHREAD_OFFCPU
;
2486 static int rcu_cpu_kthread_should_run(unsigned int cpu
)
2488 return __this_cpu_read(rcu_data
.rcu_cpu_has_work
);
2492 * Per-CPU kernel thread that invokes RCU callbacks. This replaces
2493 * the RCU softirq used in configurations of RCU that do not support RCU
2494 * priority boosting.
2496 static void rcu_cpu_kthread(unsigned int cpu
)
2498 unsigned int *statusp
= this_cpu_ptr(&rcu_data
.rcu_cpu_kthread_status
);
2499 char work
, *workp
= this_cpu_ptr(&rcu_data
.rcu_cpu_has_work
);
2502 trace_rcu_utilization(TPS("Start CPU kthread@rcu_run"));
2503 for (spincnt
= 0; spincnt
< 10; spincnt
++) {
2505 *statusp
= RCU_KTHREAD_RUNNING
;
2506 local_irq_disable();
2514 trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
2515 *statusp
= RCU_KTHREAD_WAITING
;
2519 *statusp
= RCU_KTHREAD_YIELDING
;
2520 trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
2521 schedule_timeout_interruptible(2);
2522 trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
2523 *statusp
= RCU_KTHREAD_WAITING
;
2526 static struct smp_hotplug_thread rcu_cpu_thread_spec
= {
2527 .store
= &rcu_data
.rcu_cpu_kthread_task
,
2528 .thread_should_run
= rcu_cpu_kthread_should_run
,
2529 .thread_fn
= rcu_cpu_kthread
,
2530 .thread_comm
= "rcuc/%u",
2531 .setup
= rcu_cpu_kthread_setup
,
2532 .park
= rcu_cpu_kthread_park
,
2536 * Spawn per-CPU RCU core processing kthreads.
2538 static int __init
rcu_spawn_core_kthreads(void)
2542 for_each_possible_cpu(cpu
)
2543 per_cpu(rcu_data
.rcu_cpu_has_work
, cpu
) = 0;
2544 if (!IS_ENABLED(CONFIG_RCU_BOOST
) && use_softirq
)
2546 WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec
),
2547 "%s: Could not start rcuc kthread, OOM is now expected behavior\n", __func__
);
2550 early_initcall(rcu_spawn_core_kthreads
);
2553 * Handle any core-RCU processing required by a call_rcu() invocation.
2555 static void __call_rcu_core(struct rcu_data
*rdp
, struct rcu_head
*head
,
2556 unsigned long flags
)
2559 * If called from an extended quiescent state, invoke the RCU
2560 * core in order to force a re-evaluation of RCU's idleness.
2562 if (!rcu_is_watching())
2565 /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
2566 if (irqs_disabled_flags(flags
) || cpu_is_offline(smp_processor_id()))
2570 * Force the grace period if too many callbacks or too long waiting.
2571 * Enforce hysteresis, and don't invoke rcu_force_quiescent_state()
2572 * if some other CPU has recently done so. Also, don't bother
2573 * invoking rcu_force_quiescent_state() if the newly enqueued callback
2574 * is the only one waiting for a grace period to complete.
2576 if (unlikely(rcu_segcblist_n_cbs(&rdp
->cblist
) >
2577 rdp
->qlen_last_fqs_check
+ qhimark
)) {
2579 /* Are we ignoring a completed grace period? */
2580 note_gp_changes(rdp
);
2582 /* Start a new grace period if one not already started. */
2583 if (!rcu_gp_in_progress()) {
2584 rcu_accelerate_cbs_unlocked(rdp
->mynode
, rdp
);
2586 /* Give the grace period a kick. */
2587 rdp
->blimit
= DEFAULT_MAX_RCU_BLIMIT
;
2588 if (rcu_state
.n_force_qs
== rdp
->n_force_qs_snap
&&
2589 rcu_segcblist_first_pend_cb(&rdp
->cblist
) != head
)
2590 rcu_force_quiescent_state();
2591 rdp
->n_force_qs_snap
= rcu_state
.n_force_qs
;
2592 rdp
->qlen_last_fqs_check
= rcu_segcblist_n_cbs(&rdp
->cblist
);
2598 * RCU callback function to leak a callback.
2600 static void rcu_leak_callback(struct rcu_head
*rhp
)
2605 * Check and if necessary update the leaf rcu_node structure's
2606 * ->cbovldmask bit corresponding to the current CPU based on that CPU's
2607 * number of queued RCU callbacks. The caller must hold the leaf rcu_node
2608 * structure's ->lock.
2610 static void check_cb_ovld_locked(struct rcu_data
*rdp
, struct rcu_node
*rnp
)
2612 raw_lockdep_assert_held_rcu_node(rnp
);
2613 if (qovld_calc
<= 0)
2614 return; // Early boot and wildcard value set.
2615 if (rcu_segcblist_n_cbs(&rdp
->cblist
) >= qovld_calc
)
2616 WRITE_ONCE(rnp
->cbovldmask
, rnp
->cbovldmask
| rdp
->grpmask
);
2618 WRITE_ONCE(rnp
->cbovldmask
, rnp
->cbovldmask
& ~rdp
->grpmask
);
2622 * Check and if necessary update the leaf rcu_node structure's
2623 * ->cbovldmask bit corresponding to the current CPU based on that CPU's
2624 * number of queued RCU callbacks. No locks need be held, but the
2625 * caller must have disabled interrupts.
2627 * Note that this function ignores the possibility that there are a lot
2628 * of callbacks all of which have already seen the end of their respective
2629 * grace periods. This omission is due to the need for no-CBs CPUs to
2630 * be holding ->nocb_lock to do this check, which is too heavy for a
2631 * common-case operation.
2633 static void check_cb_ovld(struct rcu_data
*rdp
)
2635 struct rcu_node
*const rnp
= rdp
->mynode
;
2637 if (qovld_calc
<= 0 ||
2638 ((rcu_segcblist_n_cbs(&rdp
->cblist
) >= qovld_calc
) ==
2639 !!(READ_ONCE(rnp
->cbovldmask
) & rdp
->grpmask
)))
2640 return; // Early boot wildcard value or already set correctly.
2641 raw_spin_lock_rcu_node(rnp
);
2642 check_cb_ovld_locked(rdp
, rnp
);
2643 raw_spin_unlock_rcu_node(rnp
);
2646 /* Helper function for call_rcu() and friends. */
2648 __call_rcu(struct rcu_head
*head
, rcu_callback_t func
)
2650 unsigned long flags
;
2651 struct rcu_data
*rdp
;
2654 /* Misaligned rcu_head! */
2655 WARN_ON_ONCE((unsigned long)head
& (sizeof(void *) - 1));
2657 if (debug_rcu_head_queue(head
)) {
2659 * Probable double call_rcu(), so leak the callback.
2660 * Use rcu:rcu_callback trace event to find the previous
2661 * time callback was passed to __call_rcu().
2663 WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pS()!!!\n",
2665 WRITE_ONCE(head
->func
, rcu_leak_callback
);
2670 local_irq_save(flags
);
2671 rdp
= this_cpu_ptr(&rcu_data
);
2673 /* Add the callback to our list. */
2674 if (unlikely(!rcu_segcblist_is_enabled(&rdp
->cblist
))) {
2675 // This can trigger due to call_rcu() from offline CPU:
2676 WARN_ON_ONCE(rcu_scheduler_active
!= RCU_SCHEDULER_INACTIVE
);
2677 WARN_ON_ONCE(!rcu_is_watching());
2678 // Very early boot, before rcu_init(). Initialize if needed
2679 // and then drop through to queue the callback.
2680 if (rcu_segcblist_empty(&rdp
->cblist
))
2681 rcu_segcblist_init(&rdp
->cblist
);
2685 if (rcu_nocb_try_bypass(rdp
, head
, &was_alldone
, flags
))
2686 return; // Enqueued onto ->nocb_bypass, so just leave.
2687 // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
2688 rcu_segcblist_enqueue(&rdp
->cblist
, head
);
2689 if (__is_kfree_rcu_offset((unsigned long)func
))
2690 trace_rcu_kfree_callback(rcu_state
.name
, head
,
2691 (unsigned long)func
,
2692 rcu_segcblist_n_cbs(&rdp
->cblist
));
2694 trace_rcu_callback(rcu_state
.name
, head
,
2695 rcu_segcblist_n_cbs(&rdp
->cblist
));
2697 /* Go handle any RCU core processing required. */
2698 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU
) &&
2699 unlikely(rcu_segcblist_is_offloaded(&rdp
->cblist
))) {
2700 __call_rcu_nocb_wake(rdp
, was_alldone
, flags
); /* unlocks */
2702 __call_rcu_core(rdp
, head
, flags
);
2703 local_irq_restore(flags
);
2708 * call_rcu() - Queue an RCU callback for invocation after a grace period.
2709 * @head: structure to be used for queueing the RCU updates.
2710 * @func: actual callback function to be invoked after the grace period
2712 * The callback function will be invoked some time after a full grace
2713 * period elapses, in other words after all pre-existing RCU read-side
2714 * critical sections have completed. However, the callback function
2715 * might well execute concurrently with RCU read-side critical sections
2716 * that started after call_rcu() was invoked. RCU read-side critical
2717 * sections are delimited by rcu_read_lock() and rcu_read_unlock(), and
2718 * may be nested. In addition, regions of code across which interrupts,
2719 * preemption, or softirqs have been disabled also serve as RCU read-side
2720 * critical sections. This includes hardware interrupt handlers, softirq
2721 * handlers, and NMI handlers.
2723 * Note that all CPUs must agree that the grace period extended beyond
2724 * all pre-existing RCU read-side critical section. On systems with more
2725 * than one CPU, this means that when "func()" is invoked, each CPU is
2726 * guaranteed to have executed a full memory barrier since the end of its
2727 * last RCU read-side critical section whose beginning preceded the call
2728 * to call_rcu(). It also means that each CPU executing an RCU read-side
2729 * critical section that continues beyond the start of "func()" must have
2730 * executed a memory barrier after the call_rcu() but before the beginning
2731 * of that RCU read-side critical section. Note that these guarantees
2732 * include CPUs that are offline, idle, or executing in user mode, as
2733 * well as CPUs that are executing in the kernel.
2735 * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
2736 * resulting RCU callback function "func()", then both CPU A and CPU B are
2737 * guaranteed to execute a full memory barrier during the time interval
2738 * between the call to call_rcu() and the invocation of "func()" -- even
2739 * if CPU A and CPU B are the same CPU (but again only if the system has
2740 * more than one CPU).
2742 void call_rcu(struct rcu_head
*head
, rcu_callback_t func
)
2744 __call_rcu(head
, func
);
2746 EXPORT_SYMBOL_GPL(call_rcu
);
2749 /* Maximum number of jiffies to wait before draining a batch. */
2750 #define KFREE_DRAIN_JIFFIES (HZ / 50)
2751 #define KFREE_N_BATCHES 2
2754 * This macro defines how many entries the "records" array
2755 * will contain. It is based on the fact that the size of
2756 * kfree_rcu_bulk_data structure becomes exactly one page.
2758 #define KFREE_BULK_MAX_ENTR ((PAGE_SIZE / sizeof(void *)) - 3)
2761 * struct kfree_rcu_bulk_data - single block to store kfree_rcu() pointers
2762 * @nr_records: Number of active pointers in the array
2763 * @records: Array of the kfree_rcu() pointers
2764 * @next: Next bulk object in the block chain
2765 * @head_free_debug: For debug, when CONFIG_DEBUG_OBJECTS_RCU_HEAD is set
2767 struct kfree_rcu_bulk_data
{
2768 unsigned long nr_records
;
2769 void *records
[KFREE_BULK_MAX_ENTR
];
2770 struct kfree_rcu_bulk_data
*next
;
2771 struct rcu_head
*head_free_debug
;
2775 * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
2776 * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
2777 * @head_free: List of kfree_rcu() objects waiting for a grace period
2778 * @bhead_free: Bulk-List of kfree_rcu() objects waiting for a grace period
2779 * @krcp: Pointer to @kfree_rcu_cpu structure
2782 struct kfree_rcu_cpu_work
{
2783 struct rcu_work rcu_work
;
2784 struct rcu_head
*head_free
;
2785 struct kfree_rcu_bulk_data
*bhead_free
;
2786 struct kfree_rcu_cpu
*krcp
;
2790 * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
2791 * @head: List of kfree_rcu() objects not yet waiting for a grace period
2792 * @bhead: Bulk-List of kfree_rcu() objects not yet waiting for a grace period
2793 * @bcached: Keeps at most one object for later reuse when build chain blocks
2794 * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
2795 * @lock: Synchronize access to this structure
2796 * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
2797 * @monitor_todo: Tracks whether a @monitor_work delayed work is pending
2798 * @initialized: The @lock and @rcu_work fields have been initialized
2800 * This is a per-CPU structure. The reason that it is not included in
2801 * the rcu_data structure is to permit this code to be extracted from
2802 * the RCU files. Such extraction could allow further optimization of
2803 * the interactions with the slab allocators.
2805 struct kfree_rcu_cpu
{
2806 struct rcu_head
*head
;
2807 struct kfree_rcu_bulk_data
*bhead
;
2808 struct kfree_rcu_bulk_data
*bcached
;
2809 struct kfree_rcu_cpu_work krw_arr
[KFREE_N_BATCHES
];
2811 struct delayed_work monitor_work
;
2816 static DEFINE_PER_CPU(struct kfree_rcu_cpu
, krc
);
2818 static __always_inline
void
2819 debug_rcu_head_unqueue_bulk(struct rcu_head
*head
)
2821 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
2822 for (; head
; head
= head
->next
)
2823 debug_rcu_head_unqueue(head
);
2828 * This function is invoked in workqueue context after a grace period.
2829 * It frees all the objects queued on ->bhead_free or ->head_free.
2831 static void kfree_rcu_work(struct work_struct
*work
)
2833 unsigned long flags
;
2834 struct rcu_head
*head
, *next
;
2835 struct kfree_rcu_bulk_data
*bhead
, *bnext
;
2836 struct kfree_rcu_cpu
*krcp
;
2837 struct kfree_rcu_cpu_work
*krwp
;
2839 krwp
= container_of(to_rcu_work(work
),
2840 struct kfree_rcu_cpu_work
, rcu_work
);
2842 spin_lock_irqsave(&krcp
->lock
, flags
);
2843 head
= krwp
->head_free
;
2844 krwp
->head_free
= NULL
;
2845 bhead
= krwp
->bhead_free
;
2846 krwp
->bhead_free
= NULL
;
2847 spin_unlock_irqrestore(&krcp
->lock
, flags
);
2849 /* "bhead" is now private, so traverse locklessly. */
2850 for (; bhead
; bhead
= bnext
) {
2851 bnext
= bhead
->next
;
2853 debug_rcu_head_unqueue_bulk(bhead
->head_free_debug
);
2855 rcu_lock_acquire(&rcu_callback_map
);
2856 trace_rcu_invoke_kfree_bulk_callback(rcu_state
.name
,
2857 bhead
->nr_records
, bhead
->records
);
2859 kfree_bulk(bhead
->nr_records
, bhead
->records
);
2860 rcu_lock_release(&rcu_callback_map
);
2862 if (cmpxchg(&krcp
->bcached
, NULL
, bhead
))
2863 free_page((unsigned long) bhead
);
2865 cond_resched_tasks_rcu_qs();
2869 * Emergency case only. It can happen under low memory
2870 * condition when an allocation gets failed, so the "bulk"
2871 * path can not be temporary maintained.
2873 for (; head
; head
= next
) {
2874 unsigned long offset
= (unsigned long)head
->func
;
2877 debug_rcu_head_unqueue(head
);
2878 rcu_lock_acquire(&rcu_callback_map
);
2879 trace_rcu_invoke_kfree_callback(rcu_state
.name
, head
, offset
);
2881 if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset
)))
2882 kfree((void *)head
- offset
);
2884 rcu_lock_release(&rcu_callback_map
);
2885 cond_resched_tasks_rcu_qs();
2890 * Schedule the kfree batch RCU work to run in workqueue context after a GP.
2892 * This function is invoked by kfree_rcu_monitor() when the KFREE_DRAIN_JIFFIES
2893 * timeout has been reached.
2895 static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu
*krcp
)
2897 struct kfree_rcu_cpu_work
*krwp
;
2898 bool queued
= false;
2901 lockdep_assert_held(&krcp
->lock
);
2903 for (i
= 0; i
< KFREE_N_BATCHES
; i
++) {
2904 krwp
= &(krcp
->krw_arr
[i
]);
2907 * Try to detach bhead or head and attach it over any
2908 * available corresponding free channel. It can be that
2909 * a previous RCU batch is in progress, it means that
2910 * immediately to queue another one is not possible so
2911 * return false to tell caller to retry.
2913 if ((krcp
->bhead
&& !krwp
->bhead_free
) ||
2914 (krcp
->head
&& !krwp
->head_free
)) {
2916 if (!krwp
->bhead_free
) {
2917 krwp
->bhead_free
= krcp
->bhead
;
2922 if (!krwp
->head_free
) {
2923 krwp
->head_free
= krcp
->head
;
2928 * One work is per one batch, so there are two "free channels",
2929 * "bhead_free" and "head_free" the batch can handle. It can be
2930 * that the work is in the pending state when two channels have
2931 * been detached following each other, one by one.
2933 queue_rcu_work(system_wq
, &krwp
->rcu_work
);
2941 static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu
*krcp
,
2942 unsigned long flags
)
2944 // Attempt to start a new batch.
2945 krcp
->monitor_todo
= false;
2946 if (queue_kfree_rcu_work(krcp
)) {
2947 // Success! Our job is done here.
2948 spin_unlock_irqrestore(&krcp
->lock
, flags
);
2952 // Previous RCU batch still in progress, try again later.
2953 krcp
->monitor_todo
= true;
2954 schedule_delayed_work(&krcp
->monitor_work
, KFREE_DRAIN_JIFFIES
);
2955 spin_unlock_irqrestore(&krcp
->lock
, flags
);
2959 * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
2960 * It invokes kfree_rcu_drain_unlock() to attempt to start another batch.
2962 static void kfree_rcu_monitor(struct work_struct
*work
)
2964 unsigned long flags
;
2965 struct kfree_rcu_cpu
*krcp
= container_of(work
, struct kfree_rcu_cpu
,
2968 spin_lock_irqsave(&krcp
->lock
, flags
);
2969 if (krcp
->monitor_todo
)
2970 kfree_rcu_drain_unlock(krcp
, flags
);
2972 spin_unlock_irqrestore(&krcp
->lock
, flags
);
2976 kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu
*krcp
,
2977 struct rcu_head
*head
, rcu_callback_t func
)
2979 struct kfree_rcu_bulk_data
*bnode
;
2981 if (unlikely(!krcp
->initialized
))
2984 lockdep_assert_held(&krcp
->lock
);
2986 /* Check if a new block is required. */
2988 krcp
->bhead
->nr_records
== KFREE_BULK_MAX_ENTR
) {
2989 bnode
= xchg(&krcp
->bcached
, NULL
);
2991 WARN_ON_ONCE(sizeof(struct kfree_rcu_bulk_data
) > PAGE_SIZE
);
2993 bnode
= (struct kfree_rcu_bulk_data
*)
2994 __get_free_page(GFP_NOWAIT
| __GFP_NOWARN
);
2997 /* Switch to emergency path. */
2998 if (unlikely(!bnode
))
3001 /* Initialize the new block. */
3002 bnode
->nr_records
= 0;
3003 bnode
->next
= krcp
->bhead
;
3004 bnode
->head_free_debug
= NULL
;
3006 /* Attach it to the head. */
3007 krcp
->bhead
= bnode
;
3010 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
3012 head
->next
= krcp
->bhead
->head_free_debug
;
3013 krcp
->bhead
->head_free_debug
= head
;
3016 /* Finally insert. */
3017 krcp
->bhead
->records
[krcp
->bhead
->nr_records
++] =
3018 (void *) head
- (unsigned long) func
;
3024 * Queue a request for lazy invocation of kfree_bulk()/kfree() after a grace
3025 * period. Please note there are two paths are maintained, one is the main one
3026 * that uses kfree_bulk() interface and second one is emergency one, that is
3027 * used only when the main path can not be maintained temporary, due to memory
3030 * Each kfree_call_rcu() request is added to a batch. The batch will be drained
3031 * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
3032 * be free'd in workqueue context. This allows us to: batch requests together to
3033 * reduce the number of grace periods during heavy kfree_rcu() load.
3035 void kfree_call_rcu(struct rcu_head
*head
, rcu_callback_t func
)
3037 unsigned long flags
;
3038 struct kfree_rcu_cpu
*krcp
;
3040 local_irq_save(flags
); // For safely calling this_cpu_ptr().
3041 krcp
= this_cpu_ptr(&krc
);
3042 if (krcp
->initialized
)
3043 spin_lock(&krcp
->lock
);
3045 // Queue the object but don't yet schedule the batch.
3046 if (debug_rcu_head_queue(head
)) {
3047 // Probable double kfree_rcu(), just leak.
3048 WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
3054 * Under high memory pressure GFP_NOWAIT can fail,
3055 * in that case the emergency path is maintained.
3057 if (unlikely(!kfree_call_rcu_add_ptr_to_bulk(krcp
, head
, func
))) {
3059 head
->next
= krcp
->head
;
3063 // Set timer to drain after KFREE_DRAIN_JIFFIES.
3064 if (rcu_scheduler_active
== RCU_SCHEDULER_RUNNING
&&
3065 !krcp
->monitor_todo
) {
3066 krcp
->monitor_todo
= true;
3067 schedule_delayed_work(&krcp
->monitor_work
, KFREE_DRAIN_JIFFIES
);
3071 if (krcp
->initialized
)
3072 spin_unlock(&krcp
->lock
);
3073 local_irq_restore(flags
);
3075 EXPORT_SYMBOL_GPL(kfree_call_rcu
);
3077 void __init
kfree_rcu_scheduler_running(void)
3080 unsigned long flags
;
3082 for_each_online_cpu(cpu
) {
3083 struct kfree_rcu_cpu
*krcp
= per_cpu_ptr(&krc
, cpu
);
3085 spin_lock_irqsave(&krcp
->lock
, flags
);
3086 if (!krcp
->head
|| krcp
->monitor_todo
) {
3087 spin_unlock_irqrestore(&krcp
->lock
, flags
);
3090 krcp
->monitor_todo
= true;
3091 schedule_delayed_work_on(cpu
, &krcp
->monitor_work
,
3092 KFREE_DRAIN_JIFFIES
);
3093 spin_unlock_irqrestore(&krcp
->lock
, flags
);
3098 * During early boot, any blocking grace-period wait automatically
3099 * implies a grace period. Later on, this is never the case for PREEMPTION.
3101 * Howevr, because a context switch is a grace period for !PREEMPTION, any
3102 * blocking grace-period wait automatically implies a grace period if
3103 * there is only one CPU online at any point time during execution of
3104 * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
3105 * occasionally incorrectly indicate that there are multiple CPUs online
3106 * when there was in fact only one the whole time, as this just adds some
3107 * overhead: RCU still operates correctly.
3109 static int rcu_blocking_is_gp(void)
3113 if (IS_ENABLED(CONFIG_PREEMPTION
))
3114 return rcu_scheduler_active
== RCU_SCHEDULER_INACTIVE
;
3115 might_sleep(); /* Check for RCU read-side critical section. */
3117 ret
= num_online_cpus() <= 1;
3123 * synchronize_rcu - wait until a grace period has elapsed.
3125 * Control will return to the caller some time after a full grace
3126 * period has elapsed, in other words after all currently executing RCU
3127 * read-side critical sections have completed. Note, however, that
3128 * upon return from synchronize_rcu(), the caller might well be executing
3129 * concurrently with new RCU read-side critical sections that began while
3130 * synchronize_rcu() was waiting. RCU read-side critical sections are
3131 * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
3132 * In addition, regions of code across which interrupts, preemption, or
3133 * softirqs have been disabled also serve as RCU read-side critical
3134 * sections. This includes hardware interrupt handlers, softirq handlers,
3137 * Note that this guarantee implies further memory-ordering guarantees.
3138 * On systems with more than one CPU, when synchronize_rcu() returns,
3139 * each CPU is guaranteed to have executed a full memory barrier since
3140 * the end of its last RCU read-side critical section whose beginning
3141 * preceded the call to synchronize_rcu(). In addition, each CPU having
3142 * an RCU read-side critical section that extends beyond the return from
3143 * synchronize_rcu() is guaranteed to have executed a full memory barrier
3144 * after the beginning of synchronize_rcu() and before the beginning of
3145 * that RCU read-side critical section. Note that these guarantees include
3146 * CPUs that are offline, idle, or executing in user mode, as well as CPUs
3147 * that are executing in the kernel.
3149 * Furthermore, if CPU A invoked synchronize_rcu(), which returned
3150 * to its caller on CPU B, then both CPU A and CPU B are guaranteed
3151 * to have executed a full memory barrier during the execution of
3152 * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
3153 * again only if the system has more than one CPU).
3155 void synchronize_rcu(void)
3157 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map
) ||
3158 lock_is_held(&rcu_lock_map
) ||
3159 lock_is_held(&rcu_sched_lock_map
),
3160 "Illegal synchronize_rcu() in RCU read-side critical section");
3161 if (rcu_blocking_is_gp())
3163 if (rcu_gp_is_expedited())
3164 synchronize_rcu_expedited();
3166 wait_rcu_gp(call_rcu
);
3168 EXPORT_SYMBOL_GPL(synchronize_rcu
);
3171 * get_state_synchronize_rcu - Snapshot current RCU state
3173 * Returns a cookie that is used by a later call to cond_synchronize_rcu()
3174 * to determine whether or not a full grace period has elapsed in the
3177 unsigned long get_state_synchronize_rcu(void)
3180 * Any prior manipulation of RCU-protected data must happen
3181 * before the load from ->gp_seq.
3184 return rcu_seq_snap(&rcu_state
.gp_seq
);
3186 EXPORT_SYMBOL_GPL(get_state_synchronize_rcu
);
3189 * cond_synchronize_rcu - Conditionally wait for an RCU grace period
3191 * @oldstate: return value from earlier call to get_state_synchronize_rcu()
3193 * If a full RCU grace period has elapsed since the earlier call to
3194 * get_state_synchronize_rcu(), just return. Otherwise, invoke
3195 * synchronize_rcu() to wait for a full grace period.
3197 * Yes, this function does not take counter wrap into account. But
3198 * counter wrap is harmless. If the counter wraps, we have waited for
3199 * more than 2 billion grace periods (and way more on a 64-bit system!),
3200 * so waiting for one additional grace period should be just fine.
3202 void cond_synchronize_rcu(unsigned long oldstate
)
3204 if (!rcu_seq_done(&rcu_state
.gp_seq
, oldstate
))
3207 smp_mb(); /* Ensure GP ends before subsequent accesses. */
3209 EXPORT_SYMBOL_GPL(cond_synchronize_rcu
);
3212 * Check to see if there is any immediate RCU-related work to be done by
3213 * the current CPU, returning 1 if so and zero otherwise. The checks are
3214 * in order of increasing expense: checks that can be carried out against
3215 * CPU-local state are performed first. However, we must check for CPU
3216 * stalls first, else we might not get a chance.
3218 static int rcu_pending(int user
)
3220 bool gp_in_progress
;
3221 struct rcu_data
*rdp
= this_cpu_ptr(&rcu_data
);
3222 struct rcu_node
*rnp
= rdp
->mynode
;
3224 /* Check for CPU stalls, if enabled. */
3225 check_cpu_stall(rdp
);
3227 /* Does this CPU need a deferred NOCB wakeup? */
3228 if (rcu_nocb_need_deferred_wakeup(rdp
))
3231 /* Is this a nohz_full CPU in userspace or idle? (Ignore RCU if so.) */
3232 if ((user
|| rcu_is_cpu_rrupt_from_idle()) && rcu_nohz_full_cpu())
3235 /* Is the RCU core waiting for a quiescent state from this CPU? */
3236 gp_in_progress
= rcu_gp_in_progress();
3237 if (rdp
->core_needs_qs
&& !rdp
->cpu_no_qs
.b
.norm
&& gp_in_progress
)
3240 /* Does this CPU have callbacks ready to invoke? */
3241 if (rcu_segcblist_ready_cbs(&rdp
->cblist
))
3244 /* Has RCU gone idle with this CPU needing another grace period? */
3245 if (!gp_in_progress
&& rcu_segcblist_is_enabled(&rdp
->cblist
) &&
3246 (!IS_ENABLED(CONFIG_RCU_NOCB_CPU
) ||
3247 !rcu_segcblist_is_offloaded(&rdp
->cblist
)) &&
3248 !rcu_segcblist_restempty(&rdp
->cblist
, RCU_NEXT_READY_TAIL
))
3251 /* Have RCU grace period completed or started? */
3252 if (rcu_seq_current(&rnp
->gp_seq
) != rdp
->gp_seq
||
3253 unlikely(READ_ONCE(rdp
->gpwrap
))) /* outside lock */
3261 * Helper function for rcu_barrier() tracing. If tracing is disabled,
3262 * the compiler is expected to optimize this away.
3264 static void rcu_barrier_trace(const char *s
, int cpu
, unsigned long done
)
3266 trace_rcu_barrier(rcu_state
.name
, s
, cpu
,
3267 atomic_read(&rcu_state
.barrier_cpu_count
), done
);
3271 * RCU callback function for rcu_barrier(). If we are last, wake
3272 * up the task executing rcu_barrier().
3274 * Note that the value of rcu_state.barrier_sequence must be captured
3275 * before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
3276 * other CPUs might count the value down to zero before this CPU gets
3277 * around to invoking rcu_barrier_trace(), which might result in bogus
3278 * data from the next instance of rcu_barrier().
3280 static void rcu_barrier_callback(struct rcu_head
*rhp
)
3282 unsigned long __maybe_unused s
= rcu_state
.barrier_sequence
;
3284 if (atomic_dec_and_test(&rcu_state
.barrier_cpu_count
)) {
3285 rcu_barrier_trace(TPS("LastCB"), -1, s
);
3286 complete(&rcu_state
.barrier_completion
);
3288 rcu_barrier_trace(TPS("CB"), -1, s
);
3293 * Called with preemption disabled, and from cross-cpu IRQ context.
3295 static void rcu_barrier_func(void *cpu_in
)
3297 uintptr_t cpu
= (uintptr_t)cpu_in
;
3298 struct rcu_data
*rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3300 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state
.barrier_sequence
);
3301 rdp
->barrier_head
.func
= rcu_barrier_callback
;
3302 debug_rcu_head_queue(&rdp
->barrier_head
);
3304 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp
, NULL
, jiffies
));
3305 if (rcu_segcblist_entrain(&rdp
->cblist
, &rdp
->barrier_head
)) {
3306 atomic_inc(&rcu_state
.barrier_cpu_count
);
3308 debug_rcu_head_unqueue(&rdp
->barrier_head
);
3309 rcu_barrier_trace(TPS("IRQNQ"), -1,
3310 rcu_state
.barrier_sequence
);
3312 rcu_nocb_unlock(rdp
);
3316 * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
3318 * Note that this primitive does not necessarily wait for an RCU grace period
3319 * to complete. For example, if there are no RCU callbacks queued anywhere
3320 * in the system, then rcu_barrier() is within its rights to return
3321 * immediately, without waiting for anything, much less an RCU grace period.
3323 void rcu_barrier(void)
3326 struct rcu_data
*rdp
;
3327 unsigned long s
= rcu_seq_snap(&rcu_state
.barrier_sequence
);
3329 rcu_barrier_trace(TPS("Begin"), -1, s
);
3331 /* Take mutex to serialize concurrent rcu_barrier() requests. */
3332 mutex_lock(&rcu_state
.barrier_mutex
);
3334 /* Did someone else do our work for us? */
3335 if (rcu_seq_done(&rcu_state
.barrier_sequence
, s
)) {
3336 rcu_barrier_trace(TPS("EarlyExit"), -1,
3337 rcu_state
.barrier_sequence
);
3338 smp_mb(); /* caller's subsequent code after above check. */
3339 mutex_unlock(&rcu_state
.barrier_mutex
);
3343 /* Mark the start of the barrier operation. */
3344 rcu_seq_start(&rcu_state
.barrier_sequence
);
3345 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state
.barrier_sequence
);
3348 * Initialize the count to two rather than to zero in order
3349 * to avoid a too-soon return to zero in case of an immediate
3350 * invocation of the just-enqueued callback (or preemption of
3351 * this task). Exclude CPU-hotplug operations to ensure that no
3352 * offline non-offloaded CPU has callbacks queued.
3354 init_completion(&rcu_state
.barrier_completion
);
3355 atomic_set(&rcu_state
.barrier_cpu_count
, 2);
3359 * Force each CPU with callbacks to register a new callback.
3360 * When that callback is invoked, we will know that all of the
3361 * corresponding CPU's preceding callbacks have been invoked.
3363 for_each_possible_cpu(cpu
) {
3364 rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3365 if (cpu_is_offline(cpu
) &&
3366 !rcu_segcblist_is_offloaded(&rdp
->cblist
))
3368 if (rcu_segcblist_n_cbs(&rdp
->cblist
) && cpu_online(cpu
)) {
3369 rcu_barrier_trace(TPS("OnlineQ"), cpu
,
3370 rcu_state
.barrier_sequence
);
3371 smp_call_function_single(cpu
, rcu_barrier_func
, (void *)cpu
, 1);
3372 } else if (rcu_segcblist_n_cbs(&rdp
->cblist
) &&
3373 cpu_is_offline(cpu
)) {
3374 rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu
,
3375 rcu_state
.barrier_sequence
);
3376 local_irq_disable();
3377 rcu_barrier_func((void *)cpu
);
3379 } else if (cpu_is_offline(cpu
)) {
3380 rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu
,
3381 rcu_state
.barrier_sequence
);
3383 rcu_barrier_trace(TPS("OnlineNQ"), cpu
,
3384 rcu_state
.barrier_sequence
);
3390 * Now that we have an rcu_barrier_callback() callback on each
3391 * CPU, and thus each counted, remove the initial count.
3393 if (atomic_sub_and_test(2, &rcu_state
.barrier_cpu_count
))
3394 complete(&rcu_state
.barrier_completion
);
3396 /* Wait for all rcu_barrier_callback() callbacks to be invoked. */
3397 wait_for_completion(&rcu_state
.barrier_completion
);
3399 /* Mark the end of the barrier operation. */
3400 rcu_barrier_trace(TPS("Inc2"), -1, rcu_state
.barrier_sequence
);
3401 rcu_seq_end(&rcu_state
.barrier_sequence
);
3403 /* Other rcu_barrier() invocations can now safely proceed. */
3404 mutex_unlock(&rcu_state
.barrier_mutex
);
3406 EXPORT_SYMBOL_GPL(rcu_barrier
);
3409 * Propagate ->qsinitmask bits up the rcu_node tree to account for the
3410 * first CPU in a given leaf rcu_node structure coming online. The caller
3411 * must hold the corresponding leaf rcu_node ->lock with interrrupts
3414 static void rcu_init_new_rnp(struct rcu_node
*rnp_leaf
)
3418 struct rcu_node
*rnp
= rnp_leaf
;
3420 raw_lockdep_assert_held_rcu_node(rnp_leaf
);
3421 WARN_ON_ONCE(rnp
->wait_blkd_tasks
);
3423 mask
= rnp
->grpmask
;
3427 raw_spin_lock_rcu_node(rnp
); /* Interrupts already disabled. */
3428 oldmask
= rnp
->qsmaskinit
;
3429 rnp
->qsmaskinit
|= mask
;
3430 raw_spin_unlock_rcu_node(rnp
); /* Interrupts remain disabled. */
3437 * Do boot-time initialization of a CPU's per-CPU RCU data.
3440 rcu_boot_init_percpu_data(int cpu
)
3442 struct rcu_data
*rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3444 /* Set up local state, ensuring consistent view of global state. */
3445 rdp
->grpmask
= leaf_node_cpu_bit(rdp
->mynode
, cpu
);
3446 WARN_ON_ONCE(rdp
->dynticks_nesting
!= 1);
3447 WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp
)));
3448 rdp
->rcu_ofl_gp_seq
= rcu_state
.gp_seq
;
3449 rdp
->rcu_ofl_gp_flags
= RCU_GP_CLEANED
;
3450 rdp
->rcu_onl_gp_seq
= rcu_state
.gp_seq
;
3451 rdp
->rcu_onl_gp_flags
= RCU_GP_CLEANED
;
3453 rcu_boot_init_nocb_percpu_data(rdp
);
3457 * Invoked early in the CPU-online process, when pretty much all services
3458 * are available. The incoming CPU is not present.
3460 * Initializes a CPU's per-CPU RCU data. Note that only one online or
3461 * offline event can be happening at a given time. Note also that we can
3462 * accept some slop in the rsp->gp_seq access due to the fact that this
3463 * CPU cannot possibly have any non-offloaded RCU callbacks in flight yet.
3464 * And any offloaded callbacks are being numbered elsewhere.
3466 int rcutree_prepare_cpu(unsigned int cpu
)
3468 unsigned long flags
;
3469 struct rcu_data
*rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3470 struct rcu_node
*rnp
= rcu_get_root();
3472 /* Set up local state, ensuring consistent view of global state. */
3473 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3474 rdp
->qlen_last_fqs_check
= 0;
3475 rdp
->n_force_qs_snap
= rcu_state
.n_force_qs
;
3476 rdp
->blimit
= blimit
;
3477 if (rcu_segcblist_empty(&rdp
->cblist
) && /* No early-boot CBs? */
3478 !rcu_segcblist_is_offloaded(&rdp
->cblist
))
3479 rcu_segcblist_init(&rdp
->cblist
); /* Re-enable callbacks. */
3480 rdp
->dynticks_nesting
= 1; /* CPU not up, no tearing. */
3481 rcu_dynticks_eqs_online();
3482 raw_spin_unlock_rcu_node(rnp
); /* irqs remain disabled. */
3485 * Add CPU to leaf rcu_node pending-online bitmask. Any needed
3486 * propagation up the rcu_node tree will happen at the beginning
3487 * of the next grace period.
3490 raw_spin_lock_rcu_node(rnp
); /* irqs already disabled. */
3491 rdp
->beenonline
= true; /* We have now been online. */
3492 rdp
->gp_seq
= READ_ONCE(rnp
->gp_seq
);
3493 rdp
->gp_seq_needed
= rdp
->gp_seq
;
3494 rdp
->cpu_no_qs
.b
.norm
= true;
3495 rdp
->core_needs_qs
= false;
3496 rdp
->rcu_iw_pending
= false;
3497 rdp
->rcu_iw_gp_seq
= rdp
->gp_seq
- 1;
3498 trace_rcu_grace_period(rcu_state
.name
, rdp
->gp_seq
, TPS("cpuonl"));
3499 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3500 rcu_prepare_kthreads(cpu
);
3501 rcu_spawn_cpu_nocb_kthread(cpu
);
3507 * Update RCU priority boot kthread affinity for CPU-hotplug changes.
3509 static void rcutree_affinity_setting(unsigned int cpu
, int outgoing
)
3511 struct rcu_data
*rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3513 rcu_boost_kthread_setaffinity(rdp
->mynode
, outgoing
);
3517 * Near the end of the CPU-online process. Pretty much all services
3518 * enabled, and the CPU is now very much alive.
3520 int rcutree_online_cpu(unsigned int cpu
)
3522 unsigned long flags
;
3523 struct rcu_data
*rdp
;
3524 struct rcu_node
*rnp
;
3526 rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3528 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3529 rnp
->ffmask
|= rdp
->grpmask
;
3530 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3531 if (rcu_scheduler_active
== RCU_SCHEDULER_INACTIVE
)
3532 return 0; /* Too early in boot for scheduler work. */
3533 sync_sched_exp_online_cleanup(cpu
);
3534 rcutree_affinity_setting(cpu
, -1);
3536 // Stop-machine done, so allow nohz_full to disable tick.
3537 tick_dep_clear(TICK_DEP_BIT_RCU
);
3542 * Near the beginning of the process. The CPU is still very much alive
3543 * with pretty much all services enabled.
3545 int rcutree_offline_cpu(unsigned int cpu
)
3547 unsigned long flags
;
3548 struct rcu_data
*rdp
;
3549 struct rcu_node
*rnp
;
3551 rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3553 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3554 rnp
->ffmask
&= ~rdp
->grpmask
;
3555 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3557 rcutree_affinity_setting(cpu
, cpu
);
3559 // nohz_full CPUs need the tick for stop-machine to work quickly
3560 tick_dep_set(TICK_DEP_BIT_RCU
);
3564 static DEFINE_PER_CPU(int, rcu_cpu_started
);
3567 * Mark the specified CPU as being online so that subsequent grace periods
3568 * (both expedited and normal) will wait on it. Note that this means that
3569 * incoming CPUs are not allowed to use RCU read-side critical sections
3570 * until this function is called. Failing to observe this restriction
3571 * will result in lockdep splats.
3573 * Note that this function is special in that it is invoked directly
3574 * from the incoming CPU rather than from the cpuhp_step mechanism.
3575 * This is because this function must be invoked at a precise location.
3577 void rcu_cpu_starting(unsigned int cpu
)
3579 unsigned long flags
;
3582 unsigned long oldmask
;
3583 struct rcu_data
*rdp
;
3584 struct rcu_node
*rnp
;
3586 if (per_cpu(rcu_cpu_started
, cpu
))
3589 per_cpu(rcu_cpu_started
, cpu
) = 1;
3591 rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3593 mask
= rdp
->grpmask
;
3594 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3595 WRITE_ONCE(rnp
->qsmaskinitnext
, rnp
->qsmaskinitnext
| mask
);
3596 oldmask
= rnp
->expmaskinitnext
;
3597 rnp
->expmaskinitnext
|= mask
;
3598 oldmask
^= rnp
->expmaskinitnext
;
3599 nbits
= bitmap_weight(&oldmask
, BITS_PER_LONG
);
3600 /* Allow lockless access for expedited grace periods. */
3601 smp_store_release(&rcu_state
.ncpus
, rcu_state
.ncpus
+ nbits
); /* ^^^ */
3602 rcu_gpnum_ovf(rnp
, rdp
); /* Offline-induced counter wrap? */
3603 rdp
->rcu_onl_gp_seq
= READ_ONCE(rcu_state
.gp_seq
);
3604 rdp
->rcu_onl_gp_flags
= READ_ONCE(rcu_state
.gp_flags
);
3605 if (rnp
->qsmask
& mask
) { /* RCU waiting on incoming CPU? */
3606 rcu_disable_urgency_upon_qs(rdp
);
3607 /* Report QS -after- changing ->qsmaskinitnext! */
3608 rcu_report_qs_rnp(mask
, rnp
, rnp
->gp_seq
, flags
);
3610 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3612 smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
3615 #ifdef CONFIG_HOTPLUG_CPU
3617 * The outgoing function has no further need of RCU, so remove it from
3618 * the rcu_node tree's ->qsmaskinitnext bit masks.
3620 * Note that this function is special in that it is invoked directly
3621 * from the outgoing CPU rather than from the cpuhp_step mechanism.
3622 * This is because this function must be invoked at a precise location.
3624 void rcu_report_dead(unsigned int cpu
)
3626 unsigned long flags
;
3628 struct rcu_data
*rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3629 struct rcu_node
*rnp
= rdp
->mynode
; /* Outgoing CPU's rdp & rnp. */
3631 /* QS for any half-done expedited grace period. */
3633 rcu_report_exp_rdp(this_cpu_ptr(&rcu_data
));
3635 rcu_preempt_deferred_qs(current
);
3637 /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
3638 mask
= rdp
->grpmask
;
3639 raw_spin_lock(&rcu_state
.ofl_lock
);
3640 raw_spin_lock_irqsave_rcu_node(rnp
, flags
); /* Enforce GP memory-order guarantee. */
3641 rdp
->rcu_ofl_gp_seq
= READ_ONCE(rcu_state
.gp_seq
);
3642 rdp
->rcu_ofl_gp_flags
= READ_ONCE(rcu_state
.gp_flags
);
3643 if (rnp
->qsmask
& mask
) { /* RCU waiting on outgoing CPU? */
3644 /* Report quiescent state -before- changing ->qsmaskinitnext! */
3645 rcu_report_qs_rnp(mask
, rnp
, rnp
->gp_seq
, flags
);
3646 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3648 WRITE_ONCE(rnp
->qsmaskinitnext
, rnp
->qsmaskinitnext
& ~mask
);
3649 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3650 raw_spin_unlock(&rcu_state
.ofl_lock
);
3652 per_cpu(rcu_cpu_started
, cpu
) = 0;
3656 * The outgoing CPU has just passed through the dying-idle state, and we
3657 * are being invoked from the CPU that was IPIed to continue the offline
3658 * operation. Migrate the outgoing CPU's callbacks to the current CPU.
3660 void rcutree_migrate_callbacks(int cpu
)
3662 unsigned long flags
;
3663 struct rcu_data
*my_rdp
;
3664 struct rcu_node
*my_rnp
;
3665 struct rcu_data
*rdp
= per_cpu_ptr(&rcu_data
, cpu
);
3668 if (rcu_segcblist_is_offloaded(&rdp
->cblist
) ||
3669 rcu_segcblist_empty(&rdp
->cblist
))
3670 return; /* No callbacks to migrate. */
3672 local_irq_save(flags
);
3673 my_rdp
= this_cpu_ptr(&rcu_data
);
3674 my_rnp
= my_rdp
->mynode
;
3675 rcu_nocb_lock(my_rdp
); /* irqs already disabled. */
3676 WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp
, NULL
, jiffies
));
3677 raw_spin_lock_rcu_node(my_rnp
); /* irqs already disabled. */
3678 /* Leverage recent GPs and set GP for new callbacks. */
3679 needwake
= rcu_advance_cbs(my_rnp
, rdp
) ||
3680 rcu_advance_cbs(my_rnp
, my_rdp
);
3681 rcu_segcblist_merge(&my_rdp
->cblist
, &rdp
->cblist
);
3682 needwake
= needwake
|| rcu_advance_cbs(my_rnp
, my_rdp
);
3683 rcu_segcblist_disable(&rdp
->cblist
);
3684 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp
->cblist
) !=
3685 !rcu_segcblist_n_cbs(&my_rdp
->cblist
));
3686 if (rcu_segcblist_is_offloaded(&my_rdp
->cblist
)) {
3687 raw_spin_unlock_rcu_node(my_rnp
); /* irqs remain disabled. */
3688 __call_rcu_nocb_wake(my_rdp
, true, flags
);
3690 rcu_nocb_unlock(my_rdp
); /* irqs remain disabled. */
3691 raw_spin_unlock_irqrestore_rcu_node(my_rnp
, flags
);
3694 rcu_gp_kthread_wake();
3695 lockdep_assert_irqs_enabled();
3696 WARN_ONCE(rcu_segcblist_n_cbs(&rdp
->cblist
) != 0 ||
3697 !rcu_segcblist_empty(&rdp
->cblist
),
3698 "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
3699 cpu
, rcu_segcblist_n_cbs(&rdp
->cblist
),
3700 rcu_segcblist_first_cb(&rdp
->cblist
));
3705 * On non-huge systems, use expedited RCU grace periods to make suspend
3706 * and hibernation run faster.
3708 static int rcu_pm_notify(struct notifier_block
*self
,
3709 unsigned long action
, void *hcpu
)
3712 case PM_HIBERNATION_PREPARE
:
3713 case PM_SUSPEND_PREPARE
:
3716 case PM_POST_HIBERNATION
:
3717 case PM_POST_SUSPEND
:
3718 rcu_unexpedite_gp();
3727 * Spawn the kthreads that handle RCU's grace periods.
3729 static int __init
rcu_spawn_gp_kthread(void)
3731 unsigned long flags
;
3732 int kthread_prio_in
= kthread_prio
;
3733 struct rcu_node
*rnp
;
3734 struct sched_param sp
;
3735 struct task_struct
*t
;
3737 /* Force priority into range. */
3738 if (IS_ENABLED(CONFIG_RCU_BOOST
) && kthread_prio
< 2
3739 && IS_BUILTIN(CONFIG_RCU_TORTURE_TEST
))
3741 else if (IS_ENABLED(CONFIG_RCU_BOOST
) && kthread_prio
< 1)
3743 else if (kthread_prio
< 0)
3745 else if (kthread_prio
> 99)
3748 if (kthread_prio
!= kthread_prio_in
)
3749 pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n",
3750 kthread_prio
, kthread_prio_in
);
3752 rcu_scheduler_fully_active
= 1;
3753 t
= kthread_create(rcu_gp_kthread
, NULL
, "%s", rcu_state
.name
);
3754 if (WARN_ONCE(IS_ERR(t
), "%s: Could not start grace-period kthread, OOM is now expected behavior\n", __func__
))
3757 sp
.sched_priority
= kthread_prio
;
3758 sched_setscheduler_nocheck(t
, SCHED_FIFO
, &sp
);
3760 rnp
= rcu_get_root();
3761 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3762 WRITE_ONCE(rcu_state
.gp_activity
, jiffies
);
3763 WRITE_ONCE(rcu_state
.gp_req_activity
, jiffies
);
3764 // Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
3765 smp_store_release(&rcu_state
.gp_kthread
, t
); /* ^^^ */
3766 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3768 rcu_spawn_nocb_kthreads();
3769 rcu_spawn_boost_kthreads();
3772 early_initcall(rcu_spawn_gp_kthread
);
3775 * This function is invoked towards the end of the scheduler's
3776 * initialization process. Before this is called, the idle task might
3777 * contain synchronous grace-period primitives (during which time, this idle
3778 * task is booting the system, and such primitives are no-ops). After this
3779 * function is called, any synchronous grace-period primitives are run as
3780 * expedited, with the requesting task driving the grace period forward.
3781 * A later core_initcall() rcu_set_runtime_mode() will switch to full
3782 * runtime RCU functionality.
3784 void rcu_scheduler_starting(void)
3786 WARN_ON(num_online_cpus() != 1);
3787 WARN_ON(nr_context_switches() > 0);
3788 rcu_test_sync_prims();
3789 rcu_scheduler_active
= RCU_SCHEDULER_INIT
;
3790 rcu_test_sync_prims();
3794 * Helper function for rcu_init() that initializes the rcu_state structure.
3796 static void __init
rcu_init_one(void)
3798 static const char * const buf
[] = RCU_NODE_NAME_INIT
;
3799 static const char * const fqs
[] = RCU_FQS_NAME_INIT
;
3800 static struct lock_class_key rcu_node_class
[RCU_NUM_LVLS
];
3801 static struct lock_class_key rcu_fqs_class
[RCU_NUM_LVLS
];
3803 int levelspread
[RCU_NUM_LVLS
]; /* kids/node in each level. */
3807 struct rcu_node
*rnp
;
3809 BUILD_BUG_ON(RCU_NUM_LVLS
> ARRAY_SIZE(buf
)); /* Fix buf[] init! */
3811 /* Silence gcc 4.8 false positive about array index out of range. */
3812 if (rcu_num_lvls
<= 0 || rcu_num_lvls
> RCU_NUM_LVLS
)
3813 panic("rcu_init_one: rcu_num_lvls out of range");
3815 /* Initialize the level-tracking arrays. */
3817 for (i
= 1; i
< rcu_num_lvls
; i
++)
3818 rcu_state
.level
[i
] =
3819 rcu_state
.level
[i
- 1] + num_rcu_lvl
[i
- 1];
3820 rcu_init_levelspread(levelspread
, num_rcu_lvl
);
3822 /* Initialize the elements themselves, starting from the leaves. */
3824 for (i
= rcu_num_lvls
- 1; i
>= 0; i
--) {
3825 cpustride
*= levelspread
[i
];
3826 rnp
= rcu_state
.level
[i
];
3827 for (j
= 0; j
< num_rcu_lvl
[i
]; j
++, rnp
++) {
3828 raw_spin_lock_init(&ACCESS_PRIVATE(rnp
, lock
));
3829 lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp
, lock
),
3830 &rcu_node_class
[i
], buf
[i
]);
3831 raw_spin_lock_init(&rnp
->fqslock
);
3832 lockdep_set_class_and_name(&rnp
->fqslock
,
3833 &rcu_fqs_class
[i
], fqs
[i
]);
3834 rnp
->gp_seq
= rcu_state
.gp_seq
;
3835 rnp
->gp_seq_needed
= rcu_state
.gp_seq
;
3836 rnp
->completedqs
= rcu_state
.gp_seq
;
3838 rnp
->qsmaskinit
= 0;
3839 rnp
->grplo
= j
* cpustride
;
3840 rnp
->grphi
= (j
+ 1) * cpustride
- 1;
3841 if (rnp
->grphi
>= nr_cpu_ids
)
3842 rnp
->grphi
= nr_cpu_ids
- 1;
3848 rnp
->grpnum
= j
% levelspread
[i
- 1];
3849 rnp
->grpmask
= BIT(rnp
->grpnum
);
3850 rnp
->parent
= rcu_state
.level
[i
- 1] +
3851 j
/ levelspread
[i
- 1];
3854 INIT_LIST_HEAD(&rnp
->blkd_tasks
);
3855 rcu_init_one_nocb(rnp
);
3856 init_waitqueue_head(&rnp
->exp_wq
[0]);
3857 init_waitqueue_head(&rnp
->exp_wq
[1]);
3858 init_waitqueue_head(&rnp
->exp_wq
[2]);
3859 init_waitqueue_head(&rnp
->exp_wq
[3]);
3860 spin_lock_init(&rnp
->exp_lock
);
3864 init_swait_queue_head(&rcu_state
.gp_wq
);
3865 init_swait_queue_head(&rcu_state
.expedited_wq
);
3866 rnp
= rcu_first_leaf_node();
3867 for_each_possible_cpu(i
) {
3868 while (i
> rnp
->grphi
)
3870 per_cpu_ptr(&rcu_data
, i
)->mynode
= rnp
;
3871 rcu_boot_init_percpu_data(i
);
3876 * Compute the rcu_node tree geometry from kernel parameters. This cannot
3877 * replace the definitions in tree.h because those are needed to size
3878 * the ->node array in the rcu_state structure.
3880 static void __init
rcu_init_geometry(void)
3884 int rcu_capacity
[RCU_NUM_LVLS
];
3887 * Initialize any unspecified boot parameters.
3888 * The default values of jiffies_till_first_fqs and
3889 * jiffies_till_next_fqs are set to the RCU_JIFFIES_TILL_FORCE_QS
3890 * value, which is a function of HZ, then adding one for each
3891 * RCU_JIFFIES_FQS_DIV CPUs that might be on the system.
3893 d
= RCU_JIFFIES_TILL_FORCE_QS
+ nr_cpu_ids
/ RCU_JIFFIES_FQS_DIV
;
3894 if (jiffies_till_first_fqs
== ULONG_MAX
)
3895 jiffies_till_first_fqs
= d
;
3896 if (jiffies_till_next_fqs
== ULONG_MAX
)
3897 jiffies_till_next_fqs
= d
;
3898 adjust_jiffies_till_sched_qs();
3900 /* If the compile-time values are accurate, just leave. */
3901 if (rcu_fanout_leaf
== RCU_FANOUT_LEAF
&&
3902 nr_cpu_ids
== NR_CPUS
)
3904 pr_info("Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n",
3905 rcu_fanout_leaf
, nr_cpu_ids
);
3908 * The boot-time rcu_fanout_leaf parameter must be at least two
3909 * and cannot exceed the number of bits in the rcu_node masks.
3910 * Complain and fall back to the compile-time values if this
3911 * limit is exceeded.
3913 if (rcu_fanout_leaf
< 2 ||
3914 rcu_fanout_leaf
> sizeof(unsigned long) * 8) {
3915 rcu_fanout_leaf
= RCU_FANOUT_LEAF
;
3921 * Compute number of nodes that can be handled an rcu_node tree
3922 * with the given number of levels.
3924 rcu_capacity
[0] = rcu_fanout_leaf
;
3925 for (i
= 1; i
< RCU_NUM_LVLS
; i
++)
3926 rcu_capacity
[i
] = rcu_capacity
[i
- 1] * RCU_FANOUT
;
3929 * The tree must be able to accommodate the configured number of CPUs.
3930 * If this limit is exceeded, fall back to the compile-time values.
3932 if (nr_cpu_ids
> rcu_capacity
[RCU_NUM_LVLS
- 1]) {
3933 rcu_fanout_leaf
= RCU_FANOUT_LEAF
;
3938 /* Calculate the number of levels in the tree. */
3939 for (i
= 0; nr_cpu_ids
> rcu_capacity
[i
]; i
++) {
3941 rcu_num_lvls
= i
+ 1;
3943 /* Calculate the number of rcu_nodes at each level of the tree. */
3944 for (i
= 0; i
< rcu_num_lvls
; i
++) {
3945 int cap
= rcu_capacity
[(rcu_num_lvls
- 1) - i
];
3946 num_rcu_lvl
[i
] = DIV_ROUND_UP(nr_cpu_ids
, cap
);
3949 /* Calculate the total number of rcu_node structures. */
3951 for (i
= 0; i
< rcu_num_lvls
; i
++)
3952 rcu_num_nodes
+= num_rcu_lvl
[i
];
3956 * Dump out the structure of the rcu_node combining tree associated
3957 * with the rcu_state structure.
3959 static void __init
rcu_dump_rcu_node_tree(void)
3962 struct rcu_node
*rnp
;
3964 pr_info("rcu_node tree layout dump\n");
3966 rcu_for_each_node_breadth_first(rnp
) {
3967 if (rnp
->level
!= level
) {
3972 pr_cont("%d:%d ^%d ", rnp
->grplo
, rnp
->grphi
, rnp
->grpnum
);
3977 struct workqueue_struct
*rcu_gp_wq
;
3978 struct workqueue_struct
*rcu_par_gp_wq
;
3980 static void __init
kfree_rcu_batch_init(void)
3985 for_each_possible_cpu(cpu
) {
3986 struct kfree_rcu_cpu
*krcp
= per_cpu_ptr(&krc
, cpu
);
3988 spin_lock_init(&krcp
->lock
);
3989 for (i
= 0; i
< KFREE_N_BATCHES
; i
++) {
3990 INIT_RCU_WORK(&krcp
->krw_arr
[i
].rcu_work
, kfree_rcu_work
);
3991 krcp
->krw_arr
[i
].krcp
= krcp
;
3994 INIT_DELAYED_WORK(&krcp
->monitor_work
, kfree_rcu_monitor
);
3995 krcp
->initialized
= true;
3999 void __init
rcu_init(void)
4003 rcu_early_boot_tests();
4005 kfree_rcu_batch_init();
4006 rcu_bootup_announce();
4007 rcu_init_geometry();
4010 rcu_dump_rcu_node_tree();
4012 open_softirq(RCU_SOFTIRQ
, rcu_core_si
);
4015 * We don't need protection against CPU-hotplug here because
4016 * this is called early in boot, before either interrupts
4017 * or the scheduler are operational.
4019 pm_notifier(rcu_pm_notify
, 0);
4020 for_each_online_cpu(cpu
) {
4021 rcutree_prepare_cpu(cpu
);
4022 rcu_cpu_starting(cpu
);
4023 rcutree_online_cpu(cpu
);
4026 /* Create workqueue for expedited GPs and for Tree SRCU. */
4027 rcu_gp_wq
= alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM
, 0);
4028 WARN_ON(!rcu_gp_wq
);
4029 rcu_par_gp_wq
= alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM
, 0);
4030 WARN_ON(!rcu_par_gp_wq
);
4033 /* Fill in default value for rcutree.qovld boot parameter. */
4034 /* -After- the rcu_node ->lock fields are initialized! */
4036 qovld_calc
= DEFAULT_RCU_QOVLD_MULT
* qhimark
;
4041 #include "tree_stall.h"
4042 #include "tree_exp.h"
4043 #include "tree_plugin.h"