]>
Commit | Line | Data |
---|---|---|
eb7935e4 | 1 | /* SPDX-License-Identifier: GPL-2.0+ */ |
45753c5f | 2 | /* |
98059b98 | 3 | * RCU segmented callback lists, internal-to-rcu header file |
45753c5f | 4 | * |
45753c5f IM |
5 | * Copyright IBM Corporation, 2017 |
6 | * | |
eb7935e4 | 7 | * Authors: Paul E. McKenney <paulmck@linux.ibm.com> |
45753c5f IM |
8 | */ |
9 | ||
10 | #include <linux/rcu_segcblist.h> | |
11 | ||
eda669a6 PM |
12 | /* Return number of callbacks in the specified callback list. */ |
13 | static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) | |
14 | { | |
15 | return READ_ONCE(rclp->len); | |
16 | } | |
17 | ||
98059b98 | 18 | void rcu_cblist_init(struct rcu_cblist *rclp); |
d1b222c6 PM |
19 | void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); |
20 | void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, | |
21 | struct rcu_cblist *srclp, | |
22 | struct rcu_head *rhp); | |
98059b98 | 23 | struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); |
45753c5f IM |
24 | |
25 | /* | |
26 | * Is the specified rcu_segcblist structure empty? | |
27 | * | |
28 | * But careful! The fact that the ->head field is NULL does not | |
29 | * necessarily imply that there are no callbacks associated with | |
30 | * this structure. When callbacks are being invoked, they are | |
31 | * removed as a group. If callback invocation must be preempted, | |
32 | * the remaining callbacks will be added back to the list. Either | |
33 | * way, the counts are updated later. | |
34 | * | |
35 | * So it is often the case that rcu_segcblist_n_cbs() should be used | |
36 | * instead. | |
37 | */ | |
38 | static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) | |
39 | { | |
e6060b41 | 40 | return !READ_ONCE(rsclp->head); |
45753c5f IM |
41 | } |
42 | ||
43 | /* Return number of callbacks in segmented callback list. */ | |
44 | static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) | |
45 | { | |
eda669a6 PM |
46 | #ifdef CONFIG_RCU_NOCB_CPU |
47 | return atomic_long_read(&rsclp->len); | |
48 | #else | |
45753c5f | 49 | return READ_ONCE(rsclp->len); |
eda669a6 | 50 | #endif |
45753c5f IM |
51 | } |
52 | ||
45753c5f IM |
53 | /* |
54 | * Is the specified rcu_segcblist enabled, for example, not corresponding | |
e83e73f5 | 55 | * to an offline CPU? |
45753c5f IM |
56 | */ |
57 | static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) | |
58 | { | |
1bb5f9b9 | 59 | return rsclp->enabled; |
45753c5f IM |
60 | } |
61 | ||
ce5215c1 PM |
62 | /* Is the specified rcu_segcblist offloaded? */ |
63 | static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) | |
64 | { | |
65 | return rsclp->offloaded; | |
66 | } | |
67 | ||
45753c5f IM |
68 | /* |
69 | * Are all segments following the specified segment of the specified | |
70 | * rcu_segcblist structure empty of callbacks? (The specified | |
71 | * segment might well contain callbacks.) | |
72 | */ | |
73 | static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) | |
74 | { | |
76c6927c | 75 | return !READ_ONCE(*READ_ONCE(rsclp->tails[seg])); |
45753c5f IM |
76 | } |
77 | ||
d1b222c6 | 78 | void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); |
98059b98 PM |
79 | void rcu_segcblist_init(struct rcu_segcblist *rsclp); |
80 | void rcu_segcblist_disable(struct rcu_segcblist *rsclp); | |
ce5215c1 | 81 | void rcu_segcblist_offload(struct rcu_segcblist *rsclp); |
98059b98 PM |
82 | bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); |
83 | bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); | |
98059b98 PM |
84 | struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); |
85 | struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); | |
5d6742b3 | 86 | bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); |
98059b98 | 87 | void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, |
77a40f97 | 88 | struct rcu_head *rhp); |
98059b98 | 89 | bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, |
77a40f97 | 90 | struct rcu_head *rhp); |
98059b98 PM |
91 | void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, |
92 | struct rcu_cblist *rclp); | |
93 | void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, | |
94 | struct rcu_cblist *rclp); | |
95 | void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, | |
96 | struct rcu_cblist *rclp); | |
97 | void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, | |
98 | struct rcu_cblist *rclp); | |
99 | void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, | |
100 | struct rcu_cblist *rclp); | |
101 | void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, | |
102 | struct rcu_cblist *rclp); | |
103 | void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); | |
104 | bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); | |
f2dbe4a5 PM |
105 | void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, |
106 | struct rcu_segcblist *src_rsclp); |