]>
Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
9390ef0c | 2 | #include <linux/atomic.h> |
a1fd3e24 ON |
3 | #include <linux/rwsem.h> |
4 | #include <linux/percpu.h> | |
8ebe3473 | 5 | #include <linux/lockdep.h> |
a1fd3e24 ON |
6 | #include <linux/percpu-rwsem.h> |
7 | #include <linux/rcupdate.h> | |
8 | #include <linux/sched.h> | |
9 | #include <linux/errno.h> | |
10 | ||
46ad0840 WL |
11 | #include "rwsem.h" |
12 | ||
80127a39 | 13 | int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, |
8ebe3473 | 14 | const char *name, struct lock_class_key *rwsem_key) |
a1fd3e24 | 15 | { |
80127a39 PZ |
16 | sem->read_count = alloc_percpu(int); |
17 | if (unlikely(!sem->read_count)) | |
a1fd3e24 ON |
18 | return -ENOMEM; |
19 | ||
8ebe3473 | 20 | /* ->rw_sem represents the whole percpu_rw_semaphore for lockdep */ |
80127a39 PZ |
21 | rcu_sync_init(&sem->rss, RCU_SCHED_SYNC); |
22 | __init_rwsem(&sem->rw_sem, name, rwsem_key); | |
52b94129 | 23 | rcuwait_init(&sem->writer); |
80127a39 | 24 | sem->readers_block = 0; |
a1fd3e24 ON |
25 | return 0; |
26 | } | |
302707fd | 27 | EXPORT_SYMBOL_GPL(__percpu_init_rwsem); |
a1fd3e24 | 28 | |
80127a39 | 29 | void percpu_free_rwsem(struct percpu_rw_semaphore *sem) |
a1fd3e24 | 30 | { |
95b19f68 ON |
31 | /* |
32 | * XXX: temporary kludge. The error path in alloc_super() | |
33 | * assumes that percpu_free_rwsem() is safe after kzalloc(). | |
34 | */ | |
80127a39 | 35 | if (!sem->read_count) |
95b19f68 ON |
36 | return; |
37 | ||
80127a39 PZ |
38 | rcu_sync_dtor(&sem->rss); |
39 | free_percpu(sem->read_count); | |
40 | sem->read_count = NULL; /* catch use after free bugs */ | |
a1fd3e24 | 41 | } |
c8585c6f | 42 | EXPORT_SYMBOL_GPL(percpu_free_rwsem); |
a1fd3e24 | 43 | |
80127a39 | 44 | int __percpu_down_read(struct percpu_rw_semaphore *sem, int try) |
a1fd3e24 | 45 | { |
80127a39 PZ |
46 | /* |
47 | * Due to having preemption disabled the decrement happens on | |
48 | * the same CPU as the increment, avoiding the | |
49 | * increment-on-one-CPU-and-decrement-on-another problem. | |
50 | * | |
51 | * If the reader misses the writer's assignment of readers_block, then | |
52 | * the writer is guaranteed to see the reader's increment. | |
53 | * | |
54 | * Conversely, any readers that increment their sem->read_count after | |
55 | * the writer looks are guaranteed to see the readers_block value, | |
56 | * which in turn means that they are guaranteed to immediately | |
57 | * decrement their sem->read_count, so that it doesn't matter that the | |
58 | * writer missed them. | |
59 | */ | |
a1fd3e24 | 60 | |
80127a39 | 61 | smp_mb(); /* A matches D */ |
a1fd3e24 | 62 | |
80127a39 PZ |
63 | /* |
64 | * If !readers_block the critical section starts here, matched by the | |
65 | * release in percpu_up_write(). | |
66 | */ | |
67 | if (likely(!smp_load_acquire(&sem->readers_block))) | |
68 | return 1; | |
a1fd3e24 | 69 | |
80127a39 PZ |
70 | /* |
71 | * Per the above comment; we still have preemption disabled and | |
72 | * will thus decrement on the same CPU as we incremented. | |
73 | */ | |
74 | __percpu_up_read(sem); | |
cc5f730b | 75 | |
80127a39 PZ |
76 | if (try) |
77 | return 0; | |
a1fd3e24 | 78 | |
80127a39 PZ |
79 | /* |
80 | * We either call schedule() in the wait, or we'll fall through | |
81 | * and reschedule on the preempt_enable() in percpu_down_read(). | |
82 | */ | |
83 | preempt_enable_no_resched(); | |
a1fd3e24 | 84 | |
80127a39 PZ |
85 | /* |
86 | * Avoid lockdep for the down/up_read() we already have them. | |
87 | */ | |
88 | __down_read(&sem->rw_sem); | |
89 | this_cpu_inc(*sem->read_count); | |
90 | __up_read(&sem->rw_sem); | |
91 | ||
92 | preempt_disable(); | |
9287f692 ON |
93 | return 1; |
94 | } | |
80127a39 | 95 | EXPORT_SYMBOL_GPL(__percpu_down_read); |
9287f692 | 96 | |
80127a39 | 97 | void __percpu_up_read(struct percpu_rw_semaphore *sem) |
a1fd3e24 | 98 | { |
80127a39 PZ |
99 | smp_mb(); /* B matches C */ |
100 | /* | |
101 | * In other words, if they see our decrement (presumably to aggregate | |
102 | * zero, as that is the only time it matters) they will also see our | |
103 | * critical section. | |
104 | */ | |
105 | __this_cpu_dec(*sem->read_count); | |
a1fd3e24 | 106 | |
80127a39 | 107 | /* Prod writer to recheck readers_active */ |
52b94129 | 108 | rcuwait_wake_up(&sem->writer); |
a1fd3e24 | 109 | } |
80127a39 PZ |
110 | EXPORT_SYMBOL_GPL(__percpu_up_read); |
111 | ||
112 | #define per_cpu_sum(var) \ | |
113 | ({ \ | |
114 | typeof(var) __sum = 0; \ | |
115 | int cpu; \ | |
116 | compiletime_assert_atomic_type(__sum); \ | |
117 | for_each_possible_cpu(cpu) \ | |
118 | __sum += per_cpu(var, cpu); \ | |
119 | __sum; \ | |
120 | }) | |
a1fd3e24 | 121 | |
80127a39 PZ |
122 | /* |
123 | * Return true if the modular sum of the sem->read_count per-CPU variable is | |
124 | * zero. If this sum is zero, then it is stable due to the fact that if any | |
125 | * newly arriving readers increment a given counter, they will immediately | |
126 | * decrement that same counter. | |
127 | */ | |
128 | static bool readers_active_check(struct percpu_rw_semaphore *sem) | |
a1fd3e24 | 129 | { |
80127a39 PZ |
130 | if (per_cpu_sum(*sem->read_count) != 0) |
131 | return false; | |
132 | ||
133 | /* | |
134 | * If we observed the decrement; ensure we see the entire critical | |
135 | * section. | |
136 | */ | |
a1fd3e24 | 137 | |
80127a39 | 138 | smp_mb(); /* C matches B */ |
a1fd3e24 | 139 | |
80127a39 | 140 | return true; |
a1fd3e24 ON |
141 | } |
142 | ||
80127a39 | 143 | void percpu_down_write(struct percpu_rw_semaphore *sem) |
a1fd3e24 | 144 | { |
80127a39 PZ |
145 | /* Notify readers to take the slow path. */ |
146 | rcu_sync_enter(&sem->rss); | |
147 | ||
148 | down_write(&sem->rw_sem); | |
149 | ||
a1fd3e24 | 150 | /* |
80127a39 PZ |
151 | * Notify new readers to block; up until now, and thus throughout the |
152 | * longish rcu_sync_enter() above, new readers could still come in. | |
a1fd3e24 | 153 | */ |
80127a39 | 154 | WRITE_ONCE(sem->readers_block, 1); |
a1fd3e24 | 155 | |
80127a39 | 156 | smp_mb(); /* D matches A */ |
9390ef0c | 157 | |
80127a39 PZ |
158 | /* |
159 | * If they don't see our writer of readers_block, then we are | |
160 | * guaranteed to see their sem->read_count increment, and therefore | |
161 | * will wait for them. | |
162 | */ | |
a1fd3e24 | 163 | |
80127a39 | 164 | /* Wait for all now active readers to complete. */ |
52b94129 | 165 | rcuwait_wait_event(&sem->writer, readers_active_check(sem)); |
a1fd3e24 | 166 | } |
302707fd | 167 | EXPORT_SYMBOL_GPL(percpu_down_write); |
a1fd3e24 | 168 | |
80127a39 | 169 | void percpu_up_write(struct percpu_rw_semaphore *sem) |
a1fd3e24 | 170 | { |
a1fd3e24 | 171 | /* |
80127a39 PZ |
172 | * Signal the writer is done, no fast path yet. |
173 | * | |
174 | * One reason that we cannot just immediately flip to readers_fast is | |
175 | * that new readers might fail to see the results of this writer's | |
176 | * critical section. | |
177 | * | |
178 | * Therefore we force it through the slow path which guarantees an | |
179 | * acquire and thereby guarantees the critical section's consistency. | |
180 | */ | |
181 | smp_store_release(&sem->readers_block, 0); | |
182 | ||
183 | /* | |
184 | * Release the write lock, this will allow readers back in the game. | |
185 | */ | |
186 | up_write(&sem->rw_sem); | |
187 | ||
188 | /* | |
189 | * Once this completes (at least one RCU-sched grace period hence) the | |
190 | * reader fast path will be available again. Safe to use outside the | |
191 | * exclusive write lock because its counting. | |
a1fd3e24 | 192 | */ |
80127a39 | 193 | rcu_sync_exit(&sem->rss); |
a1fd3e24 | 194 | } |
302707fd | 195 | EXPORT_SYMBOL_GPL(percpu_up_write); |