]>
Commit | Line | Data |
---|---|---|
d15851ec | 1 | /* Internal macros for atomic operations for GNU C Library. |
dff8da6b | 2 | Copyright (C) 2002-2024 Free Software Foundation, Inc. |
76a50749 | 3 | This file is part of the GNU C Library. |
76a50749 UD |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 | 16 | License along with the GNU C Library; if not, see |
5a82c748 | 17 | <https://www.gnu.org/licenses/>. */ |
76a50749 UD |
18 | |
19 | #ifndef _ATOMIC_H | |
20 | #define _ATOMIC_H 1 | |
21 | ||
11bf311e UD |
22 | /* This header defines three types of macros: |
23 | ||
24 | - atomic arithmetic and logic operation on memory. They all | |
25 | have the prefix "atomic_". | |
26 | ||
27 | - conditionally atomic operations of the same kinds. These | |
28 | always behave identical but can be faster when atomicity | |
29 | is not really needed since only one thread has access to | |
30 | the memory location. In that case the code is slower in | |
31 | the multi-thread case. The interfaces have the prefix | |
32 | "catomic_". | |
33 | ||
eaea92f1 | 34 | - support functions like barriers. They also have the prefix |
11bf311e UD |
35 | "atomic_". |
36 | ||
37 | Architectures must provide a few lowlevel macros (the compare | |
38 | and exchange definitions). All others are optional. They | |
39 | should only be provided if the architecture has specific | |
40 | support for the operation. | |
41 | ||
42 | As <atomic.h> macros are usually heavily nested and often use local | |
43 | variables to make sure side-effects are evaluated properly, use for | |
44 | macro local variables a per-macro unique prefix. This file uses | |
45 | __atgN_ prefix where N is different in each macro. */ | |
46 | ||
76a50749 UD |
47 | #include <stdlib.h> |
48 | ||
de071d19 | 49 | #include <atomic-machine.h> |
76a50749 | 50 | |
fe4610b3 RM |
51 | /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the |
52 | bit width of *MEM. The calling macro puts parens around MEM | |
53 | and following args. */ | |
54 | #define __atomic_val_bysize(pre, post, mem, ...) \ | |
55 | ({ \ | |
12d2dd70 | 56 | __typeof ((__typeof (*(mem))) *(mem)) __atg1_result; \ |
fe4610b3 | 57 | if (sizeof (*mem) == 1) \ |
11bf311e | 58 | __atg1_result = pre##_8_##post (mem, __VA_ARGS__); \ |
fe4610b3 | 59 | else if (sizeof (*mem) == 2) \ |
11bf311e | 60 | __atg1_result = pre##_16_##post (mem, __VA_ARGS__); \ |
fe4610b3 | 61 | else if (sizeof (*mem) == 4) \ |
11bf311e | 62 | __atg1_result = pre##_32_##post (mem, __VA_ARGS__); \ |
fe4610b3 | 63 | else if (sizeof (*mem) == 8) \ |
11bf311e | 64 | __atg1_result = pre##_64_##post (mem, __VA_ARGS__); \ |
fe4610b3 RM |
65 | else \ |
66 | abort (); \ | |
11bf311e | 67 | __atg1_result; \ |
fe4610b3 RM |
68 | }) |
69 | #define __atomic_bool_bysize(pre, post, mem, ...) \ | |
70 | ({ \ | |
11bf311e | 71 | int __atg2_result; \ |
fe4610b3 | 72 | if (sizeof (*mem) == 1) \ |
11bf311e | 73 | __atg2_result = pre##_8_##post (mem, __VA_ARGS__); \ |
fe4610b3 | 74 | else if (sizeof (*mem) == 2) \ |
11bf311e | 75 | __atg2_result = pre##_16_##post (mem, __VA_ARGS__); \ |
fe4610b3 | 76 | else if (sizeof (*mem) == 4) \ |
11bf311e | 77 | __atg2_result = pre##_32_##post (mem, __VA_ARGS__); \ |
fe4610b3 | 78 | else if (sizeof (*mem) == 8) \ |
11bf311e | 79 | __atg2_result = pre##_64_##post (mem, __VA_ARGS__); \ |
fe4610b3 RM |
80 | else \ |
81 | abort (); \ | |
11bf311e | 82 | __atg2_result; \ |
fe4610b3 RM |
83 | }) |
84 | ||
76a50749 | 85 | |
d15851ec RM |
86 | /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. |
87 | Return the old *MEM value. */ | |
88 | #if !defined atomic_compare_and_exchange_val_acq \ | |
89 | && defined __arch_compare_and_exchange_val_32_acq | |
18627f61 | 90 | # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \ |
a1b5424f UD |
91 | __atomic_val_bysize (__arch_compare_and_exchange_val,acq, \ |
92 | mem, newval, oldval) | |
76a50749 UD |
93 | #endif |
94 | ||
95 | ||
3b1b533b JJ |
96 | #ifndef catomic_compare_and_exchange_val_acq |
97 | # ifdef __arch_c_compare_and_exchange_val_32_acq | |
98 | # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ | |
11bf311e UD |
99 | __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \ |
100 | mem, newval, oldval) | |
3b1b533b JJ |
101 | # else |
102 | # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ | |
11bf311e | 103 | atomic_compare_and_exchange_val_acq (mem, newval, oldval) |
3b1b533b | 104 | # endif |
11bf311e UD |
105 | #endif |
106 | ||
107 | ||
bea0ac1d JJ |
108 | #ifndef catomic_compare_and_exchange_val_rel |
109 | # ifndef atomic_compare_and_exchange_val_rel | |
110 | # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ | |
111 | catomic_compare_and_exchange_val_acq (mem, newval, oldval) | |
112 | # else | |
113 | # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ | |
114 | atomic_compare_and_exchange_val_rel (mem, newval, oldval) | |
115 | # endif | |
18627f61 UD |
116 | #endif |
117 | ||
118 | ||
bea0ac1d JJ |
119 | #ifndef atomic_compare_and_exchange_val_rel |
120 | # define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \ | |
11bf311e UD |
121 | atomic_compare_and_exchange_val_acq (mem, newval, oldval) |
122 | #endif | |
123 | ||
124 | ||
d15851ec RM |
125 | /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. |
126 | Return zero if *MEM was changed or non-zero if no exchange happened. */ | |
18627f61 UD |
127 | #ifndef atomic_compare_and_exchange_bool_acq |
128 | # ifdef __arch_compare_and_exchange_bool_32_acq | |
18627f61 | 129 | # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ |
a1b5424f UD |
130 | __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq, \ |
131 | mem, newval, oldval) | |
3b1b533b JJ |
132 | # else |
133 | # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ | |
d15851ec RM |
134 | ({ /* Cannot use __oldval here, because macros later in this file might \ |
135 | call this macro with __oldval argument. */ \ | |
11bf311e UD |
136 | __typeof (oldval) __atg3_old = (oldval); \ |
137 | atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old) \ | |
138 | != __atg3_old; \ | |
139 | }) | |
140 | # endif | |
141 | #endif | |
142 | ||
143 | ||
144 | #ifndef catomic_compare_and_exchange_bool_acq | |
145 | # ifdef __arch_c_compare_and_exchange_bool_32_acq | |
146 | # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ | |
147 | __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq, \ | |
148 | mem, newval, oldval) | |
3b1b533b JJ |
149 | # else |
150 | # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ | |
11bf311e UD |
151 | ({ /* Cannot use __oldval here, because macros later in this file might \ |
152 | call this macro with __oldval argument. */ \ | |
153 | __typeof (oldval) __atg4_old = (oldval); \ | |
154 | catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \ | |
155 | != __atg4_old; \ | |
8099361e UD |
156 | }) |
157 | # endif | |
158 | #endif | |
159 | ||
160 | ||
5132931e | 161 | /* Store NEWVALUE in *MEM and return the old value. */ |
949ec764 UD |
162 | #ifndef atomic_exchange_acq |
163 | # define atomic_exchange_acq(mem, newvalue) \ | |
12d2dd70 | 164 | ({ __typeof ((__typeof (*(mem))) *(mem)) __atg5_oldval; \ |
11bf311e | 165 | __typeof (mem) __atg5_memp = (mem); \ |
12d2dd70 | 166 | __typeof ((__typeof (*(mem))) *(mem)) __atg5_value = (newvalue); \ |
5132931e UD |
167 | \ |
168 | do \ | |
11bf311e UD |
169 | __atg5_oldval = *__atg5_memp; \ |
170 | while (__builtin_expect \ | |
171 | (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \ | |
172 | __atg5_oldval), 0)); \ | |
5132931e | 173 | \ |
11bf311e | 174 | __atg5_oldval; }) |
5132931e UD |
175 | #endif |
176 | ||
949ec764 UD |
177 | #ifndef atomic_exchange_rel |
178 | # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue) | |
179 | #endif | |
180 | ||
5132931e UD |
181 | |
182 | /* Add VALUE to *MEM and return the old value of *MEM. */ | |
51a9ba86 MK |
183 | #ifndef atomic_exchange_and_add_acq |
184 | # ifdef atomic_exchange_and_add | |
185 | # define atomic_exchange_and_add_acq(mem, value) \ | |
186 | atomic_exchange_and_add (mem, value) | |
187 | # else | |
188 | # define atomic_exchange_and_add_acq(mem, value) \ | |
11bf311e UD |
189 | ({ __typeof (*(mem)) __atg6_oldval; \ |
190 | __typeof (mem) __atg6_memp = (mem); \ | |
191 | __typeof (*(mem)) __atg6_value = (value); \ | |
76a50749 UD |
192 | \ |
193 | do \ | |
11bf311e UD |
194 | __atg6_oldval = *__atg6_memp; \ |
195 | while (__builtin_expect \ | |
196 | (atomic_compare_and_exchange_bool_acq (__atg6_memp, \ | |
197 | __atg6_oldval \ | |
198 | + __atg6_value, \ | |
199 | __atg6_oldval), 0)); \ | |
76a50749 | 200 | \ |
11bf311e | 201 | __atg6_oldval; }) |
51a9ba86 | 202 | # endif |
76a50749 UD |
203 | #endif |
204 | ||
51a9ba86 MK |
205 | #ifndef atomic_exchange_and_add_rel |
206 | # define atomic_exchange_and_add_rel(mem, value) \ | |
207 | atomic_exchange_and_add_acq(mem, value) | |
208 | #endif | |
209 | ||
210 | #ifndef atomic_exchange_and_add | |
211 | # define atomic_exchange_and_add(mem, value) \ | |
212 | atomic_exchange_and_add_acq(mem, value) | |
213 | #endif | |
76a50749 | 214 | |
11bf311e UD |
215 | #ifndef catomic_exchange_and_add |
216 | # define catomic_exchange_and_add(mem, value) \ | |
217 | ({ __typeof (*(mem)) __atg7_oldv; \ | |
218 | __typeof (mem) __atg7_memp = (mem); \ | |
219 | __typeof (*(mem)) __atg7_value = (value); \ | |
220 | \ | |
221 | do \ | |
222 | __atg7_oldv = *__atg7_memp; \ | |
223 | while (__builtin_expect \ | |
224 | (catomic_compare_and_exchange_bool_acq (__atg7_memp, \ | |
225 | __atg7_oldv \ | |
226 | + __atg7_value, \ | |
227 | __atg7_oldv), 0)); \ | |
228 | \ | |
229 | __atg7_oldv; }) | |
230 | #endif | |
231 | ||
d6c7294e UD |
232 | |
233 | #ifndef atomic_max | |
234 | # define atomic_max(mem, value) \ | |
235 | do { \ | |
11bf311e UD |
236 | __typeof (*(mem)) __atg8_oldval; \ |
237 | __typeof (mem) __atg8_memp = (mem); \ | |
238 | __typeof (*(mem)) __atg8_value = (value); \ | |
239 | do { \ | |
240 | __atg8_oldval = *__atg8_memp; \ | |
241 | if (__atg8_oldval >= __atg8_value) \ | |
242 | break; \ | |
243 | } while (__builtin_expect \ | |
244 | (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\ | |
245 | __atg8_oldval), 0)); \ | |
246 | } while (0) | |
247 | #endif | |
248 | ||
249 | ||
250 | #ifndef catomic_max | |
251 | # define catomic_max(mem, value) \ | |
252 | do { \ | |
253 | __typeof (*(mem)) __atg9_oldv; \ | |
254 | __typeof (mem) __atg9_memp = (mem); \ | |
255 | __typeof (*(mem)) __atg9_value = (value); \ | |
8099361e | 256 | do { \ |
11bf311e UD |
257 | __atg9_oldv = *__atg9_memp; \ |
258 | if (__atg9_oldv >= __atg9_value) \ | |
8099361e | 259 | break; \ |
11bf311e UD |
260 | } while (__builtin_expect \ |
261 | (catomic_compare_and_exchange_bool_acq (__atg9_memp, \ | |
262 | __atg9_value, \ | |
263 | __atg9_oldv), 0)); \ | |
8099361e UD |
264 | } while (0) |
265 | #endif | |
266 | ||
11bf311e | 267 | |
d6c7294e UD |
268 | #ifndef atomic_min |
269 | # define atomic_min(mem, value) \ | |
270 | do { \ | |
11bf311e UD |
271 | __typeof (*(mem)) __atg10_oldval; \ |
272 | __typeof (mem) __atg10_memp = (mem); \ | |
273 | __typeof (*(mem)) __atg10_value = (value); \ | |
d6c7294e | 274 | do { \ |
11bf311e UD |
275 | __atg10_oldval = *__atg10_memp; \ |
276 | if (__atg10_oldval <= __atg10_value) \ | |
d6c7294e | 277 | break; \ |
11bf311e UD |
278 | } while (__builtin_expect \ |
279 | (atomic_compare_and_exchange_bool_acq (__atg10_memp, \ | |
280 | __atg10_value, \ | |
281 | __atg10_oldval), 0)); \ | |
d6c7294e UD |
282 | } while (0) |
283 | #endif | |
284 | ||
11bf311e | 285 | |
76a50749 | 286 | #ifndef atomic_add |
fe4610b3 | 287 | # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value)) |
76a50749 UD |
288 | #endif |
289 | ||
290 | ||
11bf311e UD |
291 | #ifndef catomic_add |
292 | # define catomic_add(mem, value) \ | |
293 | (void) catomic_exchange_and_add ((mem), (value)) | |
294 | #endif | |
295 | ||
296 | ||
76a50749 | 297 | #ifndef atomic_increment |
fe4610b3 | 298 | # define atomic_increment(mem) atomic_add ((mem), 1) |
76a50749 UD |
299 | #endif |
300 | ||
301 | ||
11bf311e UD |
302 | #ifndef catomic_increment |
303 | # define catomic_increment(mem) catomic_add ((mem), 1) | |
304 | #endif | |
305 | ||
306 | ||
f377d022 UD |
307 | #ifndef atomic_increment_val |
308 | # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1) | |
309 | #endif | |
310 | ||
311 | ||
11bf311e UD |
312 | #ifndef catomic_increment_val |
313 | # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1) | |
314 | #endif | |
315 | ||
316 | ||
fe4610b3 | 317 | /* Add one to *MEM and return true iff it's now zero. */ |
ec609a8e UD |
318 | #ifndef atomic_increment_and_test |
319 | # define atomic_increment_and_test(mem) \ | |
fe4610b3 | 320 | (atomic_exchange_and_add ((mem), 1) + 1 == 0) |
ec609a8e UD |
321 | #endif |
322 | ||
323 | ||
76a50749 | 324 | #ifndef atomic_decrement |
fe4610b3 | 325 | # define atomic_decrement(mem) atomic_add ((mem), -1) |
76a50749 UD |
326 | #endif |
327 | ||
328 | ||
11bf311e UD |
329 | #ifndef catomic_decrement |
330 | # define catomic_decrement(mem) catomic_add ((mem), -1) | |
331 | #endif | |
332 | ||
333 | ||
f377d022 UD |
334 | #ifndef atomic_decrement_val |
335 | # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1) | |
336 | #endif | |
337 | ||
338 | ||
11bf311e UD |
339 | #ifndef catomic_decrement_val |
340 | # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1) | |
341 | #endif | |
342 | ||
343 | ||
d15851ec | 344 | /* Subtract 1 from *MEM and return true iff it's now zero. */ |
ec609a8e UD |
345 | #ifndef atomic_decrement_and_test |
346 | # define atomic_decrement_and_test(mem) \ | |
fe4610b3 | 347 | (atomic_exchange_and_add ((mem), -1) == 1) |
ec609a8e UD |
348 | #endif |
349 | ||
350 | ||
5132931e | 351 | /* Decrement *MEM if it is > 0, and return the old value. */ |
2ff16e04 | 352 | #ifndef atomic_decrement_if_positive |
972209dd | 353 | # define atomic_decrement_if_positive(mem) \ |
11bf311e UD |
354 | ({ __typeof (*(mem)) __atg11_oldval; \ |
355 | __typeof (mem) __atg11_memp = (mem); \ | |
972209dd | 356 | \ |
5132931e UD |
357 | do \ |
358 | { \ | |
11bf311e | 359 | __atg11_oldval = *__atg11_memp; \ |
a1ffb40e | 360 | if (__glibc_unlikely (__atg11_oldval <= 0)) \ |
5132931e | 361 | break; \ |
5132931e | 362 | } \ |
11bf311e UD |
363 | while (__builtin_expect \ |
364 | (atomic_compare_and_exchange_bool_acq (__atg11_memp, \ | |
365 | __atg11_oldval - 1, \ | |
366 | __atg11_oldval), 0)); \ | |
367 | __atg11_oldval; }) | |
5132931e UD |
368 | #endif |
369 | ||
370 | ||
5cd09cd6 | 371 | #ifndef atomic_add_negative |
859e708f | 372 | # define atomic_add_negative(mem, value) \ |
11bf311e UD |
373 | ({ __typeof (value) __atg12_value = (value); \ |
374 | atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; }) | |
ec609a8e UD |
375 | #endif |
376 | ||
377 | ||
378 | #ifndef atomic_add_zero | |
859e708f | 379 | # define atomic_add_zero(mem, value) \ |
11bf311e UD |
380 | ({ __typeof (value) __atg13_value = (value); \ |
381 | atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; }) | |
ec609a8e UD |
382 | #endif |
383 | ||
384 | ||
0289bef9 UD |
385 | #ifndef atomic_bit_set |
386 | # define atomic_bit_set(mem, bit) \ | |
a1b5424f | 387 | (void) atomic_bit_test_set(mem, bit) |
ec609a8e UD |
388 | #endif |
389 | ||
390 | ||
391 | #ifndef atomic_bit_test_set | |
392 | # define atomic_bit_test_set(mem, bit) \ | |
11bf311e UD |
393 | ({ __typeof (*(mem)) __atg14_old; \ |
394 | __typeof (mem) __atg14_memp = (mem); \ | |
395 | __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit)); \ | |
ec609a8e UD |
396 | \ |
397 | do \ | |
11bf311e UD |
398 | __atg14_old = (*__atg14_memp); \ |
399 | while (__builtin_expect \ | |
400 | (atomic_compare_and_exchange_bool_acq (__atg14_memp, \ | |
401 | __atg14_old | __atg14_mask,\ | |
402 | __atg14_old), 0)); \ | |
76a50749 | 403 | \ |
11bf311e | 404 | __atg14_old & __atg14_mask; }) |
76a50749 UD |
405 | #endif |
406 | ||
11bf311e | 407 | /* Atomically *mem &= mask. */ |
f1740bc4 UD |
408 | #ifndef atomic_and |
409 | # define atomic_and(mem, mask) \ | |
11bf311e UD |
410 | do { \ |
411 | __typeof (*(mem)) __atg15_old; \ | |
412 | __typeof (mem) __atg15_memp = (mem); \ | |
413 | __typeof (*(mem)) __atg15_mask = (mask); \ | |
414 | \ | |
415 | do \ | |
416 | __atg15_old = (*__atg15_memp); \ | |
417 | while (__builtin_expect \ | |
418 | (atomic_compare_and_exchange_bool_acq (__atg15_memp, \ | |
419 | __atg15_old & __atg15_mask, \ | |
420 | __atg15_old), 0)); \ | |
421 | } while (0) | |
422 | #endif | |
423 | ||
6c03cd11 UD |
424 | #ifndef catomic_and |
425 | # define catomic_and(mem, mask) \ | |
426 | do { \ | |
427 | __typeof (*(mem)) __atg20_old; \ | |
428 | __typeof (mem) __atg20_memp = (mem); \ | |
429 | __typeof (*(mem)) __atg20_mask = (mask); \ | |
430 | \ | |
431 | do \ | |
432 | __atg20_old = (*__atg20_memp); \ | |
433 | while (__builtin_expect \ | |
434 | (catomic_compare_and_exchange_bool_acq (__atg20_memp, \ | |
435 | __atg20_old & __atg20_mask,\ | |
436 | __atg20_old), 0)); \ | |
437 | } while (0) | |
438 | #endif | |
439 | ||
11bf311e UD |
440 | /* Atomically *mem &= mask and return the old value of *mem. */ |
441 | #ifndef atomic_and_val | |
442 | # define atomic_and_val(mem, mask) \ | |
443 | ({ __typeof (*(mem)) __atg16_old; \ | |
444 | __typeof (mem) __atg16_memp = (mem); \ | |
445 | __typeof (*(mem)) __atg16_mask = (mask); \ | |
f1740bc4 UD |
446 | \ |
447 | do \ | |
11bf311e UD |
448 | __atg16_old = (*__atg16_memp); \ |
449 | while (__builtin_expect \ | |
450 | (atomic_compare_and_exchange_bool_acq (__atg16_memp, \ | |
451 | __atg16_old & __atg16_mask,\ | |
452 | __atg16_old), 0)); \ | |
f1740bc4 | 453 | \ |
11bf311e | 454 | __atg16_old; }) |
f1740bc4 UD |
455 | #endif |
456 | ||
457 | /* Atomically *mem |= mask and return the old value of *mem. */ | |
458 | #ifndef atomic_or | |
459 | # define atomic_or(mem, mask) \ | |
11bf311e UD |
460 | do { \ |
461 | __typeof (*(mem)) __atg17_old; \ | |
462 | __typeof (mem) __atg17_memp = (mem); \ | |
463 | __typeof (*(mem)) __atg17_mask = (mask); \ | |
464 | \ | |
465 | do \ | |
466 | __atg17_old = (*__atg17_memp); \ | |
467 | while (__builtin_expect \ | |
468 | (atomic_compare_and_exchange_bool_acq (__atg17_memp, \ | |
469 | __atg17_old | __atg17_mask, \ | |
470 | __atg17_old), 0)); \ | |
471 | } while (0) | |
472 | #endif | |
473 | ||
474 | #ifndef catomic_or | |
475 | # define catomic_or(mem, mask) \ | |
476 | do { \ | |
477 | __typeof (*(mem)) __atg18_old; \ | |
478 | __typeof (mem) __atg18_memp = (mem); \ | |
479 | __typeof (*(mem)) __atg18_mask = (mask); \ | |
480 | \ | |
481 | do \ | |
482 | __atg18_old = (*__atg18_memp); \ | |
483 | while (__builtin_expect \ | |
484 | (catomic_compare_and_exchange_bool_acq (__atg18_memp, \ | |
485 | __atg18_old | __atg18_mask,\ | |
486 | __atg18_old), 0)); \ | |
487 | } while (0) | |
488 | #endif | |
489 | ||
490 | /* Atomically *mem |= mask and return the old value of *mem. */ | |
491 | #ifndef atomic_or_val | |
492 | # define atomic_or_val(mem, mask) \ | |
493 | ({ __typeof (*(mem)) __atg19_old; \ | |
494 | __typeof (mem) __atg19_memp = (mem); \ | |
495 | __typeof (*(mem)) __atg19_mask = (mask); \ | |
f1740bc4 UD |
496 | \ |
497 | do \ | |
11bf311e UD |
498 | __atg19_old = (*__atg19_memp); \ |
499 | while (__builtin_expect \ | |
500 | (atomic_compare_and_exchange_bool_acq (__atg19_memp, \ | |
501 | __atg19_old | __atg19_mask,\ | |
502 | __atg19_old), 0)); \ | |
f1740bc4 | 503 | \ |
11bf311e | 504 | __atg19_old; }) |
f1740bc4 | 505 | #endif |
76a50749 UD |
506 | |
507 | #ifndef atomic_full_barrier | |
733f25e6 | 508 | # define atomic_full_barrier() __asm ("" ::: "memory") |
76a50749 UD |
509 | #endif |
510 | ||
511 | ||
512 | #ifndef atomic_read_barrier | |
a1b5424f | 513 | # define atomic_read_barrier() atomic_full_barrier () |
76a50749 UD |
514 | #endif |
515 | ||
516 | ||
517 | #ifndef atomic_write_barrier | |
a1b5424f | 518 | # define atomic_write_barrier() atomic_full_barrier () |
76a50749 UD |
519 | #endif |
520 | ||
f377d022 | 521 | |
fc242bef UD |
522 | #ifndef atomic_forced_read |
523 | # define atomic_forced_read(x) \ | |
524 | ({ __typeof (x) __x; __asm ("" : "=r" (__x) : "0" (x)); __x; }) | |
525 | #endif | |
526 | ||
ff871426 TR |
527 | /* This is equal to 1 iff the architecture supports 64b atomic operations. */ |
528 | #ifndef __HAVE_64B_ATOMICS | |
529 | #error Unable to determine if 64-bit atomics are present. | |
530 | #endif | |
531 | ||
532 | /* The following functions are a subset of the atomic operations provided by | |
533 | C11. Usually, a function named atomic_OP_MO(args) is equivalent to C11's | |
534 | atomic_OP_explicit(args, memory_order_MO); exceptions noted below. */ | |
535 | ||
536 | /* Each arch can request to use compiler built-ins for C11 atomics. If it | |
537 | does, all atomics will be based on these. */ | |
538 | #if USE_ATOMIC_COMPILER_BUILTINS | |
539 | ||
540 | /* We require 32b atomic operations; some archs also support 64b atomic | |
541 | operations. */ | |
542 | void __atomic_link_error (void); | |
543 | # if __HAVE_64B_ATOMICS == 1 | |
544 | # define __atomic_check_size(mem) \ | |
545 | if ((sizeof (*mem) != 4) && (sizeof (*mem) != 8)) \ | |
546 | __atomic_link_error (); | |
547 | # else | |
548 | # define __atomic_check_size(mem) \ | |
549 | if (sizeof (*mem) != 4) \ | |
550 | __atomic_link_error (); | |
551 | # endif | |
ca6e601a TR |
552 | /* We additionally provide 8b and 16b atomic loads and stores; we do not yet |
553 | need other atomic operations of such sizes, and restricting the support to | |
554 | loads and stores makes this easier for archs that do not have native | |
555 | support for atomic operations to less-than-word-sized data. */ | |
556 | # if __HAVE_64B_ATOMICS == 1 | |
557 | # define __atomic_check_size_ls(mem) \ | |
558 | if ((sizeof (*mem) != 1) && (sizeof (*mem) != 2) && (sizeof (*mem) != 4) \ | |
559 | && (sizeof (*mem) != 8)) \ | |
560 | __atomic_link_error (); | |
561 | # else | |
562 | # define __atomic_check_size_ls(mem) \ | |
563 | if ((sizeof (*mem) != 1) && (sizeof (*mem) != 2) && sizeof (*mem) != 4) \ | |
564 | __atomic_link_error (); | |
565 | # endif | |
ff871426 TR |
566 | |
567 | # define atomic_thread_fence_acquire() \ | |
568 | __atomic_thread_fence (__ATOMIC_ACQUIRE) | |
569 | # define atomic_thread_fence_release() \ | |
570 | __atomic_thread_fence (__ATOMIC_RELEASE) | |
571 | # define atomic_thread_fence_seq_cst() \ | |
572 | __atomic_thread_fence (__ATOMIC_SEQ_CST) | |
573 | ||
574 | # define atomic_load_relaxed(mem) \ | |
ca6e601a TR |
575 | ({ __atomic_check_size_ls((mem)); \ |
576 | __atomic_load_n ((mem), __ATOMIC_RELAXED); }) | |
ff871426 | 577 | # define atomic_load_acquire(mem) \ |
ca6e601a TR |
578 | ({ __atomic_check_size_ls((mem)); \ |
579 | __atomic_load_n ((mem), __ATOMIC_ACQUIRE); }) | |
ff871426 TR |
580 | |
581 | # define atomic_store_relaxed(mem, val) \ | |
582 | do { \ | |
ca6e601a | 583 | __atomic_check_size_ls((mem)); \ |
ff871426 TR |
584 | __atomic_store_n ((mem), (val), __ATOMIC_RELAXED); \ |
585 | } while (0) | |
586 | # define atomic_store_release(mem, val) \ | |
587 | do { \ | |
ca6e601a | 588 | __atomic_check_size_ls((mem)); \ |
ff871426 TR |
589 | __atomic_store_n ((mem), (val), __ATOMIC_RELEASE); \ |
590 | } while (0) | |
591 | ||
592 | /* On failure, this CAS has memory_order_relaxed semantics. */ | |
593 | # define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \ | |
594 | ({ __atomic_check_size((mem)); \ | |
595 | __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ | |
596 | __ATOMIC_RELAXED, __ATOMIC_RELAXED); }) | |
597 | # define atomic_compare_exchange_weak_acquire(mem, expected, desired) \ | |
598 | ({ __atomic_check_size((mem)); \ | |
599 | __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ | |
600 | __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); }) | |
601 | # define atomic_compare_exchange_weak_release(mem, expected, desired) \ | |
602 | ({ __atomic_check_size((mem)); \ | |
603 | __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ | |
604 | __ATOMIC_RELEASE, __ATOMIC_RELAXED); }) | |
605 | ||
f0e3925b TR |
606 | # define atomic_exchange_relaxed(mem, desired) \ |
607 | ({ __atomic_check_size((mem)); \ | |
608 | __atomic_exchange_n ((mem), (desired), __ATOMIC_RELAXED); }) | |
ff871426 TR |
609 | # define atomic_exchange_acquire(mem, desired) \ |
610 | ({ __atomic_check_size((mem)); \ | |
611 | __atomic_exchange_n ((mem), (desired), __ATOMIC_ACQUIRE); }) | |
612 | # define atomic_exchange_release(mem, desired) \ | |
613 | ({ __atomic_check_size((mem)); \ | |
614 | __atomic_exchange_n ((mem), (desired), __ATOMIC_RELEASE); }) | |
615 | ||
616 | # define atomic_fetch_add_relaxed(mem, operand) \ | |
617 | ({ __atomic_check_size((mem)); \ | |
618 | __atomic_fetch_add ((mem), (operand), __ATOMIC_RELAXED); }) | |
619 | # define atomic_fetch_add_acquire(mem, operand) \ | |
620 | ({ __atomic_check_size((mem)); \ | |
621 | __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQUIRE); }) | |
622 | # define atomic_fetch_add_release(mem, operand) \ | |
623 | ({ __atomic_check_size((mem)); \ | |
624 | __atomic_fetch_add ((mem), (operand), __ATOMIC_RELEASE); }) | |
625 | # define atomic_fetch_add_acq_rel(mem, operand) \ | |
626 | ({ __atomic_check_size((mem)); \ | |
627 | __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQ_REL); }) | |
628 | ||
13cb8f76 TR |
629 | # define atomic_fetch_and_relaxed(mem, operand) \ |
630 | ({ __atomic_check_size((mem)); \ | |
631 | __atomic_fetch_and ((mem), (operand), __ATOMIC_RELAXED); }) | |
ff871426 TR |
632 | # define atomic_fetch_and_acquire(mem, operand) \ |
633 | ({ __atomic_check_size((mem)); \ | |
634 | __atomic_fetch_and ((mem), (operand), __ATOMIC_ACQUIRE); }) | |
13cb8f76 TR |
635 | # define atomic_fetch_and_release(mem, operand) \ |
636 | ({ __atomic_check_size((mem)); \ | |
637 | __atomic_fetch_and ((mem), (operand), __ATOMIC_RELEASE); }) | |
ff871426 TR |
638 | |
639 | # define atomic_fetch_or_relaxed(mem, operand) \ | |
640 | ({ __atomic_check_size((mem)); \ | |
641 | __atomic_fetch_or ((mem), (operand), __ATOMIC_RELAXED); }) | |
642 | # define atomic_fetch_or_acquire(mem, operand) \ | |
643 | ({ __atomic_check_size((mem)); \ | |
644 | __atomic_fetch_or ((mem), (operand), __ATOMIC_ACQUIRE); }) | |
13cb8f76 TR |
645 | # define atomic_fetch_or_release(mem, operand) \ |
646 | ({ __atomic_check_size((mem)); \ | |
647 | __atomic_fetch_or ((mem), (operand), __ATOMIC_RELEASE); }) | |
648 | ||
649 | # define atomic_fetch_xor_release(mem, operand) \ | |
650 | ({ __atomic_check_size((mem)); \ | |
651 | __atomic_fetch_xor ((mem), (operand), __ATOMIC_RELEASE); }) | |
ff871426 TR |
652 | |
653 | #else /* !USE_ATOMIC_COMPILER_BUILTINS */ | |
654 | ||
655 | /* By default, we assume that read, write, and full barriers are equivalent | |
656 | to acquire, release, and seq_cst barriers. Archs for which this does not | |
657 | hold have to provide custom definitions of the fences. */ | |
658 | # ifndef atomic_thread_fence_acquire | |
659 | # define atomic_thread_fence_acquire() atomic_read_barrier () | |
660 | # endif | |
661 | # ifndef atomic_thread_fence_release | |
662 | # define atomic_thread_fence_release() atomic_write_barrier () | |
663 | # endif | |
664 | # ifndef atomic_thread_fence_seq_cst | |
665 | # define atomic_thread_fence_seq_cst() atomic_full_barrier () | |
666 | # endif | |
667 | ||
668 | # ifndef atomic_load_relaxed | |
669 | # define atomic_load_relaxed(mem) \ | |
12d2dd70 | 670 | ({ __typeof ((__typeof (*(mem))) *(mem)) __atg100_val; \ |
ff871426 TR |
671 | __asm ("" : "=r" (__atg100_val) : "0" (*(mem))); \ |
672 | __atg100_val; }) | |
673 | # endif | |
674 | # ifndef atomic_load_acquire | |
675 | # define atomic_load_acquire(mem) \ | |
676 | ({ __typeof (*(mem)) __atg101_val = atomic_load_relaxed (mem); \ | |
677 | atomic_thread_fence_acquire (); \ | |
678 | __atg101_val; }) | |
679 | # endif | |
680 | ||
681 | # ifndef atomic_store_relaxed | |
682 | /* XXX Use inline asm here? */ | |
683 | # define atomic_store_relaxed(mem, val) do { *(mem) = (val); } while (0) | |
684 | # endif | |
685 | # ifndef atomic_store_release | |
686 | # define atomic_store_release(mem, val) \ | |
687 | do { \ | |
688 | atomic_thread_fence_release (); \ | |
689 | atomic_store_relaxed ((mem), (val)); \ | |
690 | } while (0) | |
691 | # endif | |
692 | ||
693 | /* On failure, this CAS has memory_order_relaxed semantics. */ | |
694 | /* XXX This potentially has one branch more than necessary, but archs | |
695 | currently do not define a CAS that returns both the previous value and | |
696 | the success flag. */ | |
697 | # ifndef atomic_compare_exchange_weak_acquire | |
698 | # define atomic_compare_exchange_weak_acquire(mem, expected, desired) \ | |
699 | ({ typeof (*(expected)) __atg102_expected = *(expected); \ | |
700 | *(expected) = \ | |
701 | atomic_compare_and_exchange_val_acq ((mem), (desired), *(expected)); \ | |
702 | *(expected) == __atg102_expected; }) | |
703 | # endif | |
704 | # ifndef atomic_compare_exchange_weak_relaxed | |
705 | /* XXX Fall back to CAS with acquire MO because archs do not define a weaker | |
706 | CAS. */ | |
707 | # define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \ | |
708 | atomic_compare_exchange_weak_acquire ((mem), (expected), (desired)) | |
709 | # endif | |
710 | # ifndef atomic_compare_exchange_weak_release | |
711 | # define atomic_compare_exchange_weak_release(mem, expected, desired) \ | |
712 | ({ typeof (*(expected)) __atg103_expected = *(expected); \ | |
713 | *(expected) = \ | |
714 | atomic_compare_and_exchange_val_rel ((mem), (desired), *(expected)); \ | |
715 | *(expected) == __atg103_expected; }) | |
716 | # endif | |
717 | ||
f0e3925b TR |
718 | /* XXX Fall back to acquire MO because archs do not define a weaker |
719 | atomic_exchange. */ | |
720 | # ifndef atomic_exchange_relaxed | |
721 | # define atomic_exchange_relaxed(mem, val) \ | |
722 | atomic_exchange_acq ((mem), (val)) | |
723 | # endif | |
ff871426 TR |
724 | # ifndef atomic_exchange_acquire |
725 | # define atomic_exchange_acquire(mem, val) \ | |
726 | atomic_exchange_acq ((mem), (val)) | |
727 | # endif | |
728 | # ifndef atomic_exchange_release | |
729 | # define atomic_exchange_release(mem, val) \ | |
730 | atomic_exchange_rel ((mem), (val)) | |
731 | # endif | |
732 | ||
733 | # ifndef atomic_fetch_add_acquire | |
734 | # define atomic_fetch_add_acquire(mem, operand) \ | |
735 | atomic_exchange_and_add_acq ((mem), (operand)) | |
736 | # endif | |
737 | # ifndef atomic_fetch_add_relaxed | |
738 | /* XXX Fall back to acquire MO because the MO semantics of | |
739 | atomic_exchange_and_add are not documented; the generic version falls back | |
740 | to atomic_exchange_and_add_acq if atomic_exchange_and_add is not defined, | |
741 | and vice versa. */ | |
742 | # define atomic_fetch_add_relaxed(mem, operand) \ | |
743 | atomic_fetch_add_acquire ((mem), (operand)) | |
744 | # endif | |
745 | # ifndef atomic_fetch_add_release | |
746 | # define atomic_fetch_add_release(mem, operand) \ | |
747 | atomic_exchange_and_add_rel ((mem), (operand)) | |
748 | # endif | |
749 | # ifndef atomic_fetch_add_acq_rel | |
750 | # define atomic_fetch_add_acq_rel(mem, operand) \ | |
751 | ({ atomic_thread_fence_release (); \ | |
752 | atomic_exchange_and_add_acq ((mem), (operand)); }) | |
753 | # endif | |
754 | ||
13cb8f76 TR |
755 | /* XXX Fall back to acquire MO because archs do not define a weaker |
756 | atomic_and_val. */ | |
757 | # ifndef atomic_fetch_and_relaxed | |
758 | # define atomic_fetch_and_relaxed(mem, operand) \ | |
759 | atomic_fetch_and_acquire ((mem), (operand)) | |
760 | # endif | |
ff871426 TR |
761 | /* XXX The default for atomic_and_val has acquire semantics, but this is not |
762 | documented. */ | |
763 | # ifndef atomic_fetch_and_acquire | |
764 | # define atomic_fetch_and_acquire(mem, operand) \ | |
765 | atomic_and_val ((mem), (operand)) | |
766 | # endif | |
13cb8f76 TR |
767 | # ifndef atomic_fetch_and_release |
768 | /* XXX This unnecessarily has acquire MO. */ | |
769 | # define atomic_fetch_and_release(mem, operand) \ | |
770 | ({ atomic_thread_fence_release (); \ | |
771 | atomic_and_val ((mem), (operand)); }) | |
772 | # endif | |
ff871426 TR |
773 | |
774 | /* XXX The default for atomic_or_val has acquire semantics, but this is not | |
775 | documented. */ | |
776 | # ifndef atomic_fetch_or_acquire | |
777 | # define atomic_fetch_or_acquire(mem, operand) \ | |
778 | atomic_or_val ((mem), (operand)) | |
779 | # endif | |
780 | /* XXX Fall back to acquire MO because archs do not define a weaker | |
781 | atomic_or_val. */ | |
782 | # ifndef atomic_fetch_or_relaxed | |
783 | # define atomic_fetch_or_relaxed(mem, operand) \ | |
784 | atomic_fetch_or_acquire ((mem), (operand)) | |
785 | # endif | |
13cb8f76 TR |
786 | /* XXX Contains an unnecessary acquire MO because archs do not define a weaker |
787 | atomic_or_val. */ | |
788 | # ifndef atomic_fetch_or_release | |
789 | # define atomic_fetch_or_release(mem, operand) \ | |
790 | ({ atomic_thread_fence_release (); \ | |
791 | atomic_fetch_or_acquire ((mem), (operand)); }) | |
792 | # endif | |
793 | ||
794 | # ifndef atomic_fetch_xor_release | |
abff18c0 CD |
795 | /* Failing the atomic_compare_exchange_weak_release reloads the value in |
796 | __atg104_expected, so we need only do the XOR again and retry. */ | |
13cb8f76 | 797 | # define atomic_fetch_xor_release(mem, operand) \ |
abff18c0 CD |
798 | ({ __typeof (mem) __atg104_memp = (mem); \ |
799 | __typeof (*(mem)) __atg104_expected = (*__atg104_memp); \ | |
800 | __typeof (*(mem)) __atg104_desired; \ | |
13cb8f76 TR |
801 | __typeof (*(mem)) __atg104_op = (operand); \ |
802 | \ | |
803 | do \ | |
abff18c0 CD |
804 | __atg104_desired = __atg104_expected ^ __atg104_op; \ |
805 | while (__glibc_unlikely \ | |
806 | (atomic_compare_exchange_weak_release ( \ | |
807 | __atg104_memp, &__atg104_expected, __atg104_desired) \ | |
808 | == 0)); \ | |
809 | __atg104_expected; }) | |
13cb8f76 | 810 | #endif |
ff871426 TR |
811 | |
812 | #endif /* !USE_ATOMIC_COMPILER_BUILTINS */ | |
813 | ||
4eb984d3 TR |
814 | /* This operation does not affect synchronization semantics but can be used |
815 | in the body of a spin loop to potentially improve its efficiency. */ | |
816 | #ifndef atomic_spin_nop | |
817 | # define atomic_spin_nop() do { /* nothing */ } while (0) | |
f377d022 UD |
818 | #endif |
819 | ||
12d2dd70 SL |
820 | /* ATOMIC_EXCHANGE_USES_CAS is non-zero if atomic_exchange operations |
821 | are implemented based on a CAS loop; otherwise, this is zero and we assume | |
822 | that the atomic_exchange operations could provide better performance | |
823 | than a CAS loop. */ | |
824 | #ifndef ATOMIC_EXCHANGE_USES_CAS | |
825 | # error ATOMIC_EXCHANGE_USES_CAS has to be defined. | |
826 | #endif | |
827 | ||
76a50749 | 828 | #endif /* atomic.h */ |