]>
git.ipfire.org Git - thirdparty/glibc.git/blob - sysdeps/x86/atomic-machine.h
1 /* Atomic operations. X86 version.
2 Copyright (C) 2018-2022 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
19 #ifndef _X86_ATOMIC_MACHINE_H
20 #define _X86_ATOMIC_MACHINE_H 1
23 #include <tls.h> /* For tcbhead_t. */
24 #include <libc-pointer-arith.h> /* For cast_to_integer. */
26 #define LOCK_PREFIX "lock;"
28 #define USE_ATOMIC_COMPILER_BUILTINS 1
31 # define __HAVE_64B_ATOMICS 1
34 # define BR_CONSTRAINT "q"
35 # define IBR_CONSTRAINT "iq"
37 /* Since the Pentium, i386 CPUs have supported 64-bit atomics, but the
38 i386 psABI supplement provides only 4-byte alignment for uint64_t
39 inside structs, so it is currently not possible to use 64-bit
40 atomics on this platform. */
41 # define __HAVE_64B_ATOMICS 0
44 # define BR_CONSTRAINT "r"
45 # define IBR_CONSTRAINT "ir"
47 #define ATOMIC_EXCHANGE_USES_CAS 0
49 #define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
50 __sync_val_compare_and_swap (mem, oldval, newval)
51 #define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
52 (! __sync_bool_compare_and_swap (mem, oldval, newval))
55 #define __arch_c_compare_and_exchange_val_8_acq(mem, newval, oldval) \
56 ({ __typeof (*mem) ret; \
57 __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \
60 "0:\tcmpxchgb %b2, %1" \
61 : "=a" (ret), "=m" (*mem) \
62 : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \
63 "i" (offsetof (tcbhead_t, multiple_threads))); \
66 #define __arch_c_compare_and_exchange_val_16_acq(mem, newval, oldval) \
67 ({ __typeof (*mem) ret; \
68 __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \
71 "0:\tcmpxchgw %w2, %1" \
72 : "=a" (ret), "=m" (*mem) \
73 : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \
74 "i" (offsetof (tcbhead_t, multiple_threads))); \
77 #define __arch_c_compare_and_exchange_val_32_acq(mem, newval, oldval) \
78 ({ __typeof (*mem) ret; \
79 __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \
82 "0:\tcmpxchgl %2, %1" \
83 : "=a" (ret), "=m" (*mem) \
84 : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \
85 "i" (offsetof (tcbhead_t, multiple_threads))); \
89 # define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
90 ({ __typeof (*mem) ret; \
91 __asm __volatile ("cmpl $0, %%fs:%P5\n\t" \
94 "0:\tcmpxchgq %q2, %1" \
95 : "=a" (ret), "=m" (*mem) \
96 : "q" ((int64_t) cast_to_integer (newval)), \
98 "0" ((int64_t) cast_to_integer (oldval)), \
99 "i" (offsetof (tcbhead_t, multiple_threads))); \
101 # define do_exchange_and_add_val_64_acq(pfx, mem, value) 0
102 # define do_add_val_64_acq(pfx, mem, value) do { } while (0)
104 /* XXX We do not really need 64-bit compare-and-exchange. At least
105 not in the moment. Using it would mean causing portability
106 problems since not many other 32-bit architectures have support for
107 such an operation. So don't define any code for now. If it is
108 really going to be used the code below can be used on Intel Pentium
109 and later, but NOT on i486. */
110 # define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
111 ({ __typeof (*mem) ret = *(mem); \
112 __atomic_link_error (); \
117 # define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \
118 ({ __typeof (*mem) ret = *(mem); \
119 __atomic_link_error (); \
124 # define do_exchange_and_add_val_64_acq(pfx, mem, value) \
125 ({ __typeof (value) __addval = (value); \
126 __typeof (*mem) __result; \
127 __typeof (mem) __memp = (mem); \
128 __typeof (*mem) __tmpval; \
129 __result = *__memp; \
131 __tmpval = __result; \
132 while ((__result = pfx##_compare_and_exchange_val_64_acq \
133 (__memp, __result + __addval, __result)) == __tmpval); \
136 # define do_add_val_64_acq(pfx, mem, value) \
138 __typeof (value) __addval = (value); \
139 __typeof (mem) __memp = (mem); \
140 __typeof (*mem) __oldval = *__memp; \
141 __typeof (*mem) __tmpval; \
143 __tmpval = __oldval; \
144 while ((__oldval = pfx##_compare_and_exchange_val_64_acq \
145 (__memp, __oldval + __addval, __oldval)) == __tmpval); \
150 /* Note that we need no lock prefix. */
151 #define atomic_exchange_acq(mem, newvalue) \
152 ({ __typeof (*mem) result; \
153 if (sizeof (*mem) == 1) \
154 __asm __volatile ("xchgb %b0, %1" \
155 : "=q" (result), "=m" (*mem) \
156 : "0" (newvalue), "m" (*mem)); \
157 else if (sizeof (*mem) == 2) \
158 __asm __volatile ("xchgw %w0, %1" \
159 : "=r" (result), "=m" (*mem) \
160 : "0" (newvalue), "m" (*mem)); \
161 else if (sizeof (*mem) == 4) \
162 __asm __volatile ("xchgl %0, %1" \
163 : "=r" (result), "=m" (*mem) \
164 : "0" (newvalue), "m" (*mem)); \
165 else if (__HAVE_64B_ATOMICS) \
166 __asm __volatile ("xchgq %q0, %1" \
167 : "=r" (result), "=m" (*mem) \
168 : "0" ((int64_t) cast_to_integer (newvalue)), \
173 __atomic_link_error (); \
178 #define __arch_exchange_and_add_body(lock, pfx, mem, value) \
179 ({ __typeof (*mem) __result; \
180 __typeof (value) __addval = (value); \
181 if (sizeof (*mem) == 1) \
182 __asm __volatile (lock "xaddb %b0, %1" \
183 : "=q" (__result), "=m" (*mem) \
184 : "0" (__addval), "m" (*mem), \
185 "i" (offsetof (tcbhead_t, multiple_threads))); \
186 else if (sizeof (*mem) == 2) \
187 __asm __volatile (lock "xaddw %w0, %1" \
188 : "=r" (__result), "=m" (*mem) \
189 : "0" (__addval), "m" (*mem), \
190 "i" (offsetof (tcbhead_t, multiple_threads))); \
191 else if (sizeof (*mem) == 4) \
192 __asm __volatile (lock "xaddl %0, %1" \
193 : "=r" (__result), "=m" (*mem) \
194 : "0" (__addval), "m" (*mem), \
195 "i" (offsetof (tcbhead_t, multiple_threads))); \
196 else if (__HAVE_64B_ATOMICS) \
197 __asm __volatile (lock "xaddq %q0, %1" \
198 : "=r" (__result), "=m" (*mem) \
199 : "0" ((int64_t) cast_to_integer (__addval)), \
201 "i" (offsetof (tcbhead_t, multiple_threads))); \
203 __result = do_exchange_and_add_val_64_acq (pfx, (mem), __addval); \
206 #define atomic_exchange_and_add(mem, value) \
207 __sync_fetch_and_add (mem, value)
209 #define __arch_exchange_and_add_cprefix \
210 "cmpl $0, %%" SEG_REG ":%P4\n\tje 0f\n\tlock\n0:\t"
212 #define catomic_exchange_and_add(mem, value) \
213 __arch_exchange_and_add_body (__arch_exchange_and_add_cprefix, __arch_c, \
217 #define __arch_add_body(lock, pfx, apfx, mem, value) \
219 if (__builtin_constant_p (value) && (value) == 1) \
220 pfx##_increment (mem); \
221 else if (__builtin_constant_p (value) && (value) == -1) \
222 pfx##_decrement (mem); \
223 else if (sizeof (*mem) == 1) \
224 __asm __volatile (lock "addb %b1, %0" \
226 : IBR_CONSTRAINT (value), "m" (*mem), \
227 "i" (offsetof (tcbhead_t, multiple_threads))); \
228 else if (sizeof (*mem) == 2) \
229 __asm __volatile (lock "addw %w1, %0" \
231 : "ir" (value), "m" (*mem), \
232 "i" (offsetof (tcbhead_t, multiple_threads))); \
233 else if (sizeof (*mem) == 4) \
234 __asm __volatile (lock "addl %1, %0" \
236 : "ir" (value), "m" (*mem), \
237 "i" (offsetof (tcbhead_t, multiple_threads))); \
238 else if (__HAVE_64B_ATOMICS) \
239 __asm __volatile (lock "addq %q1, %0" \
241 : "ir" ((int64_t) cast_to_integer (value)), \
243 "i" (offsetof (tcbhead_t, multiple_threads))); \
245 do_add_val_64_acq (apfx, (mem), (value)); \
248 # define atomic_add(mem, value) \
249 __arch_add_body (LOCK_PREFIX, atomic, __arch, mem, value)
251 #define __arch_add_cprefix \
252 "cmpl $0, %%" SEG_REG ":%P3\n\tje 0f\n\tlock\n0:\t"
254 #define catomic_add(mem, value) \
255 __arch_add_body (__arch_add_cprefix, atomic, __arch_c, mem, value)
258 #define atomic_add_negative(mem, value) \
259 ({ unsigned char __result; \
260 if (sizeof (*mem) == 1) \
261 __asm __volatile (LOCK_PREFIX "addb %b2, %0; sets %1" \
262 : "=m" (*mem), "=qm" (__result) \
263 : IBR_CONSTRAINT (value), "m" (*mem)); \
264 else if (sizeof (*mem) == 2) \
265 __asm __volatile (LOCK_PREFIX "addw %w2, %0; sets %1" \
266 : "=m" (*mem), "=qm" (__result) \
267 : "ir" (value), "m" (*mem)); \
268 else if (sizeof (*mem) == 4) \
269 __asm __volatile (LOCK_PREFIX "addl %2, %0; sets %1" \
270 : "=m" (*mem), "=qm" (__result) \
271 : "ir" (value), "m" (*mem)); \
272 else if (__HAVE_64B_ATOMICS) \
273 __asm __volatile (LOCK_PREFIX "addq %q2, %0; sets %1" \
274 : "=m" (*mem), "=qm" (__result) \
275 : "ir" ((int64_t) cast_to_integer (value)), \
278 __atomic_link_error (); \
282 #define atomic_add_zero(mem, value) \
283 ({ unsigned char __result; \
284 if (sizeof (*mem) == 1) \
285 __asm __volatile (LOCK_PREFIX "addb %b2, %0; setz %1" \
286 : "=m" (*mem), "=qm" (__result) \
287 : IBR_CONSTRAINT (value), "m" (*mem)); \
288 else if (sizeof (*mem) == 2) \
289 __asm __volatile (LOCK_PREFIX "addw %w2, %0; setz %1" \
290 : "=m" (*mem), "=qm" (__result) \
291 : "ir" (value), "m" (*mem)); \
292 else if (sizeof (*mem) == 4) \
293 __asm __volatile (LOCK_PREFIX "addl %2, %0; setz %1" \
294 : "=m" (*mem), "=qm" (__result) \
295 : "ir" (value), "m" (*mem)); \
296 else if (__HAVE_64B_ATOMICS) \
297 __asm __volatile (LOCK_PREFIX "addq %q2, %0; setz %1" \
298 : "=m" (*mem), "=qm" (__result) \
299 : "ir" ((int64_t) cast_to_integer (value)), \
302 __atomic_link_error (); \
306 #define __arch_increment_body(lock, pfx, mem) \
308 if (sizeof (*mem) == 1) \
309 __asm __volatile (lock "incb %b0" \
312 "i" (offsetof (tcbhead_t, multiple_threads))); \
313 else if (sizeof (*mem) == 2) \
314 __asm __volatile (lock "incw %w0" \
317 "i" (offsetof (tcbhead_t, multiple_threads))); \
318 else if (sizeof (*mem) == 4) \
319 __asm __volatile (lock "incl %0" \
322 "i" (offsetof (tcbhead_t, multiple_threads))); \
323 else if (__HAVE_64B_ATOMICS) \
324 __asm __volatile (lock "incq %q0" \
327 "i" (offsetof (tcbhead_t, multiple_threads))); \
329 do_add_val_64_acq (pfx, mem, 1); \
332 #define atomic_increment(mem) __arch_increment_body (LOCK_PREFIX, __arch, mem)
334 #define __arch_increment_cprefix \
335 "cmpl $0, %%" SEG_REG ":%P2\n\tje 0f\n\tlock\n0:\t"
337 #define catomic_increment(mem) \
338 __arch_increment_body (__arch_increment_cprefix, __arch_c, mem)
341 #define atomic_increment_and_test(mem) \
342 ({ unsigned char __result; \
343 if (sizeof (*mem) == 1) \
344 __asm __volatile (LOCK_PREFIX "incb %b0; sete %b1" \
345 : "=m" (*mem), "=qm" (__result) \
347 else if (sizeof (*mem) == 2) \
348 __asm __volatile (LOCK_PREFIX "incw %w0; sete %w1" \
349 : "=m" (*mem), "=qm" (__result) \
351 else if (sizeof (*mem) == 4) \
352 __asm __volatile (LOCK_PREFIX "incl %0; sete %1" \
353 : "=m" (*mem), "=qm" (__result) \
355 else if (__HAVE_64B_ATOMICS) \
356 __asm __volatile (LOCK_PREFIX "incq %q0; sete %1" \
357 : "=m" (*mem), "=qm" (__result) \
360 __atomic_link_error (); \
364 #define __arch_decrement_body(lock, pfx, mem) \
366 if (sizeof (*mem) == 1) \
367 __asm __volatile (lock "decb %b0" \
370 "i" (offsetof (tcbhead_t, multiple_threads))); \
371 else if (sizeof (*mem) == 2) \
372 __asm __volatile (lock "decw %w0" \
375 "i" (offsetof (tcbhead_t, multiple_threads))); \
376 else if (sizeof (*mem) == 4) \
377 __asm __volatile (lock "decl %0" \
380 "i" (offsetof (tcbhead_t, multiple_threads))); \
381 else if (__HAVE_64B_ATOMICS) \
382 __asm __volatile (lock "decq %q0" \
385 "i" (offsetof (tcbhead_t, multiple_threads))); \
387 do_add_val_64_acq (pfx, mem, -1); \
390 #define atomic_decrement(mem) __arch_decrement_body (LOCK_PREFIX, __arch, mem)
392 #define __arch_decrement_cprefix \
393 "cmpl $0, %%" SEG_REG ":%P2\n\tje 0f\n\tlock\n0:\t"
395 #define catomic_decrement(mem) \
396 __arch_decrement_body (__arch_decrement_cprefix, __arch_c, mem)
399 #define atomic_decrement_and_test(mem) \
400 ({ unsigned char __result; \
401 if (sizeof (*mem) == 1) \
402 __asm __volatile (LOCK_PREFIX "decb %b0; sete %1" \
403 : "=m" (*mem), "=qm" (__result) \
405 else if (sizeof (*mem) == 2) \
406 __asm __volatile (LOCK_PREFIX "decw %w0; sete %1" \
407 : "=m" (*mem), "=qm" (__result) \
409 else if (sizeof (*mem) == 4) \
410 __asm __volatile (LOCK_PREFIX "decl %0; sete %1" \
411 : "=m" (*mem), "=qm" (__result) \
414 __asm __volatile (LOCK_PREFIX "decq %q0; sete %1" \
415 : "=m" (*mem), "=qm" (__result) \
420 #define atomic_bit_set(mem, bit) \
422 if (sizeof (*mem) == 1) \
423 __asm __volatile (LOCK_PREFIX "orb %b2, %0" \
425 : "m" (*mem), IBR_CONSTRAINT (1L << (bit))); \
426 else if (sizeof (*mem) == 2) \
427 __asm __volatile (LOCK_PREFIX "orw %w2, %0" \
429 : "m" (*mem), "ir" (1L << (bit))); \
430 else if (sizeof (*mem) == 4) \
431 __asm __volatile (LOCK_PREFIX "orl %2, %0" \
433 : "m" (*mem), "ir" (1L << (bit))); \
434 else if (__builtin_constant_p (bit) && (bit) < 32) \
435 __asm __volatile (LOCK_PREFIX "orq %2, %0" \
437 : "m" (*mem), "i" (1L << (bit))); \
438 else if (__HAVE_64B_ATOMICS) \
439 __asm __volatile (LOCK_PREFIX "orq %q2, %0" \
441 : "m" (*mem), "r" (1UL << (bit))); \
443 __atomic_link_error (); \
447 #define atomic_bit_test_set(mem, bit) \
448 ({ unsigned char __result; \
449 if (sizeof (*mem) == 1) \
450 __asm __volatile (LOCK_PREFIX "btsb %3, %1; setc %0" \
451 : "=q" (__result), "=m" (*mem) \
452 : "m" (*mem), IBR_CONSTRAINT (bit)); \
453 else if (sizeof (*mem) == 2) \
454 __asm __volatile (LOCK_PREFIX "btsw %3, %1; setc %0" \
455 : "=q" (__result), "=m" (*mem) \
456 : "m" (*mem), "ir" (bit)); \
457 else if (sizeof (*mem) == 4) \
458 __asm __volatile (LOCK_PREFIX "btsl %3, %1; setc %0" \
459 : "=q" (__result), "=m" (*mem) \
460 : "m" (*mem), "ir" (bit)); \
461 else if (__HAVE_64B_ATOMICS) \
462 __asm __volatile (LOCK_PREFIX "btsq %3, %1; setc %0" \
463 : "=q" (__result), "=m" (*mem) \
464 : "m" (*mem), "ir" (bit)); \
466 __atomic_link_error (); \
470 #define __arch_and_body(lock, mem, mask) \
472 if (sizeof (*mem) == 1) \
473 __asm __volatile (lock "andb %b1, %0" \
475 : IBR_CONSTRAINT (mask), "m" (*mem), \
476 "i" (offsetof (tcbhead_t, multiple_threads))); \
477 else if (sizeof (*mem) == 2) \
478 __asm __volatile (lock "andw %w1, %0" \
480 : "ir" (mask), "m" (*mem), \
481 "i" (offsetof (tcbhead_t, multiple_threads))); \
482 else if (sizeof (*mem) == 4) \
483 __asm __volatile (lock "andl %1, %0" \
485 : "ir" (mask), "m" (*mem), \
486 "i" (offsetof (tcbhead_t, multiple_threads))); \
487 else if (__HAVE_64B_ATOMICS) \
488 __asm __volatile (lock "andq %q1, %0" \
490 : "ir" (mask), "m" (*mem), \
491 "i" (offsetof (tcbhead_t, multiple_threads))); \
493 __atomic_link_error (); \
496 #define __arch_cprefix \
497 "cmpl $0, %%" SEG_REG ":%P3\n\tje 0f\n\tlock\n0:\t"
499 #define atomic_and(mem, mask) __arch_and_body (LOCK_PREFIX, mem, mask)
501 #define catomic_and(mem, mask) __arch_and_body (__arch_cprefix, mem, mask)
504 #define __arch_or_body(lock, mem, mask) \
506 if (sizeof (*mem) == 1) \
507 __asm __volatile (lock "orb %b1, %0" \
509 : IBR_CONSTRAINT (mask), "m" (*mem), \
510 "i" (offsetof (tcbhead_t, multiple_threads))); \
511 else if (sizeof (*mem) == 2) \
512 __asm __volatile (lock "orw %w1, %0" \
514 : "ir" (mask), "m" (*mem), \
515 "i" (offsetof (tcbhead_t, multiple_threads))); \
516 else if (sizeof (*mem) == 4) \
517 __asm __volatile (lock "orl %1, %0" \
519 : "ir" (mask), "m" (*mem), \
520 "i" (offsetof (tcbhead_t, multiple_threads))); \
521 else if (__HAVE_64B_ATOMICS) \
522 __asm __volatile (lock "orq %q1, %0" \
524 : "ir" (mask), "m" (*mem), \
525 "i" (offsetof (tcbhead_t, multiple_threads))); \
527 __atomic_link_error (); \
530 #define atomic_or(mem, mask) __arch_or_body (LOCK_PREFIX, mem, mask)
532 #define catomic_or(mem, mask) __arch_or_body (__arch_cprefix, mem, mask)
534 /* We don't use mfence because it is supposedly slower due to having to
535 provide stronger guarantees (e.g., regarding self-modifying code). */
536 #define atomic_full_barrier() \
537 __asm __volatile (LOCK_PREFIX "orl $0, (%%" SP_REG ")" ::: "memory")
538 #define atomic_read_barrier() __asm ("" ::: "memory")
539 #define atomic_write_barrier() __asm ("" ::: "memory")
541 #define atomic_spin_nop() __asm ("pause")
543 #endif /* atomic-machine.h */