[thirdparty/glibc.git] / sysdeps / powerpc / atomic-machine.h

/* Atomic operations.  PowerPC Common version.
   Copyright (C) 2003-2021 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

/*
 * Never include sysdeps/powerpc/atomic-machine.h directly.
 * Alway use include/atomic.h which will include either
 * sysdeps/powerpc/powerpc32/atomic-machine.h
 * or
 * sysdeps/powerpc/powerpc64/atomic-machine.h
 * as appropriate and which in turn include this file.
 */

#include <stdint.h>

typedef int32_t atomic32_t;
typedef uint32_t uatomic32_t;
typedef int_fast32_t atomic_fast32_t;
typedef uint_fast32_t uatomic_fast32_t;

typedef int64_t atomic64_t;
typedef uint64_t uatomic64_t;
typedef int_fast64_t atomic_fast64_t;
typedef uint_fast64_t uatomic_fast64_t;

typedef intptr_t atomicptr_t;
typedef uintptr_t uatomicptr_t;
typedef intmax_t atomic_max_t;
typedef uintmax_t uatomic_max_t;

/*
 * Powerpc does not have byte and halfword forms of load and reserve and
 * store conditional. So for powerpc we stub out the 8- and 16-bit forms.
 */
#define __arch_compare_and_exchange_bool_8_acq(mem, newval, oldval) \
  (abort (), 0)

#define __arch_compare_and_exchange_bool_16_acq(mem, newval, oldval) \
  (abort (), 0)

#define __ARCH_ACQ_INSTR	"isync"
#ifndef __ARCH_REL_INSTR
# define __ARCH_REL_INSTR	"sync"
#endif

#ifndef MUTEX_HINT_ACQ
# define MUTEX_HINT_ACQ
#endif
#ifndef MUTEX_HINT_REL
# define MUTEX_HINT_REL
#endif

#define atomic_full_barrier()	__asm ("sync" ::: "memory")

#define __arch_compare_and_exchange_val_32_acq(mem, newval, oldval)	      \
  ({									      \
      __typeof (*(mem)) __tmp;						      \
      __typeof (mem)  __memp = (mem);					      \
      __asm __volatile (						      \
		        "1:	lwarx	%0,0,%1" MUTEX_HINT_ACQ "\n"	      \
		        "	cmpw	%0,%2\n"			      \
		        "	bne	2f\n"				      \
		        "	stwcx.	%3,0,%1\n"			      \
		        "	bne-	1b\n"				      \
		        "2:	" __ARCH_ACQ_INSTR			      \
		        : "=&r" (__tmp)					      \
		        : "b" (__memp), "r" (oldval), "r" (newval)	      \
		        : "cr0", "memory");				      \
      __tmp;								      \
  })

#define __arch_compare_and_exchange_val_32_rel(mem, newval, oldval)	      \
  ({									      \
      __typeof (*(mem)) __tmp;						      \
      __typeof (mem)  __memp = (mem);					      \
      __asm __volatile (__ARCH_REL_INSTR "\n"				      \
		        "1:	lwarx	%0,0,%1" MUTEX_HINT_REL "\n"	      \
		        "	cmpw	%0,%2\n"			      \
		        "	bne	2f\n"				      \
		        "	stwcx.	%3,0,%1\n"			      \
		        "	bne-	1b\n"				      \
		        "2:	"					      \
		        : "=&r" (__tmp)					      \
		        : "b" (__memp), "r" (oldval), "r" (newval)	      \
		        : "cr0", "memory");				      \
      __tmp;								      \
  })

#define __arch_atomic_exchange_32_acq(mem, value)			      \
  ({									      \
    __typeof (*mem) __val;						      \
    __asm __volatile (							      \
		      "1:	lwarx	%0,0,%2" MUTEX_HINT_ACQ "\n"	      \
		      "		stwcx.	%3,0,%2\n"			      \
		      "		bne-	1b\n"				      \
		      "   " __ARCH_ACQ_INSTR				      \
		      : "=&r" (__val), "=m" (*mem)			      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_exchange_32_rel(mem, value) \
  ({									      \
    __typeof (*mem) __val;						      \
    __asm __volatile (__ARCH_REL_INSTR "\n"				      \
		      "1:	lwarx	%0,0,%2" MUTEX_HINT_REL "\n"	      \
		      "		stwcx.	%3,0,%2\n"			      \
		      "		bne-	1b"				      \
		      : "=&r" (__val), "=m" (*mem)			      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_exchange_and_add_32(mem, value) \
  ({									      \
    __typeof (*mem) __val, __tmp;					      \
    __asm __volatile ("1:	lwarx	%0,0,%3\n"			      \
		      "		add	%1,%0,%4\n"			      \
		      "		stwcx.	%1,0,%3\n"			      \
		      "		bne-	1b"				      \
		      : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)	      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_exchange_and_add_32_acq(mem, value) \
  ({									      \
    __typeof (*mem) __val, __tmp;					      \
    __asm __volatile ("1:	lwarx	%0,0,%3" MUTEX_HINT_ACQ "\n"	      \
		      "		add	%1,%0,%4\n"			      \
		      "		stwcx.	%1,0,%3\n"			      \
		      "		bne-	1b\n"				      \
		      __ARCH_ACQ_INSTR					      \
		      : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)	      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_exchange_and_add_32_rel(mem, value) \
  ({									      \
    __typeof (*mem) __val, __tmp;					      \
    __asm __volatile (__ARCH_REL_INSTR "\n"				      \
		      "1:	lwarx	%0,0,%3" MUTEX_HINT_REL "\n"	      \
		      "		add	%1,%0,%4\n"			      \
		      "		stwcx.	%1,0,%3\n"			      \
		      "		bne-	1b"				      \
		      : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)	      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_increment_val_32(mem) \
  ({									      \
    __typeof (*(mem)) __val;						      \
    __asm __volatile ("1:	lwarx	%0,0,%2\n"			      \
		      "		addi	%0,%0,1\n"			      \
		      "		stwcx.	%0,0,%2\n"			      \
		      "		bne-	1b"				      \
		      : "=&b" (__val), "=m" (*mem)			      \
		      : "b" (mem), "m" (*mem)				      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_decrement_val_32(mem) \
  ({									      \
    __typeof (*(mem)) __val;						      \
    __asm __volatile ("1:	lwarx	%0,0,%2\n"			      \
		      "		subi	%0,%0,1\n"			      \
		      "		stwcx.	%0,0,%2\n"			      \
		      "		bne-	1b"				      \
		      : "=&b" (__val), "=m" (*mem)			      \
		      : "b" (mem), "m" (*mem)				      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_decrement_if_positive_32(mem) \
  ({ int __val, __tmp;							      \
     __asm __volatile ("1:	lwarx	%0,0,%3\n"			      \
		       "	cmpwi	0,%0,0\n"			      \
		       "	addi	%1,%0,-1\n"			      \
		       "	ble	2f\n"				      \
		       "	stwcx.	%1,0,%3\n"			      \
		       "	bne-	1b\n"				      \
		       "2:	" __ARCH_ACQ_INSTR			      \
		       : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)	      \
		       : "b" (mem), "m" (*mem)				      \
		       : "cr0", "memory");				      \
     __val;								      \
  })

#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_compare_and_exchange_val_32_acq(mem, newval, oldval); \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_compare_and_exchange_val_64_acq(mem, newval, oldval); \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_compare_and_exchange_val_32_rel(mem, newval, oldval); \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_compare_and_exchange_val_64_rel(mem, newval, oldval); \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_exchange_acq(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_32_acq (mem, value);		      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_64_acq (mem, value);		      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_exchange_rel(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_32_rel (mem, value);		      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_64_rel (mem, value);		      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_exchange_and_add(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_and_add_32 (mem, value);	      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_and_add_64 (mem, value);	      \
    else 								      \
       abort ();							      \
    __result;								      \
  })
#define atomic_exchange_and_add_acq(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_and_add_32_acq (mem, value);	      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_and_add_64_acq (mem, value);	      \
    else 								      \
       abort ();							      \
    __result;								      \
  })
#define atomic_exchange_and_add_rel(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_and_add_32_rel (mem, value);	      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_and_add_64_rel (mem, value);	      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_increment_val(mem) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*(mem)) == 4)						      \
      __result = __arch_atomic_increment_val_32 (mem);			      \
    else if (sizeof (*(mem)) == 8)					      \
      __result = __arch_atomic_increment_val_64 (mem);			      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_increment(mem) ({ atomic_increment_val (mem); (void) 0; })

#define atomic_decrement_val(mem) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*(mem)) == 4)						      \
      __result = __arch_atomic_decrement_val_32 (mem);			      \
    else if (sizeof (*(mem)) == 8)					      \
      __result = __arch_atomic_decrement_val_64 (mem);			      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_decrement(mem) ({ atomic_decrement_val (mem); (void) 0; })


/* Decrement *MEM if it is > 0, and return the old value.  */
#define atomic_decrement_if_positive(mem) \
  ({ __typeof (*(mem)) __result;					      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_decrement_if_positive_32 (mem);		      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_decrement_if_positive_64 (mem);		      \
    else								      \
       abort ();							      \
    __result;								      \
  })
Commit	Line	Data
94c24227	1	/* Atomic operations. PowerPC Common version.
2b778ceb	2	Copyright (C) 2003-2021 Free Software Foundation, Inc.
f3c13160	3	This file is part of the GNU C Library.
f3c13160 RM	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6	16	License along with the GNU C Library; if not, see
5a82c748	17	<https://www.gnu.org/licenses/>. */
f3c13160	18
94c24227	19	/*
de071d19	20	* Never include sysdeps/powerpc/atomic-machine.h directly.
949ec764	21	* Alway use include/atomic.h which will include either
de071d19	22	* sysdeps/powerpc/powerpc32/atomic-machine.h
949ec764	23	* or
de071d19	24	* sysdeps/powerpc/powerpc64/atomic-machine.h
94c24227 UD	25	* as appropriate and which in turn include this file.
94c24227 UD	26	*/
f3c13160	27
94c24227	28	#include <stdint.h>
f3c13160 RM	29
	30	typedef int32_t atomic32_t;
	31	typedef uint32_t uatomic32_t;
	32	typedef int_fast32_t atomic_fast32_t;
	33	typedef uint_fast32_t uatomic_fast32_t;
	34
	35	typedef int64_t atomic64_t;
	36	typedef uint64_t uatomic64_t;
	37	typedef int_fast64_t atomic_fast64_t;
	38	typedef uint_fast64_t uatomic_fast64_t;
	39
	40	typedef intptr_t atomicptr_t;
	41	typedef uintptr_t uatomicptr_t;
	42	typedef intmax_t atomic_max_t;
	43	typedef uintmax_t uatomic_max_t;
	44
949ec764 UD	45	/*
949ec764 UD	46	* Powerpc does not have byte and halfword forms of load and reserve and
94c24227 UD	47	* store conditional. So for powerpc we stub out the 8- and 16-bit forms.
94c24227 UD	48	*/
5a3ab2fc	49	#define __arch_compare_and_exchange_bool_8_acq(mem, newval, oldval) \
f3c13160 RM	50	(abort (), 0)
f3c13160 RM	51
5a3ab2fc	52	#define __arch_compare_and_exchange_bool_16_acq(mem, newval, oldval) \
f3c13160	53	(abort (), 0)
ecb2b2fb	54
d5c4cce9 FW	55	#define __ARCH_ACQ_INSTR "isync"
	56	#ifndef __ARCH_REL_INSTR
	57	# define __ARCH_REL_INSTR "sync"
f3c13160 RM	58	#endif
f3c13160 RM	59
fa6e3bc3 UD	60	#ifndef MUTEX_HINT_ACQ
	61	# define MUTEX_HINT_ACQ
	62	#endif
	63	#ifndef MUTEX_HINT_REL
	64	# define MUTEX_HINT_REL
	65	#endif
	66
94c24227	67	#define atomic_full_barrier() __asm ("sync" ::: "memory")
94c24227	68
7158eae4 UD	69	#define __arch_compare_and_exchange_val_32_acq(mem, newval, oldval) \
	70	({ \
	71	__typeof (*(mem)) __tmp; \
	72	__typeof (mem) __memp = (mem); \
	73	__asm __volatile ( \
fa6e3bc3	74	"1: lwarx %0,0,%1" MUTEX_HINT_ACQ "\n" \
7158eae4 UD	75	" cmpw %0,%2\n" \
	76	" bne 2f\n" \
	77	" stwcx. %3,0,%1\n" \
	78	" bne- 1b\n" \
	79	"2: " __ARCH_ACQ_INSTR \
	80	: "=&r" (__tmp) \
	81	: "b" (__memp), "r" (oldval), "r" (newval) \
	82	: "cr0", "memory"); \
	83	__tmp; \
	84	})
f3c13160	85
7158eae4	86	#define __arch_compare_and_exchange_val_32_rel(mem, newval, oldval) \
6087c485 UD	87	({ \
	88	__typeof (*(mem)) __tmp; \
	89	__typeof (mem) __memp = (mem); \
	90	__asm __volatile (__ARCH_REL_INSTR "\n" \
fa6e3bc3	91	"1: lwarx %0,0,%1" MUTEX_HINT_REL "\n" \
6087c485 UD	92	" cmpw %0,%2\n" \
	93	" bne 2f\n" \
	94	" stwcx. %3,0,%1\n" \
	95	" bne- 1b\n" \
7158eae4	96	"2: " \
6087c485 UD	97	: "=&r" (__tmp) \
	98	: "b" (__memp), "r" (oldval), "r" (newval) \
	99	: "cr0", "memory"); \
	100	__tmp; \
	101	})
	102
7158eae4 UD	103	#define __arch_atomic_exchange_32_acq(mem, value) \
	104	({ \
	105	__typeof (*mem) __val; \
	106	__asm __volatile ( \
fa6e3bc3	107	"1: lwarx %0,0,%2" MUTEX_HINT_ACQ "\n" \
7158eae4 UD	108	" stwcx. %3,0,%2\n" \
	109	" bne- 1b\n" \
	110	" " __ARCH_ACQ_INSTR \
	111	: "=&r" (__val), "=m" (*mem) \
ecb2b2fb	112	: "b" (mem), "r" (value), "m" (*mem) \
7158eae4 UD	113	: "cr0", "memory"); \
	114	__val; \
	115	})
	116
	117	#define __arch_atomic_exchange_32_rel(mem, value) \
3e195d93 RM	118	({ \
	119	__typeof (*mem) __val; \
	120	__asm __volatile (__ARCH_REL_INSTR "\n" \
fa6e3bc3	121	"1: lwarx %0,0,%2" MUTEX_HINT_REL "\n" \
859e708f RM	122	" stwcx. %3,0,%2\n" \
859e708f RM	123	" bne- 1b" \
3e195d93	124	: "=&r" (__val), "=m" (*mem) \
ecb2b2fb	125	: "b" (mem), "r" (value), "m" (*mem) \
7158eae4	126	: "cr0", "memory"); \
3e195d93 RM	127	__val; \
3e195d93 RM	128	})
f79466a8	129
6087c485	130	#define __arch_atomic_exchange_and_add_32(mem, value) \
3e195d93 RM	131	({ \
	132	__typeof (*mem) __val, __tmp; \
	133	__asm __volatile ("1: lwarx %0,0,%3\n" \
859e708f RM	134	" add %1,%0,%4\n" \
	135	" stwcx. %1,0,%3\n" \
	136	" bne- 1b" \
3e195d93	137	: "=&b" (__val), "=&r" (__tmp), "=m" (*mem) \
ecb2b2fb	138	: "b" (mem), "r" (value), "m" (*mem) \
7158eae4	139	: "cr0", "memory"); \
3e195d93 RM	140	__val; \
3e195d93 RM	141	})
448163ba	142
704f7947 AZ	143	#define __arch_atomic_exchange_and_add_32_acq(mem, value) \
	144	({ \
	145	__typeof (*mem) __val, __tmp; \
	146	__asm __volatile ("1: lwarx %0,0,%3" MUTEX_HINT_ACQ "\n" \
	147	" add %1,%0,%4\n" \
	148	" stwcx. %1,0,%3\n" \
	149	" bne- 1b\n" \
	150	__ARCH_ACQ_INSTR \
	151	: "=&b" (__val), "=&r" (__tmp), "=m" (*mem) \
	152	: "b" (mem), "r" (value), "m" (*mem) \
	153	: "cr0", "memory"); \
	154	__val; \
	155	})
	156
	157	#define __arch_atomic_exchange_and_add_32_rel(mem, value) \
	158	({ \
	159	__typeof (*mem) __val, __tmp; \
	160	__asm __volatile (__ARCH_REL_INSTR "\n" \
	161	"1: lwarx %0,0,%3" MUTEX_HINT_REL "\n" \
	162	" add %1,%0,%4\n" \
	163	" stwcx. %1,0,%3\n" \
	164	" bne- 1b" \
	165	: "=&b" (__val), "=&r" (__tmp), "=m" (*mem) \
	166	: "b" (mem), "r" (value), "m" (*mem) \
	167	: "cr0", "memory"); \
	168	__val; \
	169	})
	170
7ba0e52c UD	171	#define __arch_atomic_increment_val_32(mem) \
	172	({ \
	173	__typeof (*(mem)) __val; \
	174	__asm __volatile ("1: lwarx %0,0,%2\n" \
	175	" addi %0,%0,1\n" \
	176	" stwcx. %0,0,%2\n" \
	177	" bne- 1b" \
	178	: "=&b" (__val), "=m" (*mem) \
	179	: "b" (mem), "m" (*mem) \
	180	: "cr0", "memory"); \
	181	__val; \
	182	})
	183
	184	#define __arch_atomic_decrement_val_32(mem) \
	185	({ \
	186	__typeof (*(mem)) __val; \
	187	__asm __volatile ("1: lwarx %0,0,%2\n" \
	188	" subi %0,%0,1\n" \
	189	" stwcx. %0,0,%2\n" \
	190	" bne- 1b" \
	191	: "=&b" (__val), "=m" (*mem) \
	192	: "b" (mem), "m" (*mem) \
	193	: "cr0", "memory"); \
	194	__val; \
	195	})
	196
6087c485	197	#define __arch_atomic_decrement_if_positive_32(mem) \
3171ae99 UD	198	({ int __val, __tmp; \
	199	__asm __volatile ("1: lwarx %0,0,%3\n" \
	200	" cmpwi 0,%0,0\n" \
	201	" addi %1,%0,-1\n" \
	202	" ble 2f\n" \
	203	" stwcx. %1,0,%3\n" \
	204	" bne- 1b\n" \
	205	"2: " __ARCH_ACQ_INSTR \
	206	: "=&b" (__val), "=&r" (__tmp), "=m" (*mem) \
ecb2b2fb	207	: "b" (mem), "m" (*mem) \
7158eae4	208	: "cr0", "memory"); \
3171ae99 UD	209	__val; \
	210	})
	211
6087c485 UD	212	#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
	213	({ \
	214	__typeof (*(mem)) __result; \
	215	if (sizeof (*mem) == 4) \
	216	__result = __arch_compare_and_exchange_val_32_acq(mem, newval, oldval); \
	217	else if (sizeof (*mem) == 8) \
	218	__result = __arch_compare_and_exchange_val_64_acq(mem, newval, oldval); \
	219	else \
	220	abort (); \
	221	__result; \
	222	})
ecb2b2fb	223
7158eae4 UD	224	#define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \
	225	({ \
	226	__typeof (*(mem)) __result; \
	227	if (sizeof (*mem) == 4) \
	228	__result = __arch_compare_and_exchange_val_32_rel(mem, newval, oldval); \
	229	else if (sizeof (*mem) == 8) \
	230	__result = __arch_compare_and_exchange_val_64_rel(mem, newval, oldval); \
	231	else \
	232	abort (); \
	233	__result; \
	234	})
6087c485	235
949ec764	236	#define atomic_exchange_acq(mem, value) \
3e195d93 RM	237	({ \
	238	__typeof (*(mem)) __result; \
	239	if (sizeof (*mem) == 4) \
7158eae4 UD	240	__result = __arch_atomic_exchange_32_acq (mem, value); \
	241	else if (sizeof (*mem) == 8) \
	242	__result = __arch_atomic_exchange_64_acq (mem, value); \
	243	else \
	244	abort (); \
	245	__result; \
	246	})
	247
	248	#define atomic_exchange_rel(mem, value) \
	249	({ \
	250	__typeof (*(mem)) __result; \
	251	if (sizeof (*mem) == 4) \
	252	__result = __arch_atomic_exchange_32_rel (mem, value); \
3e195d93	253	else if (sizeof (*mem) == 8) \
7158eae4	254	__result = __arch_atomic_exchange_64_rel (mem, value); \
3e195d93 RM	255	else \
	256	abort (); \
	257	__result; \
	258	})
f79466a8	259
6087c485	260	#define atomic_exchange_and_add(mem, value) \
3e195d93 RM	261	({ \
	262	__typeof (*(mem)) __result; \
	263	if (sizeof (*mem) == 4) \
6087c485	264	__result = __arch_atomic_exchange_and_add_32 (mem, value); \
3e195d93	265	else if (sizeof (*mem) == 8) \
6087c485	266	__result = __arch_atomic_exchange_and_add_64 (mem, value); \
3e195d93	267	else \
f79466a8	268	abort (); \
3e195d93	269	__result; \
f79466a8	270	})
704f7947 AZ	271	#define atomic_exchange_and_add_acq(mem, value) \
	272	({ \
	273	__typeof (*(mem)) __result; \
	274	if (sizeof (*mem) == 4) \
	275	__result = __arch_atomic_exchange_and_add_32_acq (mem, value); \
	276	else if (sizeof (*mem) == 8) \
	277	__result = __arch_atomic_exchange_and_add_64_acq (mem, value); \
	278	else \
	279	abort (); \
	280	__result; \
	281	})
	282	#define atomic_exchange_and_add_rel(mem, value) \
	283	({ \
	284	__typeof (*(mem)) __result; \
	285	if (sizeof (*mem) == 4) \
	286	__result = __arch_atomic_exchange_and_add_32_rel (mem, value); \
	287	else if (sizeof (*mem) == 8) \
	288	__result = __arch_atomic_exchange_and_add_64_rel (mem, value); \
	289	else \
	290	abort (); \
	291	__result; \
	292	})
f79466a8	293
7ba0e52c UD	294	#define atomic_increment_val(mem) \
	295	({ \
	296	__typeof (*(mem)) __result; \
	297	if (sizeof (*(mem)) == 4) \
	298	__result = __arch_atomic_increment_val_32 (mem); \
	299	else if (sizeof (*(mem)) == 8) \
	300	__result = __arch_atomic_increment_val_64 (mem); \
	301	else \
	302	abort (); \
	303	__result; \
	304	})
	305
	306	#define atomic_increment(mem) ({ atomic_increment_val (mem); (void) 0; })
	307
	308	#define atomic_decrement_val(mem) \
	309	({ \
	310	__typeof (*(mem)) __result; \
	311	if (sizeof (*(mem)) == 4) \
	312	__result = __arch_atomic_decrement_val_32 (mem); \
	313	else if (sizeof (*(mem)) == 8) \
	314	__result = __arch_atomic_decrement_val_64 (mem); \
	315	else \
	316	abort (); \
	317	__result; \
	318	})
	319
	320	#define atomic_decrement(mem) ({ atomic_decrement_val (mem); (void) 0; })
	321
f79466a8 UD	322
f79466a8 UD	323	/* Decrement MEM if it is > 0, and return the old value. /
6087c485	324	#define atomic_decrement_if_positive(mem) \
448163ba	325	({ __typeof (*(mem)) __result; \
3171ae99	326	if (sizeof (*mem) == 4) \
448163ba	327	__result = __arch_atomic_decrement_if_positive_32 (mem); \
3171ae99	328	else if (sizeof (*mem) == 8) \
448163ba RM	329	__result = __arch_atomic_decrement_if_positive_64 (mem); \
448163ba RM	330	else \
f79466a8	331	abort (); \
3171ae99	332	__result; \
f79466a8	333	})