[thirdparty/glibc.git] / sysdeps / powerpc / atomic-machine.h

/* Atomic operations.  PowerPC Common version.
   Copyright (C) 2003-2019 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Paul Mackerras <paulus@au.ibm.com>, 2003.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

/*
 * Never include sysdeps/powerpc/atomic-machine.h directly.
 * Alway use include/atomic.h which will include either
 * sysdeps/powerpc/powerpc32/atomic-machine.h
 * or
 * sysdeps/powerpc/powerpc64/atomic-machine.h
 * as appropriate and which in turn include this file.
 */

#include <stdint.h>

typedef int32_t atomic32_t;
typedef uint32_t uatomic32_t;
typedef int_fast32_t atomic_fast32_t;
typedef uint_fast32_t uatomic_fast32_t;

typedef int64_t atomic64_t;
typedef uint64_t uatomic64_t;
typedef int_fast64_t atomic_fast64_t;
typedef uint_fast64_t uatomic_fast64_t;

typedef intptr_t atomicptr_t;
typedef uintptr_t uatomicptr_t;
typedef intmax_t atomic_max_t;
typedef uintmax_t uatomic_max_t;

/*
 * Powerpc does not have byte and halfword forms of load and reserve and
 * store conditional. So for powerpc we stub out the 8- and 16-bit forms.
 */
#define __arch_compare_and_exchange_bool_8_acq(mem, newval, oldval) \
  (abort (), 0)

#define __arch_compare_and_exchange_bool_16_acq(mem, newval, oldval) \
  (abort (), 0)

#ifdef UP
# define __ARCH_ACQ_INSTR	""
# define __ARCH_REL_INSTR	""
#else
# define __ARCH_ACQ_INSTR	"isync"
# ifndef __ARCH_REL_INSTR
#  define __ARCH_REL_INSTR	"sync"
# endif
#endif

#ifndef MUTEX_HINT_ACQ
# define MUTEX_HINT_ACQ
#endif
#ifndef MUTEX_HINT_REL
# define MUTEX_HINT_REL
#endif

#define atomic_full_barrier()	__asm ("sync" ::: "memory")

#define __arch_compare_and_exchange_val_32_acq(mem, newval, oldval)	      \
  ({									      \
      __typeof (*(mem)) __tmp;						      \
      __typeof (mem)  __memp = (mem);					      \
      __asm __volatile (						      \
		        "1:	lwarx	%0,0,%1" MUTEX_HINT_ACQ "\n"	      \
		        "	cmpw	%0,%2\n"			      \
		        "	bne	2f\n"				      \
		        "	stwcx.	%3,0,%1\n"			      \
		        "	bne-	1b\n"				      \
		        "2:	" __ARCH_ACQ_INSTR			      \
		        : "=&r" (__tmp)					      \
		        : "b" (__memp), "r" (oldval), "r" (newval)	      \
		        : "cr0", "memory");				      \
      __tmp;								      \
  })

#define __arch_compare_and_exchange_val_32_rel(mem, newval, oldval)	      \
  ({									      \
      __typeof (*(mem)) __tmp;						      \
      __typeof (mem)  __memp = (mem);					      \
      __asm __volatile (__ARCH_REL_INSTR "\n"				      \
		        "1:	lwarx	%0,0,%1" MUTEX_HINT_REL "\n"	      \
		        "	cmpw	%0,%2\n"			      \
		        "	bne	2f\n"				      \
		        "	stwcx.	%3,0,%1\n"			      \
		        "	bne-	1b\n"				      \
		        "2:	"					      \
		        : "=&r" (__tmp)					      \
		        : "b" (__memp), "r" (oldval), "r" (newval)	      \
		        : "cr0", "memory");				      \
      __tmp;								      \
  })

#define __arch_atomic_exchange_32_acq(mem, value)			      \
  ({									      \
    __typeof (*mem) __val;						      \
    __asm __volatile (							      \
		      "1:	lwarx	%0,0,%2" MUTEX_HINT_ACQ "\n"	      \
		      "		stwcx.	%3,0,%2\n"			      \
		      "		bne-	1b\n"				      \
		      "   " __ARCH_ACQ_INSTR				      \
		      : "=&r" (__val), "=m" (*mem)			      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_exchange_32_rel(mem, value) \
  ({									      \
    __typeof (*mem) __val;						      \
    __asm __volatile (__ARCH_REL_INSTR "\n"				      \
		      "1:	lwarx	%0,0,%2" MUTEX_HINT_REL "\n"	      \
		      "		stwcx.	%3,0,%2\n"			      \
		      "		bne-	1b"				      \
		      : "=&r" (__val), "=m" (*mem)			      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_exchange_and_add_32(mem, value) \
  ({									      \
    __typeof (*mem) __val, __tmp;					      \
    __asm __volatile ("1:	lwarx	%0,0,%3\n"			      \
		      "		add	%1,%0,%4\n"			      \
		      "		stwcx.	%1,0,%3\n"			      \
		      "		bne-	1b"				      \
		      : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)	      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_exchange_and_add_32_acq(mem, value) \
  ({									      \
    __typeof (*mem) __val, __tmp;					      \
    __asm __volatile ("1:	lwarx	%0,0,%3" MUTEX_HINT_ACQ "\n"	      \
		      "		add	%1,%0,%4\n"			      \
		      "		stwcx.	%1,0,%3\n"			      \
		      "		bne-	1b\n"				      \
		      __ARCH_ACQ_INSTR					      \
		      : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)	      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_exchange_and_add_32_rel(mem, value) \
  ({									      \
    __typeof (*mem) __val, __tmp;					      \
    __asm __volatile (__ARCH_REL_INSTR "\n"				      \
		      "1:	lwarx	%0,0,%3" MUTEX_HINT_REL "\n"	      \
		      "		add	%1,%0,%4\n"			      \
		      "		stwcx.	%1,0,%3\n"			      \
		      "		bne-	1b"				      \
		      : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)	      \
		      : "b" (mem), "r" (value), "m" (*mem)		      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_increment_val_32(mem) \
  ({									      \
    __typeof (*(mem)) __val;						      \
    __asm __volatile ("1:	lwarx	%0,0,%2\n"			      \
		      "		addi	%0,%0,1\n"			      \
		      "		stwcx.	%0,0,%2\n"			      \
		      "		bne-	1b"				      \
		      : "=&b" (__val), "=m" (*mem)			      \
		      : "b" (mem), "m" (*mem)				      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_decrement_val_32(mem) \
  ({									      \
    __typeof (*(mem)) __val;						      \
    __asm __volatile ("1:	lwarx	%0,0,%2\n"			      \
		      "		subi	%0,%0,1\n"			      \
		      "		stwcx.	%0,0,%2\n"			      \
		      "		bne-	1b"				      \
		      : "=&b" (__val), "=m" (*mem)			      \
		      : "b" (mem), "m" (*mem)				      \
		      : "cr0", "memory");				      \
    __val;								      \
  })

#define __arch_atomic_decrement_if_positive_32(mem) \
  ({ int __val, __tmp;							      \
     __asm __volatile ("1:	lwarx	%0,0,%3\n"			      \
		       "	cmpwi	0,%0,0\n"			      \
		       "	addi	%1,%0,-1\n"			      \
		       "	ble	2f\n"				      \
		       "	stwcx.	%1,0,%3\n"			      \
		       "	bne-	1b\n"				      \
		       "2:	" __ARCH_ACQ_INSTR			      \
		       : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)	      \
		       : "b" (mem), "m" (*mem)				      \
		       : "cr0", "memory");				      \
     __val;								      \
  })

#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_compare_and_exchange_val_32_acq(mem, newval, oldval); \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_compare_and_exchange_val_64_acq(mem, newval, oldval); \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_compare_and_exchange_val_32_rel(mem, newval, oldval); \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_compare_and_exchange_val_64_rel(mem, newval, oldval); \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_exchange_acq(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_32_acq (mem, value);		      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_64_acq (mem, value);		      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_exchange_rel(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_32_rel (mem, value);		      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_64_rel (mem, value);		      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_exchange_and_add(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_and_add_32 (mem, value);	      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_and_add_64 (mem, value);	      \
    else 								      \
       abort ();							      \
    __result;								      \
  })
#define atomic_exchange_and_add_acq(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_and_add_32_acq (mem, value);	      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_and_add_64_acq (mem, value);	      \
    else 								      \
       abort ();							      \
    __result;								      \
  })
#define atomic_exchange_and_add_rel(mem, value) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_exchange_and_add_32_rel (mem, value);	      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_exchange_and_add_64_rel (mem, value);	      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_increment_val(mem) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*(mem)) == 4)						      \
      __result = __arch_atomic_increment_val_32 (mem);			      \
    else if (sizeof (*(mem)) == 8)					      \
      __result = __arch_atomic_increment_val_64 (mem);			      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_increment(mem) ({ atomic_increment_val (mem); (void) 0; })

#define atomic_decrement_val(mem) \
  ({									      \
    __typeof (*(mem)) __result;						      \
    if (sizeof (*(mem)) == 4)						      \
      __result = __arch_atomic_decrement_val_32 (mem);			      \
    else if (sizeof (*(mem)) == 8)					      \
      __result = __arch_atomic_decrement_val_64 (mem);			      \
    else 								      \
       abort ();							      \
    __result;								      \
  })

#define atomic_decrement(mem) ({ atomic_decrement_val (mem); (void) 0; })


/* Decrement *MEM if it is > 0, and return the old value.  */
#define atomic_decrement_if_positive(mem) \
  ({ __typeof (*(mem)) __result;					      \
    if (sizeof (*mem) == 4)						      \
      __result = __arch_atomic_decrement_if_positive_32 (mem);		      \
    else if (sizeof (*mem) == 8)					      \
      __result = __arch_atomic_decrement_if_positive_64 (mem);		      \
    else								      \
       abort ();							      \
    __result;								      \
  })
Commit	Line	Data
94c24227	1	/* Atomic operations. PowerPC Common version.
04277e02	2	Copyright (C) 2003-2019 Free Software Foundation, Inc.
f3c13160 RM	3	This file is part of the GNU C Library.
	4	Contributed by Paul Mackerras <paulus@au.ibm.com>, 2003.
	5
	6	The GNU C Library is free software; you can redistribute it and/or
	7	modify it under the terms of the GNU Lesser General Public
	8	License as published by the Free Software Foundation; either
	9	version 2.1 of the License, or (at your option) any later version.
	10
	11	The GNU C Library is distributed in the hope that it will be useful,
	12	but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	Lesser General Public License for more details.
	15
	16	You should have received a copy of the GNU Lesser General Public
59ba27a6	17	License along with the GNU C Library; if not, see
5a82c748	18	<https://www.gnu.org/licenses/>. */
f3c13160	19
94c24227	20	/*
de071d19	21	* Never include sysdeps/powerpc/atomic-machine.h directly.
949ec764	22	* Alway use include/atomic.h which will include either
de071d19	23	* sysdeps/powerpc/powerpc32/atomic-machine.h
949ec764	24	* or
de071d19	25	* sysdeps/powerpc/powerpc64/atomic-machine.h
94c24227 UD	26	* as appropriate and which in turn include this file.
94c24227 UD	27	*/
f3c13160	28
94c24227	29	#include <stdint.h>
f3c13160 RM	30
	31	typedef int32_t atomic32_t;
	32	typedef uint32_t uatomic32_t;
	33	typedef int_fast32_t atomic_fast32_t;
	34	typedef uint_fast32_t uatomic_fast32_t;
	35
	36	typedef int64_t atomic64_t;
	37	typedef uint64_t uatomic64_t;
	38	typedef int_fast64_t atomic_fast64_t;
	39	typedef uint_fast64_t uatomic_fast64_t;
	40
	41	typedef intptr_t atomicptr_t;
	42	typedef uintptr_t uatomicptr_t;
	43	typedef intmax_t atomic_max_t;
	44	typedef uintmax_t uatomic_max_t;
	45
949ec764 UD	46	/*
949ec764 UD	47	* Powerpc does not have byte and halfword forms of load and reserve and
94c24227 UD	48	* store conditional. So for powerpc we stub out the 8- and 16-bit forms.
94c24227 UD	49	*/
5a3ab2fc	50	#define __arch_compare_and_exchange_bool_8_acq(mem, newval, oldval) \
f3c13160 RM	51	(abort (), 0)
f3c13160 RM	52
5a3ab2fc	53	#define __arch_compare_and_exchange_bool_16_acq(mem, newval, oldval) \
f3c13160	54	(abort (), 0)
ecb2b2fb	55
f3c13160	56	#ifdef UP
f79466a8 UD	57	# define __ARCH_ACQ_INSTR ""
f79466a8 UD	58	# define __ARCH_REL_INSTR ""
f3c13160	59	#else
f79466a8	60	# define __ARCH_ACQ_INSTR "isync"
7158eae4 UD	61	# ifndef __ARCH_REL_INSTR
	62	# define __ARCH_REL_INSTR "sync"
	63	# endif
f3c13160 RM	64	#endif
f3c13160 RM	65
fa6e3bc3 UD	66	#ifndef MUTEX_HINT_ACQ
	67	# define MUTEX_HINT_ACQ
	68	#endif
	69	#ifndef MUTEX_HINT_REL
	70	# define MUTEX_HINT_REL
	71	#endif
	72
94c24227	73	#define atomic_full_barrier() __asm ("sync" ::: "memory")
94c24227	74
7158eae4 UD	75	#define __arch_compare_and_exchange_val_32_acq(mem, newval, oldval) \
	76	({ \
	77	__typeof (*(mem)) __tmp; \
	78	__typeof (mem) __memp = (mem); \
	79	__asm __volatile ( \
fa6e3bc3	80	"1: lwarx %0,0,%1" MUTEX_HINT_ACQ "\n" \
7158eae4 UD	81	" cmpw %0,%2\n" \
	82	" bne 2f\n" \
	83	" stwcx. %3,0,%1\n" \
	84	" bne- 1b\n" \
	85	"2: " __ARCH_ACQ_INSTR \
	86	: "=&r" (__tmp) \
	87	: "b" (__memp), "r" (oldval), "r" (newval) \
	88	: "cr0", "memory"); \
	89	__tmp; \
	90	})
f3c13160	91
7158eae4	92	#define __arch_compare_and_exchange_val_32_rel(mem, newval, oldval) \
6087c485 UD	93	({ \
	94	__typeof (*(mem)) __tmp; \
	95	__typeof (mem) __memp = (mem); \
	96	__asm __volatile (__ARCH_REL_INSTR "\n" \
fa6e3bc3	97	"1: lwarx %0,0,%1" MUTEX_HINT_REL "\n" \
6087c485 UD	98	" cmpw %0,%2\n" \
	99	" bne 2f\n" \
	100	" stwcx. %3,0,%1\n" \
	101	" bne- 1b\n" \
7158eae4	102	"2: " \
6087c485 UD	103	: "=&r" (__tmp) \
	104	: "b" (__memp), "r" (oldval), "r" (newval) \
	105	: "cr0", "memory"); \
	106	__tmp; \
	107	})
	108
7158eae4 UD	109	#define __arch_atomic_exchange_32_acq(mem, value) \
	110	({ \
	111	__typeof (*mem) __val; \
	112	__asm __volatile ( \
fa6e3bc3	113	"1: lwarx %0,0,%2" MUTEX_HINT_ACQ "\n" \
7158eae4 UD	114	" stwcx. %3,0,%2\n" \
	115	" bne- 1b\n" \
	116	" " __ARCH_ACQ_INSTR \
	117	: "=&r" (__val), "=m" (*mem) \
ecb2b2fb	118	: "b" (mem), "r" (value), "m" (*mem) \
7158eae4 UD	119	: "cr0", "memory"); \
	120	__val; \
	121	})
	122
	123	#define __arch_atomic_exchange_32_rel(mem, value) \
3e195d93 RM	124	({ \
	125	__typeof (*mem) __val; \
	126	__asm __volatile (__ARCH_REL_INSTR "\n" \
fa6e3bc3	127	"1: lwarx %0,0,%2" MUTEX_HINT_REL "\n" \
859e708f RM	128	" stwcx. %3,0,%2\n" \
859e708f RM	129	" bne- 1b" \
3e195d93	130	: "=&r" (__val), "=m" (*mem) \
ecb2b2fb	131	: "b" (mem), "r" (value), "m" (*mem) \
7158eae4	132	: "cr0", "memory"); \
3e195d93 RM	133	__val; \
3e195d93 RM	134	})
f79466a8	135
6087c485	136	#define __arch_atomic_exchange_and_add_32(mem, value) \
3e195d93 RM	137	({ \
	138	__typeof (*mem) __val, __tmp; \
	139	__asm __volatile ("1: lwarx %0,0,%3\n" \
859e708f RM	140	" add %1,%0,%4\n" \
	141	" stwcx. %1,0,%3\n" \
	142	" bne- 1b" \
3e195d93	143	: "=&b" (__val), "=&r" (__tmp), "=m" (*mem) \
ecb2b2fb	144	: "b" (mem), "r" (value), "m" (*mem) \
7158eae4	145	: "cr0", "memory"); \
3e195d93 RM	146	__val; \
3e195d93 RM	147	})
448163ba	148
704f7947 AZ	149	#define __arch_atomic_exchange_and_add_32_acq(mem, value) \
	150	({ \
	151	__typeof (*mem) __val, __tmp; \
	152	__asm __volatile ("1: lwarx %0,0,%3" MUTEX_HINT_ACQ "\n" \
	153	" add %1,%0,%4\n" \
	154	" stwcx. %1,0,%3\n" \
	155	" bne- 1b\n" \
	156	__ARCH_ACQ_INSTR \
	157	: "=&b" (__val), "=&r" (__tmp), "=m" (*mem) \
	158	: "b" (mem), "r" (value), "m" (*mem) \
	159	: "cr0", "memory"); \
	160	__val; \
	161	})
	162
	163	#define __arch_atomic_exchange_and_add_32_rel(mem, value) \
	164	({ \
	165	__typeof (*mem) __val, __tmp; \
	166	__asm __volatile (__ARCH_REL_INSTR "\n" \
	167	"1: lwarx %0,0,%3" MUTEX_HINT_REL "\n" \
	168	" add %1,%0,%4\n" \
	169	" stwcx. %1,0,%3\n" \
	170	" bne- 1b" \
	171	: "=&b" (__val), "=&r" (__tmp), "=m" (*mem) \
	172	: "b" (mem), "r" (value), "m" (*mem) \
	173	: "cr0", "memory"); \
	174	__val; \
	175	})
	176
7ba0e52c UD	177	#define __arch_atomic_increment_val_32(mem) \
	178	({ \
	179	__typeof (*(mem)) __val; \
	180	__asm __volatile ("1: lwarx %0,0,%2\n" \
	181	" addi %0,%0,1\n" \
	182	" stwcx. %0,0,%2\n" \
	183	" bne- 1b" \
	184	: "=&b" (__val), "=m" (*mem) \
	185	: "b" (mem), "m" (*mem) \
	186	: "cr0", "memory"); \
	187	__val; \
	188	})
	189
	190	#define __arch_atomic_decrement_val_32(mem) \
	191	({ \
	192	__typeof (*(mem)) __val; \
	193	__asm __volatile ("1: lwarx %0,0,%2\n" \
	194	" subi %0,%0,1\n" \
	195	" stwcx. %0,0,%2\n" \
	196	" bne- 1b" \
	197	: "=&b" (__val), "=m" (*mem) \
	198	: "b" (mem), "m" (*mem) \
	199	: "cr0", "memory"); \
	200	__val; \
	201	})
	202
6087c485	203	#define __arch_atomic_decrement_if_positive_32(mem) \
3171ae99 UD	204	({ int __val, __tmp; \
	205	__asm __volatile ("1: lwarx %0,0,%3\n" \
	206	" cmpwi 0,%0,0\n" \
	207	" addi %1,%0,-1\n" \
	208	" ble 2f\n" \
	209	" stwcx. %1,0,%3\n" \
	210	" bne- 1b\n" \
	211	"2: " __ARCH_ACQ_INSTR \
	212	: "=&b" (__val), "=&r" (__tmp), "=m" (*mem) \
ecb2b2fb	213	: "b" (mem), "m" (*mem) \
7158eae4	214	: "cr0", "memory"); \
3171ae99 UD	215	__val; \
	216	})
	217
6087c485 UD	218	#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
	219	({ \
	220	__typeof (*(mem)) __result; \
	221	if (sizeof (*mem) == 4) \
	222	__result = __arch_compare_and_exchange_val_32_acq(mem, newval, oldval); \
	223	else if (sizeof (*mem) == 8) \
	224	__result = __arch_compare_and_exchange_val_64_acq(mem, newval, oldval); \
	225	else \
	226	abort (); \
	227	__result; \
	228	})
ecb2b2fb	229
7158eae4 UD	230	#define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \
	231	({ \
	232	__typeof (*(mem)) __result; \
	233	if (sizeof (*mem) == 4) \
	234	__result = __arch_compare_and_exchange_val_32_rel(mem, newval, oldval); \
	235	else if (sizeof (*mem) == 8) \
	236	__result = __arch_compare_and_exchange_val_64_rel(mem, newval, oldval); \
	237	else \
	238	abort (); \
	239	__result; \
	240	})
6087c485	241
949ec764	242	#define atomic_exchange_acq(mem, value) \
3e195d93 RM	243	({ \
	244	__typeof (*(mem)) __result; \
	245	if (sizeof (*mem) == 4) \
7158eae4 UD	246	__result = __arch_atomic_exchange_32_acq (mem, value); \
	247	else if (sizeof (*mem) == 8) \
	248	__result = __arch_atomic_exchange_64_acq (mem, value); \
	249	else \
	250	abort (); \
	251	__result; \
	252	})
	253
	254	#define atomic_exchange_rel(mem, value) \
	255	({ \
	256	__typeof (*(mem)) __result; \
	257	if (sizeof (*mem) == 4) \
	258	__result = __arch_atomic_exchange_32_rel (mem, value); \
3e195d93	259	else if (sizeof (*mem) == 8) \
7158eae4	260	__result = __arch_atomic_exchange_64_rel (mem, value); \
3e195d93 RM	261	else \
	262	abort (); \
	263	__result; \
	264	})
f79466a8	265
6087c485	266	#define atomic_exchange_and_add(mem, value) \
3e195d93 RM	267	({ \
	268	__typeof (*(mem)) __result; \
	269	if (sizeof (*mem) == 4) \
6087c485	270	__result = __arch_atomic_exchange_and_add_32 (mem, value); \
3e195d93	271	else if (sizeof (*mem) == 8) \
6087c485	272	__result = __arch_atomic_exchange_and_add_64 (mem, value); \
3e195d93	273	else \
f79466a8	274	abort (); \
3e195d93	275	__result; \
f79466a8	276	})
704f7947 AZ	277	#define atomic_exchange_and_add_acq(mem, value) \
	278	({ \
	279	__typeof (*(mem)) __result; \
	280	if (sizeof (*mem) == 4) \
	281	__result = __arch_atomic_exchange_and_add_32_acq (mem, value); \
	282	else if (sizeof (*mem) == 8) \
	283	__result = __arch_atomic_exchange_and_add_64_acq (mem, value); \
	284	else \
	285	abort (); \
	286	__result; \
	287	})
	288	#define atomic_exchange_and_add_rel(mem, value) \
	289	({ \
	290	__typeof (*(mem)) __result; \
	291	if (sizeof (*mem) == 4) \
	292	__result = __arch_atomic_exchange_and_add_32_rel (mem, value); \
	293	else if (sizeof (*mem) == 8) \
	294	__result = __arch_atomic_exchange_and_add_64_rel (mem, value); \
	295	else \
	296	abort (); \
	297	__result; \
	298	})
f79466a8	299
7ba0e52c UD	300	#define atomic_increment_val(mem) \
	301	({ \
	302	__typeof (*(mem)) __result; \
	303	if (sizeof (*(mem)) == 4) \
	304	__result = __arch_atomic_increment_val_32 (mem); \
	305	else if (sizeof (*(mem)) == 8) \
	306	__result = __arch_atomic_increment_val_64 (mem); \
	307	else \
	308	abort (); \
	309	__result; \
	310	})
	311
	312	#define atomic_increment(mem) ({ atomic_increment_val (mem); (void) 0; })
	313
	314	#define atomic_decrement_val(mem) \
	315	({ \
	316	__typeof (*(mem)) __result; \
	317	if (sizeof (*(mem)) == 4) \
	318	__result = __arch_atomic_decrement_val_32 (mem); \
	319	else if (sizeof (*(mem)) == 8) \
	320	__result = __arch_atomic_decrement_val_64 (mem); \
	321	else \
	322	abort (); \
	323	__result; \
	324	})
	325
	326	#define atomic_decrement(mem) ({ atomic_decrement_val (mem); (void) 0; })
	327
f79466a8 UD	328
f79466a8 UD	329	/* Decrement MEM if it is > 0, and return the old value. /
6087c485	330	#define atomic_decrement_if_positive(mem) \
448163ba	331	({ __typeof (*(mem)) __result; \
3171ae99	332	if (sizeof (*mem) == 4) \
448163ba	333	__result = __arch_atomic_decrement_if_positive_32 (mem); \
3171ae99	334	else if (sizeof (*mem) == 8) \
448163ba RM	335	__result = __arch_atomic_decrement_if_positive_64 (mem); \
448163ba RM	336	else \
f79466a8	337	abort (); \
3171ae99	338	__result; \
f79466a8	339	})