[thirdparty/glibc.git] / sysdeps / x86_64 / bits / atomic.h

/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <stdint.h>
#include <tls.h>	/* For tcbhead_t.  */
#include <libc-internal.h>


typedef int8_t atomic8_t;
typedef uint8_t uatomic8_t;
typedef int_fast8_t atomic_fast8_t;
typedef uint_fast8_t uatomic_fast8_t;

typedef int16_t atomic16_t;
typedef uint16_t uatomic16_t;
typedef int_fast16_t atomic_fast16_t;
typedef uint_fast16_t uatomic_fast16_t;

typedef int32_t atomic32_t;
typedef uint32_t uatomic32_t;
typedef int_fast32_t atomic_fast32_t;
typedef uint_fast32_t uatomic_fast32_t;

typedef int64_t atomic64_t;
typedef uint64_t uatomic64_t;
typedef int_fast64_t atomic_fast64_t;
typedef uint_fast64_t uatomic_fast64_t;

typedef intptr_t atomicptr_t;
typedef uintptr_t uatomicptr_t;
typedef intmax_t atomic_max_t;
typedef uintmax_t uatomic_max_t;


#ifndef LOCK_PREFIX
# ifdef UP
#  define LOCK_PREFIX	/* nothing */
# else
#  define LOCK_PREFIX "lock;"
# endif
#endif

#define __HAVE_64B_ATOMICS 1
#if __GNUC_PREREQ (4, 7)
#define USE_ATOMIC_COMPILER_BUILTINS 1
#else
#define USE_ATOMIC_COMPILER_BUILTINS 0
#endif

#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  __sync_val_compare_and_swap (mem, oldval, newval)
#define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  (! __sync_bool_compare_and_swap (mem, oldval, newval))


#define __arch_c_compare_and_exchange_val_8_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
    __asm __volatile ("cmpl $0, %%fs:%P5\n\t"				      \
		      "je 0f\n\t"					      \
		      "lock\n"						      \
		       "0:\tcmpxchgb %b2, %1"				      \
		       : "=a" (ret), "=m" (*mem)			      \
		       : "q" (newval), "m" (*mem), "0" (oldval),	      \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })

#define __arch_c_compare_and_exchange_val_16_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
    __asm __volatile ("cmpl $0, %%fs:%P5\n\t"				      \
		      "je 0f\n\t"					      \
		      "lock\n"						      \
		       "0:\tcmpxchgw %w2, %1"				      \
		       : "=a" (ret), "=m" (*mem)			      \
		       : "q" (newval), "m" (*mem), "0" (oldval),	      \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })

#define __arch_c_compare_and_exchange_val_32_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
    __asm __volatile ("cmpl $0, %%fs:%P5\n\t"				      \
		      "je 0f\n\t"					      \
		      "lock\n"						      \
		       "0:\tcmpxchgl %2, %1"				      \
		       : "=a" (ret), "=m" (*mem)			      \
		       : "q" (newval), "m" (*mem), "0" (oldval),	      \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })

#define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
     __asm __volatile ("cmpl $0, %%fs:%P5\n\t"				      \
		       "je 0f\n\t"					      \
		       "lock\n"						      \
		       "0:\tcmpxchgq %q2, %1"				      \
		       : "=a" (ret), "=m" (*mem)			      \
		       : "q" ((atomic64_t) cast_to_integer (newval)),	      \
			 "m" (*mem),					      \
			 "0" ((atomic64_t) cast_to_integer (oldval)),	      \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })


/* Note that we need no lock prefix.  */
#define atomic_exchange_acq(mem, newvalue) \
  ({ __typeof (*mem) result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile ("xchgb %b0, %1"				      \
			 : "=q" (result), "=m" (*mem)			      \
			 : "0" (newvalue), "m" (*mem));			      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile ("xchgw %w0, %1"				      \
			 : "=r" (result), "=m" (*mem)			      \
			 : "0" (newvalue), "m" (*mem));			      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile ("xchgl %0, %1"					      \
			 : "=r" (result), "=m" (*mem)			      \
			 : "0" (newvalue), "m" (*mem));			      \
     else								      \
       __asm __volatile ("xchgq %q0, %1"				      \
			 : "=r" (result), "=m" (*mem)			      \
			 : "0" ((atomic64_t) cast_to_integer (newvalue)),     \
			   "m" (*mem));					      \
     result; })


#define __arch_exchange_and_add_body(lock, mem, value)			      \
  ({ __typeof (*mem) result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (lock "xaddb %b0, %1"				      \
			 : "=q" (result), "=m" (*mem)			      \
			 : "0" (value), "m" (*mem),			      \
			   "i" (offsetof (tcbhead_t, multiple_threads)));     \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (lock "xaddw %w0, %1"				      \
			 : "=r" (result), "=m" (*mem)			      \
			 : "0" (value), "m" (*mem),			      \
			   "i" (offsetof (tcbhead_t, multiple_threads)));     \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (lock "xaddl %0, %1"				      \
			 : "=r" (result), "=m" (*mem)			      \
			 : "0" (value), "m" (*mem),			      \
			   "i" (offsetof (tcbhead_t, multiple_threads)));     \
     else								      \
       __asm __volatile (lock "xaddq %q0, %1"				      \
			 : "=r" (result), "=m" (*mem)			      \
			 : "0" ((atomic64_t) cast_to_integer (value)),	      \
			   "m" (*mem),					      \
			   "i" (offsetof (tcbhead_t, multiple_threads)));     \
     result; })

#define atomic_exchange_and_add(mem, value) \
  __sync_fetch_and_add (mem, value)

#define __arch_exchange_and_add_cprefix \
  "cmpl $0, %%fs:%P4\n\tje 0f\n\tlock\n0:\t"

#define catomic_exchange_and_add(mem, value) \
  __arch_exchange_and_add_body (__arch_exchange_and_add_cprefix, mem, value)


#define __arch_add_body(lock, pfx, mem, value)				      \
  do {									      \
    if (__builtin_constant_p (value) && (value) == 1)			      \
      pfx##_increment (mem);						      \
    else if (__builtin_constant_p (value) && (value) == -1)		      \
      pfx##_decrement (mem);						      \
    else if (sizeof (*mem) == 1)					      \
      __asm __volatile (lock "addb %b1, %0"				      \
			: "=m" (*mem)					      \
			: "iq" (value), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "addw %w1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (value), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "addl %1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (value), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      __asm __volatile (lock "addq %q1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" ((atomic64_t) cast_to_integer (value)),	      \
			  "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
  } while (0)

#define atomic_add(mem, value) \
  __arch_add_body (LOCK_PREFIX, atomic, mem, value)

#define __arch_add_cprefix \
  "cmpl $0, %%fs:%P3\n\tje 0f\n\tlock\n0:\t"

#define catomic_add(mem, value) \
  __arch_add_body (__arch_add_cprefix, catomic, mem, value)


#define atomic_add_negative(mem, value) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "addb %b2, %0; sets %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "iq" (value), "m" (*mem));			      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "addw %w2, %0; sets %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" (value), "m" (*mem));			      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "addl %2, %0; sets %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" (value), "m" (*mem));			      \
     else								      \
       __asm __volatile (LOCK_PREFIX "addq %q2, %0; sets %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" ((atomic64_t) cast_to_integer (value)),	      \
			   "m" (*mem));					      \
     __result; })


#define atomic_add_zero(mem, value) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "addb %b2, %0; setz %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "iq" (value), "m" (*mem));			      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "addw %w2, %0; setz %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" (value), "m" (*mem));			      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "addl %2, %0; setz %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" (value), "m" (*mem));			      \
     else								      \
       __asm __volatile (LOCK_PREFIX "addq %q2, %0; setz %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" ((atomic64_t) cast_to_integer (value)),	      \
			   "m" (*mem));					      \
     __result; })


#define __arch_increment_body(lock, mem) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (lock "incb %b0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "incw %w0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "incl %0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      __asm __volatile (lock "incq %q0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
  } while (0)

#define atomic_increment(mem) __arch_increment_body (LOCK_PREFIX, mem)

#define __arch_increment_cprefix \
  "cmpl $0, %%fs:%P2\n\tje 0f\n\tlock\n0:\t"

#define catomic_increment(mem) \
  __arch_increment_body (__arch_increment_cprefix, mem)


#define atomic_increment_and_test(mem) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "incb %b0; sete %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "incw %w0; sete %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "incl %0; sete %1"			      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else								      \
       __asm __volatile (LOCK_PREFIX "incq %q0; sete %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     __result; })


#define __arch_decrement_body(lock, mem) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (lock "decb %b0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "decw %w0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "decl %0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      __asm __volatile (lock "decq %q0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
  } while (0)

#define atomic_decrement(mem) __arch_decrement_body (LOCK_PREFIX, mem)

#define __arch_decrement_cprefix \
  "cmpl $0, %%fs:%P2\n\tje 0f\n\tlock\n0:\t"

#define catomic_decrement(mem) \
  __arch_decrement_body (__arch_decrement_cprefix, mem)


#define atomic_decrement_and_test(mem) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "decb %b0; sete %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "decw %w0; sete %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "decl %0; sete %1"			      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else								      \
       __asm __volatile (LOCK_PREFIX "decq %q0; sete %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     __result; })


#define atomic_bit_set(mem, bit) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (LOCK_PREFIX "orb %b2, %0"			      \
			: "=m" (*mem)					      \
			: "m" (*mem), "iq" (1L << (bit)));		      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (LOCK_PREFIX "orw %w2, %0"			      \
			: "=m" (*mem)					      \
			: "m" (*mem), "ir" (1L << (bit)));		      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (LOCK_PREFIX "orl %2, %0"			      \
			: "=m" (*mem)					      \
			: "m" (*mem), "ir" (1L << (bit)));		      \
    else if (__builtin_constant_p (bit) && (bit) < 32)			      \
      __asm __volatile (LOCK_PREFIX "orq %2, %0"			      \
			: "=m" (*mem)					      \
			: "m" (*mem), "i" (1L << (bit)));		      \
    else								      \
      __asm __volatile (LOCK_PREFIX "orq %q2, %0"			      \
			: "=m" (*mem)					      \
			: "m" (*mem), "r" (1UL << (bit)));		      \
  } while (0)


#define atomic_bit_test_set(mem, bit) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "btsb %3, %1; setc %0"		      \
			 : "=q" (__result), "=m" (*mem)			      \
			 : "m" (*mem), "iq" (bit));			      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "btsw %3, %1; setc %0"		      \
			 : "=q" (__result), "=m" (*mem)			      \
			 : "m" (*mem), "ir" (bit));			      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "btsl %3, %1; setc %0"		      \
			 : "=q" (__result), "=m" (*mem)			      \
			 : "m" (*mem), "ir" (bit));			      \
     else							      	      \
       __asm __volatile (LOCK_PREFIX "btsq %3, %1; setc %0"		      \
			 : "=q" (__result), "=m" (*mem)			      \
			 : "m" (*mem), "ir" (bit));			      \
     __result; })


#define atomic_delay() asm ("rep; nop")


#define __arch_and_body(lock, mem, mask) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (lock "andb %b1, %0"				      \
			: "=m" (*mem)					      \
			: "iq" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "andw %w1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "andl %1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      __asm __volatile (lock "andq %q1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
  } while (0)

#define __arch_cprefix \
  "cmpl $0, %%fs:%P3\n\tje 0f\n\tlock\n0:\t"

#define atomic_and(mem, mask) __arch_and_body (LOCK_PREFIX, mem, mask)

#define catomic_and(mem, mask) __arch_and_body (__arch_cprefix, mem, mask)


#define __arch_or_body(lock, mem, mask)					      \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (lock "orb %b1, %0"				      \
			: "=m" (*mem)					      \
			: "iq" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "orw %w1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "orl %1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      __asm __volatile (lock "orq %q1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
  } while (0)

#define atomic_or(mem, mask) __arch_or_body (LOCK_PREFIX, mem, mask)

#define catomic_or(mem, mask) __arch_or_body (__arch_cprefix, mem, mask)
Commit	Line	Data
d4697bc9	1	/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
c10c099c UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
c10c099c UD	18
c10c099c UD	19	#include <stdint.h>
11bf311e	20	#include <tls.h> /* For tcbhead_t. */
c515fb51	21	#include <libc-internal.h>
c10c099c UD	22
	23
	24	typedef int8_t atomic8_t;
	25	typedef uint8_t uatomic8_t;
	26	typedef int_fast8_t atomic_fast8_t;
	27	typedef uint_fast8_t uatomic_fast8_t;
	28
	29	typedef int16_t atomic16_t;
	30	typedef uint16_t uatomic16_t;
	31	typedef int_fast16_t atomic_fast16_t;
	32	typedef uint_fast16_t uatomic_fast16_t;
	33
	34	typedef int32_t atomic32_t;
	35	typedef uint32_t uatomic32_t;
	36	typedef int_fast32_t atomic_fast32_t;
	37	typedef uint_fast32_t uatomic_fast32_t;
	38
	39	typedef int64_t atomic64_t;
	40	typedef uint64_t uatomic64_t;
	41	typedef int_fast64_t atomic_fast64_t;
	42	typedef uint_fast64_t uatomic_fast64_t;
	43
	44	typedef intptr_t atomicptr_t;
	45	typedef uintptr_t uatomicptr_t;
	46	typedef intmax_t atomic_max_t;
	47	typedef uintmax_t uatomic_max_t;
	48
	49
bd4f43b4	50	#ifndef LOCK_PREFIX
c10c099c	51	# ifdef UP
bd4f43b4	52	# define LOCK_PREFIX /* nothing */
c10c099c	53	# else
bd4f43b4	54	# define LOCK_PREFIX "lock;"
c10c099c UD	55	# endif
	56	#endif
	57
1ea339b6 TR	58	#define __HAVE_64B_ATOMICS 1
	59	#if __GNUC_PREREQ (4, 7)
	60	#define USE_ATOMIC_COMPILER_BUILTINS 1
	61	#else
	62	#define USE_ATOMIC_COMPILER_BUILTINS 0
	63	#endif
c10c099c	64
bc957d53	65	#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
038a1a9f	66	__sync_val_compare_and_swap (mem, oldval, newval)
bc957d53	67	#define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
038a1a9f	68	(! __sync_bool_compare_and_swap (mem, oldval, newval))
11bf311e UD	69
	70
	71	#define __arch_c_compare_and_exchange_val_8_acq(mem, newval, oldval) \
	72	({ __typeof (*mem) ret; \
	73	__asm __volatile ("cmpl $0, %%fs:%P5\n\t" \
	74	"je 0f\n\t" \
	75	"lock\n" \
	76	"0:\tcmpxchgb %b2, %1" \
	77	: "=a" (ret), "=m" (*mem) \
	78	: "q" (newval), "m" (*mem), "0" (oldval), \
	79	"i" (offsetof (tcbhead_t, multiple_threads))); \
	80	ret; })
	81
	82	#define __arch_c_compare_and_exchange_val_16_acq(mem, newval, oldval) \
	83	({ __typeof (*mem) ret; \
	84	__asm __volatile ("cmpl $0, %%fs:%P5\n\t" \
	85	"je 0f\n\t" \
	86	"lock\n" \
	87	"0:\tcmpxchgw %w2, %1" \
	88	: "=a" (ret), "=m" (*mem) \
	89	: "q" (newval), "m" (*mem), "0" (oldval), \
	90	"i" (offsetof (tcbhead_t, multiple_threads))); \
	91	ret; })
	92
	93	#define __arch_c_compare_and_exchange_val_32_acq(mem, newval, oldval) \
	94	({ __typeof (*mem) ret; \
	95	__asm __volatile ("cmpl $0, %%fs:%P5\n\t" \
	96	"je 0f\n\t" \
	97	"lock\n" \
	98	"0:\tcmpxchgl %2, %1" \
	99	: "=a" (ret), "=m" (*mem) \
	100	: "q" (newval), "m" (*mem), "0" (oldval), \
	101	"i" (offsetof (tcbhead_t, multiple_threads))); \
	102	ret; })
	103
	104	#define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
	105	({ __typeof (*mem) ret; \
	106	__asm __volatile ("cmpl $0, %%fs:%P5\n\t" \
	107	"je 0f\n\t" \
	108	"lock\n" \
	109	"0:\tcmpxchgq %q2, %1" \
	110	: "=a" (ret), "=m" (*mem) \
c515fb51 L	111	: "q" ((atomic64_t) cast_to_integer (newval)), \
	112	"m" (*mem), \
	113	"0" ((atomic64_t) cast_to_integer (oldval)), \
11bf311e	114	"i" (offsetof (tcbhead_t, multiple_threads))); \
8099361e UD	115	ret; })
	116
	117
f79466a8	118	/* Note that we need no lock prefix. */
949ec764	119	#define atomic_exchange_acq(mem, newvalue) \
f79466a8 UD	120	({ __typeof (*mem) result; \
	121	if (sizeof (*mem) == 1) \
	122	__asm __volatile ("xchgb %b0, %1" \
a810e68c	123	: "=q" (result), "=m" (*mem) \
abfd53d1	124	: "0" (newvalue), "m" (*mem)); \
f79466a8 UD	125	else if (sizeof (*mem) == 2) \
	126	__asm __volatile ("xchgw %w0, %1" \
	127	: "=r" (result), "=m" (*mem) \
abfd53d1	128	: "0" (newvalue), "m" (*mem)); \
f79466a8 UD	129	else if (sizeof (*mem) == 4) \
	130	__asm __volatile ("xchgl %0, %1" \
	131	: "=r" (result), "=m" (*mem) \
abfd53d1	132	: "0" (newvalue), "m" (*mem)); \
f79466a8 UD	133	else \
	134	__asm __volatile ("xchgq %q0, %1" \
	135	: "=r" (result), "=m" (*mem) \
c515fb51 L	136	: "0" ((atomic64_t) cast_to_integer (newvalue)), \
c515fb51 L	137	"m" (*mem)); \
f79466a8 UD	138	result; })
	139
	140
11bf311e	141	#define __arch_exchange_and_add_body(lock, mem, value) \
c10c099c UD	142	({ __typeof (*mem) result; \
c10c099c UD	143	if (sizeof (*mem) == 1) \
11bf311e	144	__asm __volatile (lock "xaddb %b0, %1" \
a810e68c	145	: "=q" (result), "=m" (*mem) \
11bf311e UD	146	: "0" (value), "m" (*mem), \
11bf311e UD	147	"i" (offsetof (tcbhead_t, multiple_threads))); \
c10c099c	148	else if (sizeof (*mem) == 2) \
11bf311e	149	__asm __volatile (lock "xaddw %w0, %1" \
c10c099c	150	: "=r" (result), "=m" (*mem) \
11bf311e UD	151	: "0" (value), "m" (*mem), \
11bf311e UD	152	"i" (offsetof (tcbhead_t, multiple_threads))); \
c10c099c	153	else if (sizeof (*mem) == 4) \
11bf311e	154	__asm __volatile (lock "xaddl %0, %1" \
c10c099c	155	: "=r" (result), "=m" (*mem) \
11bf311e UD	156	: "0" (value), "m" (*mem), \
11bf311e UD	157	"i" (offsetof (tcbhead_t, multiple_threads))); \
c10c099c	158	else \
11bf311e	159	__asm __volatile (lock "xaddq %q0, %1" \
c10c099c	160	: "=r" (result), "=m" (*mem) \
c515fb51 L	161	: "0" ((atomic64_t) cast_to_integer (value)), \
c515fb51 L	162	"m" (*mem), \
11bf311e	163	"i" (offsetof (tcbhead_t, multiple_threads))); \
c10c099c UD	164	result; })
c10c099c UD	165
bc957d53	166	#define atomic_exchange_and_add(mem, value) \
038a1a9f	167	__sync_fetch_and_add (mem, value)
11bf311e UD	168
	169	#define __arch_exchange_and_add_cprefix \
	170	"cmpl $0, %%fs:%P4\n\tje 0f\n\tlock\n0:\t"
	171
	172	#define catomic_exchange_and_add(mem, value) \
	173	__arch_exchange_and_add_body (__arch_exchange_and_add_cprefix, mem, value)
	174
	175
	176	#define __arch_add_body(lock, pfx, mem, value) \
	177	do { \
	178	if (__builtin_constant_p (value) && (value) == 1) \
	179	pfx##_increment (mem); \
	180	else if (__builtin_constant_p (value) && (value) == -1) \
	181	pfx##_decrement (mem); \
	182	else if (sizeof (*mem) == 1) \
	183	__asm __volatile (lock "addb %b1, %0" \
	184	: "=m" (*mem) \
a810e68c	185	: "iq" (value), "m" (*mem), \
11bf311e UD	186	"i" (offsetof (tcbhead_t, multiple_threads))); \
	187	else if (sizeof (*mem) == 2) \
	188	__asm __volatile (lock "addw %w1, %0" \
	189	: "=m" (*mem) \
	190	: "ir" (value), "m" (*mem), \
	191	"i" (offsetof (tcbhead_t, multiple_threads))); \
	192	else if (sizeof (*mem) == 4) \
	193	__asm __volatile (lock "addl %1, %0" \
	194	: "=m" (*mem) \
	195	: "ir" (value), "m" (*mem), \
	196	"i" (offsetof (tcbhead_t, multiple_threads))); \
	197	else \
	198	__asm __volatile (lock "addq %q1, %0" \
	199	: "=m" (*mem) \
c515fb51 L	200	: "ir" ((atomic64_t) cast_to_integer (value)), \
c515fb51 L	201	"m" (*mem), \
11bf311e UD	202	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e UD	203	} while (0)
c10c099c UD	204
c10c099c UD	205	#define atomic_add(mem, value) \
11bf311e UD	206	__arch_add_body (LOCK_PREFIX, atomic, mem, value)
	207
	208	#define __arch_add_cprefix \
	209	"cmpl $0, %%fs:%P3\n\tje 0f\n\tlock\n0:\t"
	210
	211	#define catomic_add(mem, value) \
	212	__arch_add_body (__arch_add_cprefix, catomic, mem, value)
c10c099c UD	213
	214
	215	#define atomic_add_negative(mem, value) \
	216	({ unsigned char __result; \
	217	if (sizeof (*mem) == 1) \
bd4f43b4	218	__asm __volatile (LOCK_PREFIX "addb %b2, %0; sets %1" \
c10c099c	219	: "=m" (*mem), "=qm" (__result) \
a810e68c	220	: "iq" (value), "m" (*mem)); \
c10c099c	221	else if (sizeof (*mem) == 2) \
bd4f43b4	222	__asm __volatile (LOCK_PREFIX "addw %w2, %0; sets %1" \
c10c099c	223	: "=m" (*mem), "=qm" (__result) \
abfd53d1	224	: "ir" (value), "m" (*mem)); \
c10c099c	225	else if (sizeof (*mem) == 4) \
bd4f43b4	226	__asm __volatile (LOCK_PREFIX "addl %2, %0; sets %1" \
c10c099c	227	: "=m" (*mem), "=qm" (__result) \
abfd53d1	228	: "ir" (value), "m" (*mem)); \
c10c099c	229	else \
bd4f43b4	230	__asm __volatile (LOCK_PREFIX "addq %q2, %0; sets %1" \
c10c099c	231	: "=m" (*mem), "=qm" (__result) \
c515fb51 L	232	: "ir" ((atomic64_t) cast_to_integer (value)), \
c515fb51 L	233	"m" (*mem)); \
c10c099c UD	234	__result; })
	235
	236
	237	#define atomic_add_zero(mem, value) \
	238	({ unsigned char __result; \
	239	if (sizeof (*mem) == 1) \
bd4f43b4	240	__asm __volatile (LOCK_PREFIX "addb %b2, %0; setz %1" \
c10c099c	241	: "=m" (*mem), "=qm" (__result) \
a810e68c	242	: "iq" (value), "m" (*mem)); \
c10c099c	243	else if (sizeof (*mem) == 2) \
bd4f43b4	244	__asm __volatile (LOCK_PREFIX "addw %w2, %0; setz %1" \
c10c099c	245	: "=m" (*mem), "=qm" (__result) \
abfd53d1	246	: "ir" (value), "m" (*mem)); \
c10c099c	247	else if (sizeof (*mem) == 4) \
bd4f43b4	248	__asm __volatile (LOCK_PREFIX "addl %2, %0; setz %1" \
c10c099c	249	: "=m" (*mem), "=qm" (__result) \
abfd53d1	250	: "ir" (value), "m" (*mem)); \
c10c099c	251	else \
bd4f43b4	252	__asm __volatile (LOCK_PREFIX "addq %q2, %0; setz %1" \
c10c099c	253	: "=m" (*mem), "=qm" (__result) \
c515fb51 L	254	: "ir" ((atomic64_t) cast_to_integer (value)), \
c515fb51 L	255	"m" (*mem)); \
c10c099c UD	256	__result; })
	257
	258
11bf311e UD	259	#define __arch_increment_body(lock, mem) \
	260	do { \
	261	if (sizeof (*mem) == 1) \
	262	__asm __volatile (lock "incb %b0" \
	263	: "=m" (*mem) \
	264	: "m" (*mem), \
	265	"i" (offsetof (tcbhead_t, multiple_threads))); \
	266	else if (sizeof (*mem) == 2) \
	267	__asm __volatile (lock "incw %w0" \
	268	: "=m" (*mem) \
	269	: "m" (*mem), \
	270	"i" (offsetof (tcbhead_t, multiple_threads))); \
	271	else if (sizeof (*mem) == 4) \
	272	__asm __volatile (lock "incl %0" \
	273	: "=m" (*mem) \
	274	: "m" (*mem), \
	275	"i" (offsetof (tcbhead_t, multiple_threads))); \
	276	else \
	277	__asm __volatile (lock "incq %q0" \
	278	: "=m" (*mem) \
	279	: "m" (*mem), \
	280	"i" (offsetof (tcbhead_t, multiple_threads))); \
	281	} while (0)
	282
	283	#define atomic_increment(mem) __arch_increment_body (LOCK_PREFIX, mem)
	284
	285	#define __arch_increment_cprefix \
	286	"cmpl $0, %%fs:%P2\n\tje 0f\n\tlock\n0:\t"
	287
	288	#define catomic_increment(mem) \
	289	__arch_increment_body (__arch_increment_cprefix, mem)
8099361e	290
c10c099c UD	291
	292	#define atomic_increment_and_test(mem) \
	293	({ unsigned char __result; \
	294	if (sizeof (*mem) == 1) \
bd4f43b4	295	__asm __volatile (LOCK_PREFIX "incb %b0; sete %1" \
c10c099c	296	: "=m" (*mem), "=qm" (__result) \
abfd53d1	297	: "m" (*mem)); \
c10c099c	298	else if (sizeof (*mem) == 2) \
bd4f43b4	299	__asm __volatile (LOCK_PREFIX "incw %w0; sete %1" \
c10c099c	300	: "=m" (*mem), "=qm" (__result) \
abfd53d1	301	: "m" (*mem)); \
c10c099c	302	else if (sizeof (*mem) == 4) \
bd4f43b4	303	__asm __volatile (LOCK_PREFIX "incl %0; sete %1" \
c10c099c	304	: "=m" (*mem), "=qm" (__result) \
abfd53d1	305	: "m" (*mem)); \
c10c099c	306	else \
bd4f43b4	307	__asm __volatile (LOCK_PREFIX "incq %q0; sete %1" \
c10c099c	308	: "=m" (*mem), "=qm" (__result) \
abfd53d1	309	: "m" (*mem)); \
c10c099c UD	310	__result; })
	311
	312
11bf311e UD	313	#define __arch_decrement_body(lock, mem) \
	314	do { \
	315	if (sizeof (*mem) == 1) \
	316	__asm __volatile (lock "decb %b0" \
	317	: "=m" (*mem) \
	318	: "m" (*mem), \
	319	"i" (offsetof (tcbhead_t, multiple_threads))); \
	320	else if (sizeof (*mem) == 2) \
	321	__asm __volatile (lock "decw %w0" \
	322	: "=m" (*mem) \
	323	: "m" (*mem), \
	324	"i" (offsetof (tcbhead_t, multiple_threads))); \
	325	else if (sizeof (*mem) == 4) \
	326	__asm __volatile (lock "decl %0" \
	327	: "=m" (*mem) \
	328	: "m" (*mem), \
	329	"i" (offsetof (tcbhead_t, multiple_threads))); \
	330	else \
	331	__asm __volatile (lock "decq %q0" \
	332	: "=m" (*mem) \
	333	: "m" (*mem), \
	334	"i" (offsetof (tcbhead_t, multiple_threads))); \
	335	} while (0)
	336
	337	#define atomic_decrement(mem) __arch_decrement_body (LOCK_PREFIX, mem)
	338
	339	#define __arch_decrement_cprefix \
	340	"cmpl $0, %%fs:%P2\n\tje 0f\n\tlock\n0:\t"
	341
	342	#define catomic_decrement(mem) \
	343	__arch_decrement_body (__arch_decrement_cprefix, mem)
8099361e	344
c10c099c UD	345
	346	#define atomic_decrement_and_test(mem) \
	347	({ unsigned char __result; \
	348	if (sizeof (*mem) == 1) \
bd4f43b4	349	__asm __volatile (LOCK_PREFIX "decb %b0; sete %1" \
c10c099c	350	: "=m" (*mem), "=qm" (__result) \
abfd53d1	351	: "m" (*mem)); \
c10c099c	352	else if (sizeof (*mem) == 2) \
bd4f43b4	353	__asm __volatile (LOCK_PREFIX "decw %w0; sete %1" \
c10c099c	354	: "=m" (*mem), "=qm" (__result) \
abfd53d1	355	: "m" (*mem)); \
c10c099c	356	else if (sizeof (*mem) == 4) \
bd4f43b4	357	__asm __volatile (LOCK_PREFIX "decl %0; sete %1" \
c10c099c	358	: "=m" (*mem), "=qm" (__result) \
abfd53d1	359	: "m" (*mem)); \
c10c099c	360	else \
bd4f43b4	361	__asm __volatile (LOCK_PREFIX "decq %q0; sete %1" \
c10c099c	362	: "=m" (*mem), "=qm" (__result) \
abfd53d1	363	: "m" (*mem)); \
c10c099c UD	364	__result; })
	365
	366
	367	#define atomic_bit_set(mem, bit) \
11bf311e UD	368	do { \
	369	if (sizeof (*mem) == 1) \
	370	__asm __volatile (LOCK_PREFIX "orb %b2, %0" \
	371	: "=m" (*mem) \
a810e68c	372	: "m" (*mem), "iq" (1L << (bit))); \
11bf311e UD	373	else if (sizeof (*mem) == 2) \
	374	__asm __volatile (LOCK_PREFIX "orw %w2, %0" \
	375	: "=m" (*mem) \
	376	: "m" (*mem), "ir" (1L << (bit))); \
	377	else if (sizeof (*mem) == 4) \
	378	__asm __volatile (LOCK_PREFIX "orl %2, %0" \
	379	: "=m" (*mem) \
	380	: "m" (*mem), "ir" (1L << (bit))); \
	381	else if (__builtin_constant_p (bit) && (bit) < 32) \
	382	__asm __volatile (LOCK_PREFIX "orq %2, %0" \
	383	: "=m" (*mem) \
	384	: "m" (*mem), "i" (1L << (bit))); \
	385	else \
	386	__asm __volatile (LOCK_PREFIX "orq %q2, %0" \
	387	: "=m" (*mem) \
	388	: "m" (*mem), "r" (1UL << (bit))); \
	389	} while (0)
c10c099c UD	390
	391
	392	#define atomic_bit_test_set(mem, bit) \
	393	({ unsigned char __result; \
	394	if (sizeof (*mem) == 1) \
bd4f43b4	395	__asm __volatile (LOCK_PREFIX "btsb %3, %1; setc %0" \
c10c099c	396	: "=q" (__result), "=m" (*mem) \
a810e68c	397	: "m" (*mem), "iq" (bit)); \
c10c099c	398	else if (sizeof (*mem) == 2) \
bd4f43b4	399	__asm __volatile (LOCK_PREFIX "btsw %3, %1; setc %0" \
c10c099c	400	: "=q" (__result), "=m" (*mem) \
002ff853	401	: "m" (*mem), "ir" (bit)); \
c10c099c	402	else if (sizeof (*mem) == 4) \
bd4f43b4	403	__asm __volatile (LOCK_PREFIX "btsl %3, %1; setc %0" \
c10c099c	404	: "=q" (__result), "=m" (*mem) \
002ff853	405	: "m" (*mem), "ir" (bit)); \
c10c099c	406	else \
bd4f43b4	407	__asm __volatile (LOCK_PREFIX "btsq %3, %1; setc %0" \
c10c099c	408	: "=q" (__result), "=m" (*mem) \
002ff853	409	: "m" (*mem), "ir" (bit)); \
c10c099c	410	__result; })
f377d022 UD	411
	412
	413	#define atomic_delay() asm ("rep; nop")
11bf311e UD	414
11bf311e UD	415
6c03cd11	416	#define __arch_and_body(lock, mem, mask) \
11bf311e UD	417	do { \
11bf311e UD	418	if (sizeof (*mem) == 1) \
6c03cd11	419	__asm __volatile (lock "andb %b1, %0" \
11bf311e	420	: "=m" (*mem) \
6c03cd11 UD	421	: "iq" (mask), "m" (*mem), \
6c03cd11 UD	422	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e	423	else if (sizeof (*mem) == 2) \
6c03cd11	424	__asm __volatile (lock "andw %w1, %0" \
11bf311e	425	: "=m" (*mem) \
6c03cd11 UD	426	: "ir" (mask), "m" (*mem), \
6c03cd11 UD	427	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e	428	else if (sizeof (*mem) == 4) \
6c03cd11	429	__asm __volatile (lock "andl %1, %0" \
11bf311e	430	: "=m" (*mem) \
6c03cd11 UD	431	: "ir" (mask), "m" (*mem), \
6c03cd11 UD	432	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e	433	else \
6c03cd11	434	__asm __volatile (lock "andq %q1, %0" \
11bf311e	435	: "=m" (*mem) \
6c03cd11 UD	436	: "ir" (mask), "m" (*mem), \
6c03cd11 UD	437	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e UD	438	} while (0)
11bf311e UD	439
6c03cd11 UD	440	#define __arch_cprefix \
	441	"cmpl $0, %%fs:%P3\n\tje 0f\n\tlock\n0:\t"
	442
	443	#define atomic_and(mem, mask) __arch_and_body (LOCK_PREFIX, mem, mask)
	444
	445	#define catomic_and(mem, mask) __arch_and_body (__arch_cprefix, mem, mask)
	446
11bf311e UD	447
	448	#define __arch_or_body(lock, mem, mask) \
	449	do { \
	450	if (sizeof (*mem) == 1) \
a810e68c	451	__asm __volatile (lock "orb %b1, %0" \
11bf311e	452	: "=m" (*mem) \
a810e68c	453	: "iq" (mask), "m" (*mem), \
11bf311e UD	454	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e UD	455	else if (sizeof (*mem) == 2) \
77151937	456	__asm __volatile (lock "orw %w1, %0" \
11bf311e UD	457	: "=m" (*mem) \
	458	: "ir" (mask), "m" (*mem), \
	459	"i" (offsetof (tcbhead_t, multiple_threads))); \
	460	else if (sizeof (*mem) == 4) \
	461	__asm __volatile (lock "orl %1, %0" \
	462	: "=m" (*mem) \
	463	: "ir" (mask), "m" (*mem), \
	464	"i" (offsetof (tcbhead_t, multiple_threads))); \
	465	else \
77151937	466	__asm __volatile (lock "orq %q1, %0" \
11bf311e UD	467	: "=m" (*mem) \
	468	: "ir" (mask), "m" (*mem), \
	469	"i" (offsetof (tcbhead_t, multiple_threads))); \
	470	} while (0)
	471
	472	#define atomic_or(mem, mask) __arch_or_body (LOCK_PREFIX, mem, mask)
	473
6c03cd11	474	#define catomic_or(mem, mask) __arch_or_body (__arch_cprefix, mem, mask)