[thirdparty/glibc.git] / sysdeps / i386 / atomic-machine.h

/* Copyright (C) 2002-2018 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <stdint.h>
#include <tls.h>	/* For tcbhead_t.  */


typedef int8_t atomic8_t;
typedef uint8_t uatomic8_t;
typedef int_fast8_t atomic_fast8_t;
typedef uint_fast8_t uatomic_fast8_t;

typedef int16_t atomic16_t;
typedef uint16_t uatomic16_t;
typedef int_fast16_t atomic_fast16_t;
typedef uint_fast16_t uatomic_fast16_t;

typedef int32_t atomic32_t;
typedef uint32_t uatomic32_t;
typedef int_fast32_t atomic_fast32_t;
typedef uint_fast32_t uatomic_fast32_t;

typedef int64_t atomic64_t;
typedef uint64_t uatomic64_t;
typedef int_fast64_t atomic_fast64_t;
typedef uint_fast64_t uatomic_fast64_t;

typedef intptr_t atomicptr_t;
typedef uintptr_t uatomicptr_t;
typedef intmax_t atomic_max_t;
typedef uintmax_t uatomic_max_t;


#ifndef LOCK_PREFIX
# ifdef UP
#  define LOCK_PREFIX	/* nothing */
# else
#  define LOCK_PREFIX "lock;"
# endif
#endif

#define __HAVE_64B_ATOMICS 0
#define USE_ATOMIC_COMPILER_BUILTINS 0
#define ATOMIC_EXCHANGE_USES_CAS 0


#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  __sync_val_compare_and_swap (mem, oldval, newval)
#define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  (! __sync_bool_compare_and_swap (mem, oldval, newval))


#define __arch_c_compare_and_exchange_val_8_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
     __asm __volatile ("cmpl $0, %%gs:%P5\n\t"                                \
                       "je 0f\n\t"                                            \
                       "lock\n"                                               \
                       "0:\tcmpxchgb %b2, %1"				      \
		       : "=a" (ret), "=m" (*mem)			      \
		       : "q" (newval), "m" (*mem), "0" (oldval),	      \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })

#define __arch_c_compare_and_exchange_val_16_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
     __asm __volatile ("cmpl $0, %%gs:%P5\n\t"                                \
                       "je 0f\n\t"                                            \
                       "lock\n"                                               \
                       "0:\tcmpxchgw %w2, %1"				      \
		       : "=a" (ret), "=m" (*mem)			      \
		       : "r" (newval), "m" (*mem), "0" (oldval),	      \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })

#define __arch_c_compare_and_exchange_val_32_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
     __asm __volatile ("cmpl $0, %%gs:%P5\n\t"                                \
                       "je 0f\n\t"                                            \
                       "lock\n"                                               \
                       "0:\tcmpxchgl %2, %1"				      \
		       : "=a" (ret), "=m" (*mem)			      \
		       : "r" (newval), "m" (*mem), "0" (oldval),	      \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })

/* XXX We do not really need 64-bit compare-and-exchange.  At least
   not in the moment.  Using it would mean causing portability
   problems since not many other 32-bit architectures have support for
   such an operation.  So don't define any code for now.  If it is
   really going to be used the code below can be used on Intel Pentium
   and later, but NOT on i486.  */
#if 1
# define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval)	      \
  ({ __typeof (*mem) ret = *(mem);					      \
     abort ();								      \
     ret = (newval);							      \
     ret = (oldval);							      \
     ret; })
# define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval)	      \
  ({ __typeof (*mem) ret = *(mem);					      \
     abort ();								      \
     ret = (newval);							      \
     ret = (oldval);							      \
     ret; })
#else
# ifdef __PIC__
#  define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
     __asm __volatile ("xchgl %2, %%ebx\n\t"				      \
		       LOCK_PREFIX "cmpxchg8b %1\n\t"			      \
		       "xchgl %2, %%ebx"				      \
		       : "=A" (ret), "=m" (*mem)			      \
		       : "DS" (((unsigned long long int) (newval))	      \
			       & 0xffffffff),				      \
			 "c" (((unsigned long long int) (newval)) >> 32),     \
			 "m" (*mem), "a" (((unsigned long long int) (oldval)) \
					  & 0xffffffff),		      \
			 "d" (((unsigned long long int) (oldval)) >> 32));    \
     ret; })

#  define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
     __asm __volatile ("xchgl %2, %%ebx\n\t"				      \
		       "cmpl $0, %%gs:%P7\n\t"				      \
		       "je 0f\n\t"					      \
		       "lock\n"						      \
		       "0:\tcmpxchg8b %1\n\t"				      \
		       "xchgl %2, %%ebx"				      \
		       : "=A" (ret), "=m" (*mem)			      \
		       : "DS" (((unsigned long long int) (newval))	      \
			       & 0xffffffff),				      \
			 "c" (((unsigned long long int) (newval)) >> 32),     \
			 "m" (*mem), "a" (((unsigned long long int) (oldval)) \
					  & 0xffffffff),		      \
			 "d" (((unsigned long long int) (oldval)) >> 32),     \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })
# else
#  define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
     __asm __volatile (LOCK_PREFIX "cmpxchg8b %1"			      \
		       : "=A" (ret), "=m" (*mem)			      \
		       : "b" (((unsigned long long int) (newval))	      \
			      & 0xffffffff),				      \
			 "c" (((unsigned long long int) (newval)) >> 32),     \
			 "m" (*mem), "a" (((unsigned long long int) (oldval)) \
					  & 0xffffffff),		      \
			 "d" (((unsigned long long int) (oldval)) >> 32));    \
     ret; })

#  define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
  ({ __typeof (*mem) ret;						      \
     __asm __volatile ("cmpl $0, %%gs:%P7\n\t"				      \
		       "je 0f\n\t"					      \
		       "lock\n"						      \
		       "0:\tcmpxchg8b %1"				      \
		       : "=A" (ret), "=m" (*mem)			      \
		       : "b" (((unsigned long long int) (newval))	      \
			      & 0xffffffff),				      \
			 "c" (((unsigned long long int) (newval)) >> 32),     \
			 "m" (*mem), "a" (((unsigned long long int) (oldval)) \
					  & 0xffffffff),		      \
			 "d" (((unsigned long long int) (oldval)) >> 32),     \
			 "i" (offsetof (tcbhead_t, multiple_threads)));	      \
     ret; })
# endif
#endif


/* Note that we need no lock prefix.  */
#define atomic_exchange_acq(mem, newvalue) \
  ({ __typeof (*mem) result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile ("xchgb %b0, %1"				      \
			 : "=q" (result), "=m" (*mem)			      \
			 : "0" (newvalue), "m" (*mem));			      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile ("xchgw %w0, %1"				      \
			 : "=r" (result), "=m" (*mem)			      \
			 : "0" (newvalue), "m" (*mem));			      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile ("xchgl %0, %1"					      \
			 : "=r" (result), "=m" (*mem)			      \
			 : "0" (newvalue), "m" (*mem));			      \
     else								      \
       {								      \
	 result = 0;							      \
	 abort ();							      \
       }								      \
     result; })


#define __arch_exchange_and_add_body(lock, pfx, mem, value) \
  ({ __typeof (*mem) __result;						      \
     __typeof (value) __addval = (value);				      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (lock "xaddb %b0, %1"				      \
			 : "=q" (__result), "=m" (*mem)			      \
			 : "0" (__addval), "m" (*mem),			      \
			   "i" (offsetof (tcbhead_t, multiple_threads)));     \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (lock "xaddw %w0, %1"				      \
			 : "=r" (__result), "=m" (*mem)			      \
			 : "0" (__addval), "m" (*mem),			      \
			   "i" (offsetof (tcbhead_t, multiple_threads)));     \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (lock "xaddl %0, %1"				      \
			 : "=r" (__result), "=m" (*mem)			      \
			 : "0" (__addval), "m" (*mem),			      \
			   "i" (offsetof (tcbhead_t, multiple_threads)));     \
     else								      \
       {								      \
	 __typeof (mem) __memp = (mem);					      \
	 __typeof (*mem) __tmpval;					      \
	 __result = *__memp;						      \
	 do								      \
	   __tmpval = __result;						      \
	 while ((__result = pfx##_compare_and_exchange_val_64_acq	      \
		 (__memp, __result + __addval, __result)) == __tmpval);	      \
       }								      \
     __result; })

#define atomic_exchange_and_add(mem, value) \
  __sync_fetch_and_add (mem, value)

#define __arch_exchange_and_add_cprefix \
  "cmpl $0, %%gs:%P4\n\tje 0f\n\tlock\n0:\t"

#define catomic_exchange_and_add(mem, value) \
  __arch_exchange_and_add_body (__arch_exchange_and_add_cprefix, __arch_c,    \
				mem, value)


#define __arch_add_body(lock, pfx, mem, value) \
  do {									      \
    if (__builtin_constant_p (value) && (value) == 1)			      \
      atomic_increment (mem);						      \
    else if (__builtin_constant_p (value) && (value) == -1)		      \
      atomic_decrement (mem);						      \
    else if (sizeof (*mem) == 1)					      \
      __asm __volatile (lock "addb %b1, %0"				      \
			: "=m" (*mem)					      \
			: "iq" (value), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "addw %w1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (value), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "addl %1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (value), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      {									      \
	__typeof (value) __addval = (value);				      \
	__typeof (mem) __memp = (mem);					      \
	__typeof (*mem) __oldval = *__memp;				      \
	__typeof (*mem) __tmpval;					      \
	do								      \
	  __tmpval = __oldval;						      \
	while ((__oldval = pfx##_compare_and_exchange_val_64_acq	      \
		(__memp, __oldval + __addval, __oldval)) == __tmpval);	      \
      }									      \
  } while (0)

#define atomic_add(mem, value) \
  __arch_add_body (LOCK_PREFIX, __arch, mem, value)

#define __arch_add_cprefix \
  "cmpl $0, %%gs:%P3\n\tje 0f\n\tlock\n0:\t"

#define catomic_add(mem, value) \
  __arch_add_body (__arch_add_cprefix, __arch_c, mem, value)


#define atomic_add_negative(mem, value) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "addb %b2, %0; sets %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "iq" (value), "m" (*mem));			      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "addw %w2, %0; sets %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" (value), "m" (*mem));			      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "addl %2, %0; sets %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" (value), "m" (*mem));			      \
     else								      \
       abort ();							      \
     __result; })


#define atomic_add_zero(mem, value) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "addb %b2, %0; setz %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "iq" (value), "m" (*mem));			      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "addw %w2, %0; setz %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" (value), "m" (*mem));			      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "addl %2, %0; setz %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "ir" (value), "m" (*mem));			      \
     else								      \
       abort ();							      \
     __result; })


#define __arch_increment_body(lock,  pfx, mem) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (lock "incb %b0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "incw %w0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "incl %0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      {									      \
	__typeof (mem) __memp = (mem);					      \
	__typeof (*mem) __oldval = *__memp;				      \
	__typeof (*mem) __tmpval;					      \
	do								      \
	  __tmpval = __oldval;						      \
	while ((__oldval = pfx##_compare_and_exchange_val_64_acq	      \
		(__memp, __oldval + 1, __oldval)) == __tmpval);		      \
      }									      \
  } while (0)

#define atomic_increment(mem) __arch_increment_body (LOCK_PREFIX, __arch, mem)

#define __arch_increment_cprefix \
  "cmpl $0, %%gs:%P2\n\tje 0f\n\tlock\n0:\t"

#define catomic_increment(mem) \
  __arch_increment_body (__arch_increment_cprefix, __arch_c, mem)


#define atomic_increment_and_test(mem) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "incb %0; sete %b1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "incw %0; sete %w1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "incl %0; sete %1"			      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else								      \
       abort ();							      \
     __result; })


#define __arch_decrement_body(lock, pfx, mem) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (lock "decb %b0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "decw %w0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "decl %0"					      \
			: "=m" (*mem)					      \
			: "m" (*mem),					      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      {									      \
	__typeof (mem) __memp = (mem);					      \
	__typeof (*mem) __oldval = *__memp;				      \
	__typeof (*mem) __tmpval;					      \
	do								      \
	  __tmpval = __oldval;						      \
	while ((__oldval = pfx##_compare_and_exchange_val_64_acq	      \
		(__memp, __oldval - 1, __oldval)) == __tmpval); 	      \
      }									      \
  } while (0)

#define atomic_decrement(mem) __arch_decrement_body (LOCK_PREFIX, __arch, mem)

#define __arch_decrement_cprefix \
  "cmpl $0, %%gs:%P2\n\tje 0f\n\tlock\n0:\t"

#define catomic_decrement(mem) \
  __arch_decrement_body (__arch_decrement_cprefix, __arch_c, mem)


#define atomic_decrement_and_test(mem) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "decb %b0; sete %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "decw %w0; sete %1"		      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "decl %0; sete %1"			      \
			 : "=m" (*mem), "=qm" (__result)		      \
			 : "m" (*mem));					      \
     else								      \
       abort ();							      \
     __result; })


#define atomic_bit_set(mem, bit) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (LOCK_PREFIX "orb %b2, %0"			      \
			: "=m" (*mem)					      \
			: "m" (*mem), "iq" (1 << (bit)));		      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (LOCK_PREFIX "orw %w2, %0"			      \
			: "=m" (*mem)					      \
			: "m" (*mem), "ir" (1 << (bit)));		      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (LOCK_PREFIX "orl %2, %0"			      \
			: "=m" (*mem)					      \
			: "m" (*mem), "ir" (1 << (bit)));		      \
    else								      \
      abort ();								      \
  } while (0)


#define atomic_bit_test_set(mem, bit) \
  ({ unsigned char __result;						      \
     if (sizeof (*mem) == 1)						      \
       __asm __volatile (LOCK_PREFIX "btsb %3, %1; setc %0"		      \
			 : "=q" (__result), "=m" (*mem)			      \
			 : "m" (*mem), "ir" (bit));			      \
     else if (sizeof (*mem) == 2)					      \
       __asm __volatile (LOCK_PREFIX "btsw %3, %1; setc %0"		      \
			 : "=q" (__result), "=m" (*mem)			      \
			 : "m" (*mem), "ir" (bit));			      \
     else if (sizeof (*mem) == 4)					      \
       __asm __volatile (LOCK_PREFIX "btsl %3, %1; setc %0"		      \
			 : "=q" (__result), "=m" (*mem)			      \
			 : "m" (*mem), "ir" (bit));			      \
     else							      	      \
       abort ();							      \
     __result; })


#define atomic_spin_nop() asm ("rep; nop")


#define __arch_and_body(lock, mem, mask) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (lock "andb %b1, %0"				      \
			: "=m" (*mem)					      \
			: "iq" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "andw %w1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "andl %1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      abort ();								      \
  } while (0)

#define __arch_cprefix \
  "cmpl $0, %%gs:%P3\n\tje 0f\n\tlock\n0:\t"

#define atomic_and(mem, mask) __arch_and_body (LOCK_PREFIX, mem, mask)

#define catomic_and(mem, mask) __arch_and_body (__arch_cprefix, mem, mask)


#define __arch_or_body(lock, mem, mask) \
  do {									      \
    if (sizeof (*mem) == 1)						      \
      __asm __volatile (lock "orb %b1, %0"				      \
			: "=m" (*mem)					      \
			: "iq" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 2)					      \
      __asm __volatile (lock "orw %w1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else if (sizeof (*mem) == 4)					      \
      __asm __volatile (lock "orl %1, %0"				      \
			: "=m" (*mem)					      \
			: "ir" (mask), "m" (*mem),			      \
			  "i" (offsetof (tcbhead_t, multiple_threads)));      \
    else								      \
      abort ();								      \
  } while (0)

#define atomic_or(mem, mask) __arch_or_body (LOCK_PREFIX, mem, mask)

#define catomic_or(mem, mask) __arch_or_body (__arch_cprefix, mem, mask)

/* We don't use mfence because it is supposedly slower due to having to
   provide stronger guarantees (e.g., regarding self-modifying code).  */
#define atomic_full_barrier() \
    __asm __volatile (LOCK_PREFIX "orl $0, (%%esp)" ::: "memory")
#define atomic_read_barrier() __asm ("" ::: "memory")
#define atomic_write_barrier() __asm ("" ::: "memory")
Commit	Line	Data
688903eb	1	/* Copyright (C) 2002-2018 Free Software Foundation, Inc.
b1473c32 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
b1473c32 UD	18
b1473c32 UD	19	#include <stdint.h>
11bf311e	20	#include <tls.h> /* For tcbhead_t. */
b1473c32 UD	21
	22
	23	typedef int8_t atomic8_t;
	24	typedef uint8_t uatomic8_t;
	25	typedef int_fast8_t atomic_fast8_t;
	26	typedef uint_fast8_t uatomic_fast8_t;
	27
	28	typedef int16_t atomic16_t;
	29	typedef uint16_t uatomic16_t;
	30	typedef int_fast16_t atomic_fast16_t;
	31	typedef uint_fast16_t uatomic_fast16_t;
	32
	33	typedef int32_t atomic32_t;
	34	typedef uint32_t uatomic32_t;
	35	typedef int_fast32_t atomic_fast32_t;
	36	typedef uint_fast32_t uatomic_fast32_t;
	37
	38	typedef int64_t atomic64_t;
	39	typedef uint64_t uatomic64_t;
	40	typedef int_fast64_t atomic_fast64_t;
	41	typedef uint_fast64_t uatomic_fast64_t;
	42
	43	typedef intptr_t atomicptr_t;
	44	typedef uintptr_t uatomicptr_t;
	45	typedef intmax_t atomic_max_t;
	46	typedef uintmax_t uatomic_max_t;
	47
	48
bd4f43b4	49	#ifndef LOCK_PREFIX
b1473c32	50	# ifdef UP
bd4f43b4	51	# define LOCK_PREFIX /* nothing */
b1473c32	52	# else
bd4f43b4	53	# define LOCK_PREFIX "lock;"
b1473c32 UD	54	# endif
	55	#endif
	56
1ea339b6 TR	57	#define __HAVE_64B_ATOMICS 0
1ea339b6 TR	58	#define USE_ATOMIC_COMPILER_BUILTINS 0
12d2dd70	59	#define ATOMIC_EXCHANGE_USES_CAS 0
1ea339b6	60
b1473c32	61
bc957d53	62	#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
038a1a9f	63	__sync_val_compare_and_swap (mem, oldval, newval)
bc957d53	64	#define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
038a1a9f	65	(! __sync_bool_compare_and_swap (mem, oldval, newval))
b1473c32	66
11bf311e UD	67
	68	#define __arch_c_compare_and_exchange_val_8_acq(mem, newval, oldval) \
	69	({ __typeof (*mem) ret; \
	70	__asm __volatile ("cmpl $0, %%gs:%P5\n\t" \
	71	"je 0f\n\t" \
	72	"lock\n" \
	73	"0:\tcmpxchgb %b2, %1" \
	74	: "=a" (ret), "=m" (*mem) \
	75	: "q" (newval), "m" (*mem), "0" (oldval), \
	76	"i" (offsetof (tcbhead_t, multiple_threads))); \
	77	ret; })
	78
	79	#define __arch_c_compare_and_exchange_val_16_acq(mem, newval, oldval) \
	80	({ __typeof (*mem) ret; \
	81	__asm __volatile ("cmpl $0, %%gs:%P5\n\t" \
	82	"je 0f\n\t" \
	83	"lock\n" \
	84	"0:\tcmpxchgw %w2, %1" \
	85	: "=a" (ret), "=m" (*mem) \
	86	: "r" (newval), "m" (*mem), "0" (oldval), \
	87	"i" (offsetof (tcbhead_t, multiple_threads))); \
	88	ret; })
	89
	90	#define __arch_c_compare_and_exchange_val_32_acq(mem, newval, oldval) \
	91	({ __typeof (*mem) ret; \
	92	__asm __volatile ("cmpl $0, %%gs:%P5\n\t" \
	93	"je 0f\n\t" \
	94	"lock\n" \
	95	"0:\tcmpxchgl %2, %1" \
	96	: "=a" (ret), "=m" (*mem) \
	97	: "r" (newval), "m" (*mem), "0" (oldval), \
	98	"i" (offsetof (tcbhead_t, multiple_threads))); \
	99	ret; })
	100
b1473c32 UD	101	/* XXX We do not really need 64-bit compare-and-exchange. At least
	102	not in the moment. Using it would mean causing portability
	103	problems since not many other 32-bit architectures have support for
	104	such an operation. So don't define any code for now. If it is
a73ab6df UD	105	really going to be used the code below can be used on Intel Pentium
a73ab6df UD	106	and later, but NOT on i486. */
b1473c32	107	#if 1
85ae0589 RM	108	# define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \
	109	({ __typeof (mem) ret = (mem); \
	110	abort (); \
	111	ret = (newval); \
	112	ret = (oldval); \
	113	ret; })
	114	# define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
	115	({ __typeof (mem) ret = (mem); \
	116	abort (); \
	117	ret = (newval); \
	118	ret = (oldval); \
	119	ret; })
b1473c32 UD	120	#else
b1473c32 UD	121	# ifdef __PIC__
5a3ab2fc UD	122	# define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \
	123	({ __typeof (*mem) ret; \
	124	__asm __volatile ("xchgl %2, %%ebx\n\t" \
bd4f43b4	125	LOCK_PREFIX "cmpxchg8b %1\n\t" \
5a3ab2fc UD	126	"xchgl %2, %%ebx" \
5a3ab2fc UD	127	: "=A" (ret), "=m" (*mem) \
b1473c32 UD	128	: "DS" (((unsigned long long int) (newval)) \
	129	& 0xffffffff), \
	130	"c" (((unsigned long long int) (newval)) >> 32), \
4541b483	131	"m" (*mem), "a" (((unsigned long long int) (oldval)) \
b1473c32	132	& 0xffffffff), \
5a3ab2fc	133	"d" (((unsigned long long int) (oldval)) >> 32)); \
b1473c32	134	ret; })
11bf311e UD	135
	136	# define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
	137	({ __typeof (*mem) ret; \
	138	__asm __volatile ("xchgl %2, %%ebx\n\t" \
	139	"cmpl $0, %%gs:%P7\n\t" \
	140	"je 0f\n\t" \
	141	"lock\n" \
	142	"0:\tcmpxchg8b %1\n\t" \
	143	"xchgl %2, %%ebx" \
	144	: "=A" (ret), "=m" (*mem) \
	145	: "DS" (((unsigned long long int) (newval)) \
	146	& 0xffffffff), \
	147	"c" (((unsigned long long int) (newval)) >> 32), \
	148	"m" (*mem), "a" (((unsigned long long int) (oldval)) \
	149	& 0xffffffff), \
	150	"d" (((unsigned long long int) (oldval)) >> 32), \
	151	"i" (offsetof (tcbhead_t, multiple_threads))); \
	152	ret; })
b1473c32	153	# else
26f24355	154	# define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \
5a3ab2fc	155	({ __typeof (*mem) ret; \
bd4f43b4	156	__asm __volatile (LOCK_PREFIX "cmpxchg8b %1" \
5a3ab2fc	157	: "=A" (ret), "=m" (*mem) \
b1473c32 UD	158	: "b" (((unsigned long long int) (newval)) \
b1473c32 UD	159	& 0xffffffff), \
bd4f43b4	160	"c" (((unsigned long long int) (newval)) >> 32), \
4541b483	161	"m" (*mem), "a" (((unsigned long long int) (oldval)) \
b1473c32	162	& 0xffffffff), \
5a3ab2fc	163	"d" (((unsigned long long int) (oldval)) >> 32)); \
b1473c32	164	ret; })
11bf311e UD	165
	166	# define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
	167	({ __typeof (*mem) ret; \
	168	__asm __volatile ("cmpl $0, %%gs:%P7\n\t" \
	169	"je 0f\n\t" \
	170	"lock\n" \
	171	"0:\tcmpxchg8b %1" \
	172	: "=A" (ret), "=m" (*mem) \
	173	: "b" (((unsigned long long int) (newval)) \
	174	& 0xffffffff), \
	175	"c" (((unsigned long long int) (newval)) >> 32), \
	176	"m" (*mem), "a" (((unsigned long long int) (oldval)) \
	177	& 0xffffffff), \
	178	"d" (((unsigned long long int) (oldval)) >> 32), \
	179	"i" (offsetof (tcbhead_t, multiple_threads))); \
	180	ret; })
b1473c32 UD	181	# endif
	182	#endif
	183
	184
f79466a8	185	/* Note that we need no lock prefix. */
949ec764	186	#define atomic_exchange_acq(mem, newvalue) \
f79466a8 UD	187	({ __typeof (*mem) result; \
	188	if (sizeof (*mem) == 1) \
	189	__asm __volatile ("xchgb %b0, %1" \
a810e68c	190	: "=q" (result), "=m" (*mem) \
4541b483	191	: "0" (newvalue), "m" (*mem)); \
f79466a8 UD	192	else if (sizeof (*mem) == 2) \
	193	__asm __volatile ("xchgw %w0, %1" \
	194	: "=r" (result), "=m" (*mem) \
4541b483	195	: "0" (newvalue), "m" (*mem)); \
f79466a8 UD	196	else if (sizeof (*mem) == 4) \
	197	__asm __volatile ("xchgl %0, %1" \
	198	: "=r" (result), "=m" (*mem) \
4541b483	199	: "0" (newvalue), "m" (*mem)); \
f79466a8 UD	200	else \
	201	{ \
	202	result = 0; \
	203	abort (); \
	204	} \
	205	result; })
	206
	207
11bf311e	208	#define __arch_exchange_and_add_body(lock, pfx, mem, value) \
75fccede UD	209	({ __typeof (*mem) __result; \
75fccede UD	210	__typeof (value) __addval = (value); \
b1473c32	211	if (sizeof (*mem) == 1) \
11bf311e	212	__asm __volatile (lock "xaddb %b0, %1" \
a810e68c	213	: "=q" (__result), "=m" (*mem) \
11bf311e UD	214	: "0" (__addval), "m" (*mem), \
11bf311e UD	215	"i" (offsetof (tcbhead_t, multiple_threads))); \
b1473c32	216	else if (sizeof (*mem) == 2) \
11bf311e	217	__asm __volatile (lock "xaddw %w0, %1" \
75fccede	218	: "=r" (__result), "=m" (*mem) \
11bf311e UD	219	: "0" (__addval), "m" (*mem), \
11bf311e UD	220	"i" (offsetof (tcbhead_t, multiple_threads))); \
b1473c32	221	else if (sizeof (*mem) == 4) \
11bf311e	222	__asm __volatile (lock "xaddl %0, %1" \
75fccede	223	: "=r" (__result), "=m" (*mem) \
11bf311e UD	224	: "0" (__addval), "m" (*mem), \
11bf311e UD	225	"i" (offsetof (tcbhead_t, multiple_threads))); \
b1473c32 UD	226	else \
b1473c32 UD	227	{ \
75fccede UD	228	__typeof (mem) __memp = (mem); \
	229	__typeof (*mem) __tmpval; \
	230	__result = *__memp; \
b1473c32	231	do \
75fccede	232	__tmpval = __result; \
11bf311e	233	while ((__result = pfx##_compare_and_exchange_val_64_acq \
75fccede	234	(__memp, __result + __addval, __result)) == __tmpval); \
b1473c32	235	} \
75fccede	236	__result; })
b1473c32	237
bc957d53	238	#define atomic_exchange_and_add(mem, value) \
038a1a9f	239	__sync_fetch_and_add (mem, value)
11bf311e UD	240
	241	#define __arch_exchange_and_add_cprefix \
	242	"cmpl $0, %%gs:%P4\n\tje 0f\n\tlock\n0:\t"
	243
	244	#define catomic_exchange_and_add(mem, value) \
	245	__arch_exchange_and_add_body (__arch_exchange_and_add_cprefix, __arch_c, \
	246	mem, value)
	247
	248
	249	#define __arch_add_body(lock, pfx, mem, value) \
	250	do { \
	251	if (__builtin_constant_p (value) && (value) == 1) \
	252	atomic_increment (mem); \
	253	else if (__builtin_constant_p (value) && (value) == -1) \
	254	atomic_decrement (mem); \
	255	else if (sizeof (*mem) == 1) \
	256	__asm __volatile (lock "addb %b1, %0" \
	257	: "=m" (*mem) \
a810e68c	258	: "iq" (value), "m" (*mem), \
11bf311e UD	259	"i" (offsetof (tcbhead_t, multiple_threads))); \
	260	else if (sizeof (*mem) == 2) \
	261	__asm __volatile (lock "addw %w1, %0" \
	262	: "=m" (*mem) \
	263	: "ir" (value), "m" (*mem), \
	264	"i" (offsetof (tcbhead_t, multiple_threads))); \
	265	else if (sizeof (*mem) == 4) \
	266	__asm __volatile (lock "addl %1, %0" \
	267	: "=m" (*mem) \
	268	: "ir" (value), "m" (*mem), \
	269	"i" (offsetof (tcbhead_t, multiple_threads))); \
	270	else \
	271	{ \
	272	__typeof (value) __addval = (value); \
	273	__typeof (mem) __memp = (mem); \
	274	__typeof (mem) __oldval = __memp; \
	275	__typeof (*mem) __tmpval; \
	276	do \
	277	__tmpval = __oldval; \
	278	while ((__oldval = pfx##_compare_and_exchange_val_64_acq \
	279	(__memp, __oldval + __addval, __oldval)) == __tmpval); \
	280	} \
	281	} while (0)
b1473c32	282
37fb7595	283	#define atomic_add(mem, value) \
11bf311e UD	284	__arch_add_body (LOCK_PREFIX, __arch, mem, value)
	285
	286	#define __arch_add_cprefix \
	287	"cmpl $0, %%gs:%P3\n\tje 0f\n\tlock\n0:\t"
	288
	289	#define catomic_add(mem, value) \
	290	__arch_add_body (__arch_add_cprefix, __arch_c, mem, value)
37fb7595	291
b1473c32 UD	292
	293	#define atomic_add_negative(mem, value) \
	294	({ unsigned char __result; \
	295	if (sizeof (*mem) == 1) \
bd4f43b4	296	__asm __volatile (LOCK_PREFIX "addb %b2, %0; sets %1" \
b1473c32	297	: "=m" (*mem), "=qm" (__result) \
4541b483	298	: "iq" (value), "m" (*mem)); \
b1473c32	299	else if (sizeof (*mem) == 2) \
bd4f43b4	300	__asm __volatile (LOCK_PREFIX "addw %w2, %0; sets %1" \
b1473c32	301	: "=m" (*mem), "=qm" (__result) \
4541b483	302	: "ir" (value), "m" (*mem)); \
b1473c32	303	else if (sizeof (*mem) == 4) \
bd4f43b4	304	__asm __volatile (LOCK_PREFIX "addl %2, %0; sets %1" \
b1473c32	305	: "=m" (*mem), "=qm" (__result) \
4541b483	306	: "ir" (value), "m" (*mem)); \
b1473c32 UD	307	else \
	308	abort (); \
	309	__result; })
	310
	311
	312	#define atomic_add_zero(mem, value) \
	313	({ unsigned char __result; \
	314	if (sizeof (*mem) == 1) \
bd4f43b4	315	__asm __volatile (LOCK_PREFIX "addb %b2, %0; setz %1" \
b1473c32	316	: "=m" (*mem), "=qm" (__result) \
a810e68c	317	: "iq" (value), "m" (*mem)); \
b1473c32	318	else if (sizeof (*mem) == 2) \
bd4f43b4	319	__asm __volatile (LOCK_PREFIX "addw %w2, %0; setz %1" \
b1473c32	320	: "=m" (*mem), "=qm" (__result) \
4541b483	321	: "ir" (value), "m" (*mem)); \
b1473c32	322	else if (sizeof (*mem) == 4) \
bd4f43b4	323	__asm __volatile (LOCK_PREFIX "addl %2, %0; setz %1" \
b1473c32	324	: "=m" (*mem), "=qm" (__result) \
4541b483	325	: "ir" (value), "m" (*mem)); \
b1473c32 UD	326	else \
	327	abort (); \
	328	__result; })
	329
	330
11bf311e UD	331	#define __arch_increment_body(lock, pfx, mem) \
	332	do { \
	333	if (sizeof (*mem) == 1) \
	334	__asm __volatile (lock "incb %b0" \
	335	: "=m" (*mem) \
	336	: "m" (*mem), \
	337	"i" (offsetof (tcbhead_t, multiple_threads))); \
	338	else if (sizeof (*mem) == 2) \
	339	__asm __volatile (lock "incw %w0" \
	340	: "=m" (*mem) \
	341	: "m" (*mem), \
	342	"i" (offsetof (tcbhead_t, multiple_threads))); \
	343	else if (sizeof (*mem) == 4) \
	344	__asm __volatile (lock "incl %0" \
	345	: "=m" (*mem) \
	346	: "m" (*mem), \
	347	"i" (offsetof (tcbhead_t, multiple_threads))); \
	348	else \
	349	{ \
	350	__typeof (mem) __memp = (mem); \
	351	__typeof (mem) __oldval = __memp; \
	352	__typeof (*mem) __tmpval; \
	353	do \
	354	__tmpval = __oldval; \
	355	while ((__oldval = pfx##_compare_and_exchange_val_64_acq \
	356	(__memp, __oldval + 1, __oldval)) == __tmpval); \
	357	} \
	358	} while (0)
	359
	360	#define atomic_increment(mem) __arch_increment_body (LOCK_PREFIX, __arch, mem)
	361
	362	#define __arch_increment_cprefix \
	363	"cmpl $0, %%gs:%P2\n\tje 0f\n\tlock\n0:\t"
	364
	365	#define catomic_increment(mem) \
	366	__arch_increment_body (__arch_increment_cprefix, __arch_c, mem)
37fb7595	367
b1473c32 UD	368
	369	#define atomic_increment_and_test(mem) \
	370	({ unsigned char __result; \
	371	if (sizeof (*mem) == 1) \
bd4f43b4	372	__asm __volatile (LOCK_PREFIX "incb %0; sete %b1" \
b1473c32	373	: "=m" (*mem), "=qm" (__result) \
4541b483	374	: "m" (*mem)); \
b1473c32	375	else if (sizeof (*mem) == 2) \
bd4f43b4	376	__asm __volatile (LOCK_PREFIX "incw %0; sete %w1" \
b1473c32	377	: "=m" (*mem), "=qm" (__result) \
4541b483	378	: "m" (*mem)); \
b1473c32	379	else if (sizeof (*mem) == 4) \
bd4f43b4	380	__asm __volatile (LOCK_PREFIX "incl %0; sete %1" \
b1473c32	381	: "=m" (*mem), "=qm" (__result) \
4541b483	382	: "m" (*mem)); \
b1473c32 UD	383	else \
	384	abort (); \
	385	__result; })
	386
	387
11bf311e UD	388	#define __arch_decrement_body(lock, pfx, mem) \
	389	do { \
	390	if (sizeof (*mem) == 1) \
	391	__asm __volatile (lock "decb %b0" \
	392	: "=m" (*mem) \
	393	: "m" (*mem), \
	394	"i" (offsetof (tcbhead_t, multiple_threads))); \
	395	else if (sizeof (*mem) == 2) \
	396	__asm __volatile (lock "decw %w0" \
	397	: "=m" (*mem) \
	398	: "m" (*mem), \
	399	"i" (offsetof (tcbhead_t, multiple_threads))); \
	400	else if (sizeof (*mem) == 4) \
	401	__asm __volatile (lock "decl %0" \
	402	: "=m" (*mem) \
	403	: "m" (*mem), \
	404	"i" (offsetof (tcbhead_t, multiple_threads))); \
	405	else \
	406	{ \
	407	__typeof (mem) __memp = (mem); \
	408	__typeof (mem) __oldval = __memp; \
	409	__typeof (*mem) __tmpval; \
	410	do \
	411	__tmpval = __oldval; \
	412	while ((__oldval = pfx##_compare_and_exchange_val_64_acq \
	413	(__memp, __oldval - 1, __oldval)) == __tmpval); \
	414	} \
	415	} while (0)
	416
	417	#define atomic_decrement(mem) __arch_decrement_body (LOCK_PREFIX, __arch, mem)
	418
	419	#define __arch_decrement_cprefix \
	420	"cmpl $0, %%gs:%P2\n\tje 0f\n\tlock\n0:\t"
	421
	422	#define catomic_decrement(mem) \
	423	__arch_decrement_body (__arch_decrement_cprefix, __arch_c, mem)
37fb7595	424
b1473c32 UD	425
	426	#define atomic_decrement_and_test(mem) \
	427	({ unsigned char __result; \
	428	if (sizeof (*mem) == 1) \
bd4f43b4	429	__asm __volatile (LOCK_PREFIX "decb %b0; sete %1" \
b1473c32	430	: "=m" (*mem), "=qm" (__result) \
4541b483	431	: "m" (*mem)); \
b1473c32	432	else if (sizeof (*mem) == 2) \
bd4f43b4	433	__asm __volatile (LOCK_PREFIX "decw %w0; sete %1" \
b1473c32	434	: "=m" (*mem), "=qm" (__result) \
4541b483	435	: "m" (*mem)); \
b1473c32	436	else if (sizeof (*mem) == 4) \
bd4f43b4	437	__asm __volatile (LOCK_PREFIX "decl %0; sete %1" \
b1473c32	438	: "=m" (*mem), "=qm" (__result) \
4541b483	439	: "m" (*mem)); \
b1473c32 UD	440	else \
	441	abort (); \
	442	__result; })
	443
	444
	445	#define atomic_bit_set(mem, bit) \
11bf311e UD	446	do { \
	447	if (sizeof (*mem) == 1) \
	448	__asm __volatile (LOCK_PREFIX "orb %b2, %0" \
	449	: "=m" (*mem) \
a810e68c	450	: "m" (*mem), "iq" (1 << (bit))); \
11bf311e UD	451	else if (sizeof (*mem) == 2) \
	452	__asm __volatile (LOCK_PREFIX "orw %w2, %0" \
	453	: "=m" (*mem) \
	454	: "m" (*mem), "ir" (1 << (bit))); \
	455	else if (sizeof (*mem) == 4) \
	456	__asm __volatile (LOCK_PREFIX "orl %2, %0" \
	457	: "=m" (*mem) \
	458	: "m" (*mem), "ir" (1 << (bit))); \
	459	else \
	460	abort (); \
	461	} while (0)
b1473c32 UD	462
	463
	464	#define atomic_bit_test_set(mem, bit) \
	465	({ unsigned char __result; \
	466	if (sizeof (*mem) == 1) \
bd4f43b4	467	__asm __volatile (LOCK_PREFIX "btsb %3, %1; setc %0" \
b1473c32	468	: "=q" (__result), "=m" (*mem) \
002ff853	469	: "m" (*mem), "ir" (bit)); \
b1473c32	470	else if (sizeof (*mem) == 2) \
bd4f43b4	471	__asm __volatile (LOCK_PREFIX "btsw %3, %1; setc %0" \
b1473c32	472	: "=q" (__result), "=m" (*mem) \
002ff853	473	: "m" (*mem), "ir" (bit)); \
b1473c32	474	else if (sizeof (*mem) == 4) \
bd4f43b4	475	__asm __volatile (LOCK_PREFIX "btsl %3, %1; setc %0" \
b1473c32	476	: "=q" (__result), "=m" (*mem) \
002ff853	477	: "m" (*mem), "ir" (bit)); \
b1473c32 UD	478	else \
	479	abort (); \
	480	__result; })
f377d022 UD	481
f377d022 UD	482
4eb984d3	483	#define atomic_spin_nop() asm ("rep; nop")
11bf311e UD	484
11bf311e UD	485
6c03cd11	486	#define __arch_and_body(lock, mem, mask) \
11bf311e UD	487	do { \
11bf311e UD	488	if (sizeof (*mem) == 1) \
6c03cd11	489	__asm __volatile (lock "andb %b1, %0" \
11bf311e	490	: "=m" (*mem) \
6c03cd11 UD	491	: "iq" (mask), "m" (*mem), \
6c03cd11 UD	492	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e	493	else if (sizeof (*mem) == 2) \
6c03cd11	494	__asm __volatile (lock "andw %w1, %0" \
11bf311e	495	: "=m" (*mem) \
6c03cd11 UD	496	: "ir" (mask), "m" (*mem), \
6c03cd11 UD	497	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e	498	else if (sizeof (*mem) == 4) \
6c03cd11	499	__asm __volatile (lock "andl %1, %0" \
11bf311e	500	: "=m" (*mem) \
6c03cd11 UD	501	: "ir" (mask), "m" (*mem), \
6c03cd11 UD	502	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e UD	503	else \
	504	abort (); \
	505	} while (0)
	506
6c03cd11 UD	507	#define __arch_cprefix \
	508	"cmpl $0, %%gs:%P3\n\tje 0f\n\tlock\n0:\t"
	509
	510	#define atomic_and(mem, mask) __arch_and_body (LOCK_PREFIX, mem, mask)
	511
	512	#define catomic_and(mem, mask) __arch_and_body (__arch_cprefix, mem, mask)
	513
11bf311e UD	514
	515	#define __arch_or_body(lock, mem, mask) \
	516	do { \
	517	if (sizeof (*mem) == 1) \
a810e68c	518	__asm __volatile (lock "orb %b1, %0" \
11bf311e	519	: "=m" (*mem) \
a810e68c	520	: "iq" (mask), "m" (*mem), \
11bf311e UD	521	"i" (offsetof (tcbhead_t, multiple_threads))); \
11bf311e UD	522	else if (sizeof (*mem) == 2) \
77151937	523	__asm __volatile (lock "orw %w1, %0" \
11bf311e UD	524	: "=m" (*mem) \
	525	: "ir" (mask), "m" (*mem), \
	526	"i" (offsetof (tcbhead_t, multiple_threads))); \
	527	else if (sizeof (*mem) == 4) \
	528	__asm __volatile (lock "orl %1, %0" \
	529	: "=m" (*mem) \
	530	: "ir" (mask), "m" (*mem), \
	531	"i" (offsetof (tcbhead_t, multiple_threads))); \
	532	else \
	533	abort (); \
	534	} while (0)
	535
	536	#define atomic_or(mem, mask) __arch_or_body (LOCK_PREFIX, mem, mask)
	537
6c03cd11	538	#define catomic_or(mem, mask) __arch_or_body (__arch_cprefix, mem, mask)
c47ca964 TR	539
	540	/* We don't use mfence because it is supposedly slower due to having to
	541	provide stronger guarantees (e.g., regarding self-modifying code). */
	542	#define atomic_full_barrier() \
	543	__asm __volatile (LOCK_PREFIX "orl $0, (%%esp)" ::: "memory")
	544	#define atomic_read_barrier() __asm ("" ::: "memory")
	545	#define atomic_write_barrier() __asm ("" ::: "memory")