nptl/sysdeps/x86_64/bits/atomic.h

   1 /* Copyright (C) 2002 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <stdint.h>
  21
  22
  23 typedef int8_t atomic8_t;
  24 typedef uint8_t uatomic8_t;
  25 typedef int_fast8_t atomic_fast8_t;
  26 typedef uint_fast8_t uatomic_fast8_t;
  27
  28 typedef int16_t atomic16_t;
  29 typedef uint16_t uatomic16_t;
  30 typedef int_fast16_t atomic_fast16_t;
  31 typedef uint_fast16_t uatomic_fast16_t;
  32
  33 typedef int32_t atomic32_t;
  34 typedef uint32_t uatomic32_t;
  35 typedef int_fast32_t atomic_fast32_t;
  36 typedef uint_fast32_t uatomic_fast32_t;
  37
  38 typedef int64_t atomic64_t;
  39 typedef uint64_t uatomic64_t;
  40 typedef int_fast64_t atomic_fast64_t;
  41 typedef uint_fast64_t uatomic_fast64_t;
  42
  43 typedef intptr_t atomicptr_t;
  44 typedef uintptr_t uatomicptr_t;
  45 typedef intmax_t atomic_max_t;
  46 typedef uintmax_t uatomic_max_t;
  47
  48
  49 #ifndef LOCK
  50 # ifdef UP
  51 #  define LOCK  /* nothing */
  52 # else
  53 #  define LOCK "lock;"
  54 # endif
  55 #endif
  56
  57
  58 #define __arch_compare_and_exchange_8_acq(mem, newval, oldval) \
  59   ({ unsigned char ret;                                                       \
  60      __asm __volatile (LOCK "cmpxchgb %b2, %1; setne %0"                      \
  61                        : "=a" (ret), "=m" (*mem)                              \
  62                        : "q" (newval), "1" (*mem), "0" (oldval));             \
  63      ret; })
  64
  65 #define __arch_compare_and_exchange_16_acq(mem, newval, oldval) \
  66   ({ unsigned char ret;                                                       \
  67      __asm __volatile (LOCK "cmpxchgw %w2, %1; setne %0"                      \
  68                        : "=a" (ret), "=m" (*mem)                              \
  69                        : "r" (newval), "1" (*mem), "0" (oldval));             \
  70      ret; })
  71
  72 #define __arch_compare_and_exchange_32_acq(mem, newval, oldval) \
  73   ({ unsigned char ret;                                                       \
  74      __asm __volatile (LOCK "cmpxchgl %2, %1; setne %0"                       \
  75                        : "=a" (ret), "=m" (*mem)                              \
  76                        : "r" (newval), "1" (*mem), "0" (oldval));             \
  77      ret; })
  78
  79 #define __arch_compare_and_exchange_64_acq(mem, newval, oldval) \
  80   ({ unsigned char ret;                                                       \
  81      __asm __volatile (LOCK "cmpxchgq %q2, %1; setne %0"                      \
  82                        : "=a" (ret), "=m" (*mem)                              \
  83                        : "r" (newval), "1" (*mem), "0" (oldval));             \
  84      ret; })
  85
  86
  87 #define atomic_exchange_and_add(mem, value) \
  88   ({ __typeof (*mem) result;                                                  \
  89      if (sizeof (*mem) == 1)                                                  \
  90        __asm __volatile (LOCK "xaddb %b0, %1"                                 \
  91                          : "=r" (result), "=m" (*mem)                         \
  92                          : "0" (value), "1" (*mem));                          \
  93      else if (sizeof (*mem) == 2)                                             \
  94        __asm __volatile (LOCK "xaddw %w0, %1"                                 \
  95                          : "=r" (result), "=m" (*mem)                         \
  96                          : "0" (value), "1" (*mem));                          \
  97      else if (sizeof (*mem) == 4)                                             \
  98        __asm __volatile (LOCK "xaddl %0, %1"                                  \
  99                          : "=r" (result), "=m" (*mem)                         \
 100                          : "0" (value), "1" (*mem));                          \
 101      else                                                                     \
 102        __asm __volatile (LOCK "xaddq %q0, %1"                                 \
 103                          : "=r" (result), "=m" (*mem)                         \
 104                          : "0" (value), "1" (*mem));                          \
 105      result; })
 106
 107
 108 #define atomic_add(mem, value) \
 109   (void) ({ if (__builtin_constant_p (value) && (value) == 1)                 \
 110               atomic_increment (mem);                                         \
 111             else if (__builtin_constant_p (value) && (value) == 1)            \
 112               atomic_decrement (mem);                                         \
 113             else if (sizeof (*mem) == 1)                                      \
 114               __asm __volatile (LOCK "addb %b1, %0"                           \
 115                                 : "=m" (*mem)                                 \
 116                                 : "ir" (value), "0" (*mem));                  \
 117             else if (sizeof (*mem) == 2)                                      \
 118               __asm __volatile (LOCK "addw %w1, %0"                           \
 119                                 : "=m" (*mem)                                 \
 120                                 : "ir" (value), "0" (*mem));                  \
 121             else if (sizeof (*mem) == 4)                                      \
 122               __asm __volatile (LOCK "addl %1, %0"                            \
 123                                 : "=m" (*mem)                                 \
 124                                 : "ir" (value), "0" (*mem));                  \
 125             else                                                              \
 126               __asm __volatile (LOCK "addq %q1, %0"                           \
 127                                 : "=m" (*mem)                                 \
 128                                 : "ir" (value), "0" (*mem));                  \
 129             })
 130
 131
 132 #define atomic_add_negative(mem, value) \
 133   ({ unsigned char __result;                                                  \
 134      if (sizeof (*mem) == 1)                                                  \
 135        __asm __volatile (LOCK "addb %b2, %0; sets %1"                         \
 136                          : "=m" (*mem), "=qm" (__result)                      \
 137                          : "ir" (value), "0" (*mem));                         \
 138      else if (sizeof (*mem) == 2)                                             \
 139        __asm __volatile (LOCK "addw %w2, %0; sets %1"                         \
 140                          : "=m" (*mem), "=qm" (__result)                      \
 141                          : "ir" (value), "0" (*mem));                         \
 142      else if (sizeof (*mem) == 4)                                             \
 143        __asm __volatile (LOCK "addl %2, %0; sets %1"                          \
 144                          : "=m" (*mem), "=qm" (__result)                      \
 145                          : "ir" (value), "0" (*mem));                         \
 146      else                                                                     \
 147        __asm __volatile (LOCK "addq %q2, %0; sets %1"                         \
 148                          : "=m" (*mem), "=qm" (__result)                      \
 149                          : "ir" (value), "0" (*mem));                         \
 150      __result; })
 151
 152
 153 #define atomic_add_zero(mem, value) \
 154   ({ unsigned char __result;                                                  \
 155      if (sizeof (*mem) == 1)                                                  \
 156        __asm __volatile (LOCK "addb %b2, %0; setz %1"                         \
 157                          : "=m" (*mem), "=qm" (__result)                      \
 158                          : "ir" (value), "0" (*mem));                         \
 159      else if (sizeof (*mem) == 2)                                             \
 160        __asm __volatile (LOCK "addw %w2, %0; setz %1"                         \
 161                          : "=m" (*mem), "=qm" (__result)                      \
 162                          : "ir" (value), "0" (*mem));                         \
 163      else if (sizeof (*mem) == 4)                                             \
 164        __asm __volatile (LOCK "addl %2, %0; setz %1"                          \
 165                          : "=m" (*mem), "=qm" (__result)                      \
 166                          : "ir" (value), "0" (*mem));                         \
 167      else                                                                     \
 168        __asm __volatile (LOCK "addq %q2, %0; setz %1"                         \
 169                          : "=m" (*mem), "=qm" (__result)                      \
 170                          : "ir" (value), "0" (*mem));                         \
 171      __result; })
 172
 173
 174 #define atomic_increment(mem) \
 175   (void) ({ if (sizeof (*mem) == 1)                                           \
 176               __asm __volatile (LOCK "incb %b0"                               \
 177                                 : "=m" (*mem)                                 \
 178                                 : "0" (*mem));                                \
 179             else if (sizeof (*mem) == 2)                                      \
 180               __asm __volatile (LOCK "incw %w0"                               \
 181                                 : "=m" (*mem)                                 \
 182                                 : "0" (*mem));                                \
 183             else if (sizeof (*mem) == 4)                                      \
 184               __asm __volatile (LOCK "incl %0"                                \
 185                                 : "=m" (*mem)                                 \
 186                                 : "0" (*mem));                                \
 187             else                                                              \
 188               __asm __volatile (LOCK "incq %q0"                               \
 189                                 : "=m" (*mem)                                 \
 190                                 : "0" (*mem));                                \
 191             })
 192
 193
 194 #define atomic_increment_and_test(mem) \
 195   ({ unsigned char __result;                                                  \
 196      if (sizeof (*mem) == 1)                                                  \
 197        __asm __volatile (LOCK "incb %b0; sete %1"                             \
 198                          : "=m" (*mem), "=qm" (__result)                      \
 199                          : "0" (*mem));                                       \
 200      else if (sizeof (*mem) == 2)                                             \
 201        __asm __volatile (LOCK "incw %w0; sete %1"                             \
 202                          : "=m" (*mem), "=qm" (__result)                      \
 203                          : "0" (*mem));                                       \
 204      else if (sizeof (*mem) == 4)                                             \
 205        __asm __volatile (LOCK "incl %0; sete %1"                              \
 206                          : "=m" (*mem), "=qm" (__result)                      \
 207                          : "0" (*mem));                                       \
 208      else                                                                     \
 209        __asm __volatile (LOCK "incq %q0; sete %1"                             \
 210                          : "=m" (*mem), "=qm" (__result)                      \
 211                          : "0" (*mem));                                       \
 212      __result; })
 213
 214
 215 #define atomic_decrement(mem) \
 216   (void) ({ if (sizeof (*mem) == 1)                                           \
 217               __asm __volatile (LOCK "decb %b0"                               \
 218                                 : "=m" (*mem)                                 \
 219                                 : "0" (*mem));                                \
 220             else if (sizeof (*mem) == 2)                                      \
 221               __asm __volatile (LOCK "decw %w0"                               \
 222                                 : "=m" (*mem)                                 \
 223                                 : "0" (*mem));                                \
 224             else if (sizeof (*mem) == 4)                                      \
 225               __asm __volatile (LOCK "decl %0"                                \
 226                                 : "=m" (*mem)                                 \
 227                                 : "0" (*mem));                                \
 228             else                                                              \
 229               __asm __volatile (LOCK "decq %q0"                               \
 230                                 : "=m" (*mem)                                 \
 231                                 : "0" (*mem));                                \
 232             })
 233
 234
 235 #define atomic_decrement_and_test(mem) \
 236   ({ unsigned char __result;                                                  \
 237      if (sizeof (*mem) == 1)                                                  \
 238        __asm __volatile (LOCK "decb %b0; sete %1"                             \
 239                          : "=m" (*mem), "=qm" (__result)                      \
 240                          : "0" (*mem));                                       \
 241      else if (sizeof (*mem) == 2)                                             \
 242        __asm __volatile (LOCK "decw %w0; sete %1"                             \
 243                          : "=m" (*mem), "=qm" (__result)                      \
 244                          : "0" (*mem));                                       \
 245      else if (sizeof (*mem) == 4)                                             \
 246        __asm __volatile (LOCK "decl %0; sete %1"                              \
 247                          : "=m" (*mem), "=qm" (__result)                      \
 248                          : "0" (*mem));                                       \
 249      else                                                                     \
 250        __asm __volatile (LOCK "decq %q0; sete %1"                             \
 251                          : "=m" (*mem), "=qm" (__result)                      \
 252                          : "0" (*mem));                                       \
 253      __result; })
 254
 255
 256 #define atomic_bit_set(mem, bit) \
 257   (void) ({ if (sizeof (*mem) == 1)                                           \
 258               __asm __volatile (LOCK "orb %b2, %0"                            \
 259                                 : "=m" (*mem)                                 \
 260                                 : "0" (*mem), "i" (1 << (bit)));              \
 261             else if (sizeof (*mem) == 2)                                      \
 262               __asm __volatile (LOCK "orw %w2, %0"                            \
 263                                 : "=m" (*mem)                                 \
 264                                 : "0" (*mem), "i" (1 << (bit)));              \
 265             else if (sizeof (*mem) == 4)                                      \
 266               __asm __volatile (LOCK "orl %2, %0"                             \
 267                                 : "=m" (*mem)                                 \
 268                                 : "0" (*mem), "i" (1 << (bit)));              \
 269             else                                                              \
 270               __asm __volatile (LOCK "orq %q2, %0"                            \
 271                                 : "=m" (*mem)                                 \
 272                                 : "0" (*mem), "i" (1 << (bit)));              \
 273             })
 274
 275
 276 #define atomic_bit_test_set(mem, bit) \
 277   ({ unsigned char __result;                                                  \
 278      if (sizeof (*mem) == 1)                                                  \
 279        __asm __volatile (LOCK "btsb %3, %1; setc %0"                          \
 280                          : "=q" (__result), "=m" (*mem)                       \
 281                          : "1" (*mem), "i" (bit));                            \
 282      else if (sizeof (*mem) == 2)                                             \
 283        __asm __volatile (LOCK "btsw %3, %1; setc %0"                          \
 284                          : "=q" (__result), "=m" (*mem)                       \
 285                          : "1" (*mem), "i" (bit));                            \
 286      else if (sizeof (*mem) == 4)                                             \
 287        __asm __volatile (LOCK "btsl %3, %1; setc %0"                          \
 288                          : "=q" (__result), "=m" (*mem)                       \
 289                          : "1" (*mem), "i" (bit));                            \
 290      else                                                                     \
 291        __asm __volatile (LOCK "btsq %3, %1; setc %0"                          \
 292                          : "=q" (__result), "=m" (*mem)                       \
 293                          : "1" (*mem), "i" (bit));                            \
 294      __result; })