sysdeps/powerpc/powerpc64/atomic-machine.h

   1 /* Atomic operations.  PowerPC64 version.
   2    Copyright (C) 2003-2019 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Paul Mackerras <paulus@au.ibm.com>, 2003.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <https://www.gnu.org/licenses/>.  */
  19
  20 /*  POWER6 adds a "Mutex Hint" to the Load and Reserve instruction.
  21     This is a hint to the hardware to expect additional updates adjacent
  22     to the lock word or not.  If we are acquiring a Mutex, the hint
  23     should be true. Otherwise we releasing a Mutex or doing a simple
  24     atomic operation.  In that case we don't expect additional updates
  25     adjacent to the lock word after the Store Conditional and the hint
  26     should be false.  */
  27
  28 #if defined _ARCH_PWR6 || defined _ARCH_PWR6X
  29 # define MUTEX_HINT_ACQ ",1"
  30 # define MUTEX_HINT_REL ",0"
  31 #else
  32 # define MUTEX_HINT_ACQ
  33 # define MUTEX_HINT_REL
  34 #endif
  35
  36 #define __HAVE_64B_ATOMICS 1
  37 #define USE_ATOMIC_COMPILER_BUILTINS 0
  38 #define ATOMIC_EXCHANGE_USES_CAS 1
  39
  40 /* The 32-bit exchange_bool is different on powerpc64 because the subf
  41    does signed 64-bit arithmetic while the lwarx is 32-bit unsigned
  42    (a load word and zero (high 32) form) load.
  43    In powerpc64 register values are 64-bit by default,  including oldval.
  44    The value in old val unknown sign extension, lwarx loads the 32-bit
  45    value as unsigned.  So we explicitly clear the high 32 bits in oldval.  */
  46 #define __arch_compare_and_exchange_bool_32_acq(mem, newval, oldval) \
  47 ({                                                                            \
  48   unsigned int __tmp, __tmp2;                                                 \
  49   __asm __volatile ("   clrldi  %1,%1,32\n"                                   \
  50                     "1: lwarx   %0,0,%2" MUTEX_HINT_ACQ "\n"                  \
  51                     "   subf.   %0,%1,%0\n"                                   \
  52                     "   bne     2f\n"                                         \
  53                     "   stwcx.  %4,0,%2\n"                                    \
  54                     "   bne-    1b\n"                                         \
  55                     "2: " __ARCH_ACQ_INSTR                                    \
  56                     : "=&r" (__tmp), "=r" (__tmp2)                            \
  57                     : "b" (mem), "1" (oldval), "r" (newval)                   \
  58                     : "cr0", "memory");                                       \
  59   __tmp != 0;                                                                 \
  60 })
  61
  62 /*
  63  * Only powerpc64 processors support Load doubleword and reserve index (ldarx)
  64  * and Store doubleword conditional indexed (stdcx) instructions.  So here
  65  * we define the 64-bit forms.
  66  */
  67 #define __arch_compare_and_exchange_bool_64_acq(mem, newval, oldval) \
  68 ({                                                                            \
  69   unsigned long __tmp;                                                        \
  70   __asm __volatile (                                                          \
  71                     "1: ldarx   %0,0,%1" MUTEX_HINT_ACQ "\n"                  \
  72                     "   subf.   %0,%2,%0\n"                                   \
  73                     "   bne     2f\n"                                         \
  74                     "   stdcx.  %3,0,%1\n"                                    \
  75                     "   bne-    1b\n"                                         \
  76                     "2: " __ARCH_ACQ_INSTR                                    \
  77                     : "=&r" (__tmp)                                           \
  78                     : "b" (mem), "r" (oldval), "r" (newval)                   \
  79                     : "cr0", "memory");                                       \
  80   __tmp != 0;                                                                 \
  81 })
  82
  83 #define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \
  84   ({                                                                          \
  85       __typeof (*(mem)) __tmp;                                                \
  86       __typeof (mem)  __memp = (mem);                                         \
  87       __asm __volatile (                                                      \
  88                         "1:     ldarx   %0,0,%1" MUTEX_HINT_ACQ "\n"          \
  89                         "       cmpd    %0,%2\n"                              \
  90                         "       bne     2f\n"                                 \
  91                         "       stdcx.  %3,0,%1\n"                            \
  92                         "       bne-    1b\n"                                 \
  93                         "2:     " __ARCH_ACQ_INSTR                            \
  94                         : "=&r" (__tmp)                                       \
  95                         : "b" (__memp), "r" (oldval), "r" (newval)            \
  96                         : "cr0", "memory");                                   \
  97       __tmp;                                                                  \
  98   })
  99
 100 #define __arch_compare_and_exchange_val_64_rel(mem, newval, oldval) \
 101   ({                                                                          \
 102       __typeof (*(mem)) __tmp;                                                \
 103       __typeof (mem)  __memp = (mem);                                         \
 104       __asm __volatile (__ARCH_REL_INSTR "\n"                                 \
 105                         "1:     ldarx   %0,0,%1" MUTEX_HINT_REL "\n"          \
 106                         "       cmpd    %0,%2\n"                              \
 107                         "       bne     2f\n"                                 \
 108                         "       stdcx.  %3,0,%1\n"                            \
 109                         "       bne-    1b\n"                                 \
 110                         "2:     "                                             \
 111                         : "=&r" (__tmp)                                       \
 112                         : "b" (__memp), "r" (oldval), "r" (newval)            \
 113                         : "cr0", "memory");                                   \
 114       __tmp;                                                                  \
 115   })
 116
 117 #define __arch_atomic_exchange_64_acq(mem, value) \
 118     ({                                                                        \
 119       __typeof (*mem) __val;                                                  \
 120       __asm __volatile (__ARCH_REL_INSTR "\n"                                 \
 121                         "1:     ldarx   %0,0,%2" MUTEX_HINT_ACQ "\n"          \
 122                         "       stdcx.  %3,0,%2\n"                            \
 123                         "       bne-    1b\n"                                 \
 124                   " " __ARCH_ACQ_INSTR                                        \
 125                         : "=&r" (__val), "=m" (*mem)                          \
 126                         : "b" (mem), "r" (value), "m" (*mem)                  \
 127                         : "cr0", "memory");                                   \
 128       __val;                                                                  \
 129     })
 130
 131 #define __arch_atomic_exchange_64_rel(mem, value) \
 132     ({                                                                        \
 133       __typeof (*mem) __val;                                                  \
 134       __asm __volatile (__ARCH_REL_INSTR "\n"                                 \
 135                         "1:     ldarx   %0,0,%2" MUTEX_HINT_REL "\n"          \
 136                         "       stdcx.  %3,0,%2\n"                            \
 137                         "       bne-    1b"                                   \
 138                         : "=&r" (__val), "=m" (*mem)                          \
 139                         : "b" (mem), "r" (value), "m" (*mem)                  \
 140                         : "cr0", "memory");                                   \
 141       __val;                                                                  \
 142     })
 143
 144 #define __arch_atomic_exchange_and_add_64(mem, value) \
 145     ({                                                                        \
 146       __typeof (*mem) __val, __tmp;                                           \
 147       __asm __volatile ("1:     ldarx   %0,0,%3\n"                            \
 148                         "       add     %1,%0,%4\n"                           \
 149                         "       stdcx.  %1,0,%3\n"                            \
 150                         "       bne-    1b"                                   \
 151                         : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)           \
 152                         : "b" (mem), "r" (value), "m" (*mem)                  \
 153                         : "cr0", "memory");                                   \
 154       __val;                                                                  \
 155     })
 156
 157 #define __arch_atomic_exchange_and_add_64_acq(mem, value) \
 158     ({                                                                        \
 159       __typeof (*mem) __val, __tmp;                                           \
 160       __asm __volatile ("1:     ldarx   %0,0,%3" MUTEX_HINT_ACQ "\n"          \
 161                         "       add     %1,%0,%4\n"                           \
 162                         "       stdcx.  %1,0,%3\n"                            \
 163                         "       bne-    1b\n"                                 \
 164                         __ARCH_ACQ_INSTR                                      \
 165                         : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)           \
 166                         : "b" (mem), "r" (value), "m" (*mem)                  \
 167                         : "cr0", "memory");                                   \
 168       __val;                                                                  \
 169     })
 170
 171 #define __arch_atomic_exchange_and_add_64_rel(mem, value) \
 172     ({                                                                        \
 173       __typeof (*mem) __val, __tmp;                                           \
 174       __asm __volatile (__ARCH_REL_INSTR "\n"                                 \
 175                         "1:     ldarx   %0,0,%3" MUTEX_HINT_REL "\n"          \
 176                         "       add     %1,%0,%4\n"                           \
 177                         "       stdcx.  %1,0,%3\n"                            \
 178                         "       bne-    1b"                                   \
 179                         : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)           \
 180                         : "b" (mem), "r" (value), "m" (*mem)                  \
 181                         : "cr0", "memory");                                   \
 182       __val;                                                                  \
 183     })
 184
 185 #define __arch_atomic_increment_val_64(mem) \
 186     ({                                                                        \
 187       __typeof (*(mem)) __val;                                                \
 188       __asm __volatile ("1:     ldarx   %0,0,%2\n"                            \
 189                         "       addi    %0,%0,1\n"                            \
 190                         "       stdcx.  %0,0,%2\n"                            \
 191                         "       bne-    1b"                                   \
 192                         : "=&b" (__val), "=m" (*mem)                          \
 193                         : "b" (mem), "m" (*mem)                               \
 194                         : "cr0", "memory");                                   \
 195       __val;                                                                  \
 196     })
 197
 198 #define __arch_atomic_decrement_val_64(mem) \
 199     ({                                                                        \
 200       __typeof (*(mem)) __val;                                                \
 201       __asm __volatile ("1:     ldarx   %0,0,%2\n"                            \
 202                         "       subi    %0,%0,1\n"                            \
 203                         "       stdcx.  %0,0,%2\n"                            \
 204                         "       bne-    1b"                                   \
 205                         : "=&b" (__val), "=m" (*mem)                          \
 206                         : "b" (mem), "m" (*mem)                               \
 207                         : "cr0", "memory");                                   \
 208       __val;                                                                  \
 209     })
 210
 211 #define __arch_atomic_decrement_if_positive_64(mem) \
 212   ({ int __val, __tmp;                                                        \
 213      __asm __volatile ("1:      ldarx   %0,0,%3\n"                            \
 214                        "        cmpdi   0,%0,0\n"                             \
 215                        "        addi    %1,%0,-1\n"                           \
 216                        "        ble     2f\n"                                 \
 217                        "        stdcx.  %1,0,%3\n"                            \
 218                        "        bne-    1b\n"                                 \
 219                        "2:      " __ARCH_ACQ_INSTR                            \
 220                        : "=&b" (__val), "=&r" (__tmp), "=m" (*mem)            \
 221                        : "b" (mem), "m" (*mem)                                \
 222                        : "cr0", "memory");                                    \
 223      __val;                                                                   \
 224   })
 225
 226 /*
 227  * All powerpc64 processors support the new "light weight"  sync (lwsync).
 228  */
 229 #define atomic_read_barrier()   __asm ("lwsync" ::: "memory")
 230 /*
 231  * "light weight" sync can also be used for the release barrier.
 232  */
 233 #ifndef UP
 234 # define __ARCH_REL_INSTR       "lwsync"
 235 #endif
 236 #define atomic_write_barrier()  __asm ("lwsync" ::: "memory")
 237
 238 /*
 239  * Include the rest of the atomic ops macros which are common to both
 240  * powerpc32 and powerpc64.
 241  */
 242 #include_next <atomic-machine.h>