libgcc/config/pa/linux-atomic.c

   1 /* Linux-specific atomic operations for PA Linux.
   2    Copyright (C) 2008-2023 Free Software Foundation, Inc.
   3    Based on code contributed by CodeSourcery for ARM EABI Linux.
   4    Modifications for PA Linux by Helge Deller <deller@gmx.de>
   5
   6 This file is part of GCC.
   7
   8 GCC is free software; you can redistribute it and/or modify it under
   9 the terms of the GNU General Public License as published by the Free
  10 Software Foundation; either version 3, or (at your option) any later
  11 version.
  12
  13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  16 for more details.
  17
  18 Under Section 7 of GPL version 3, you are granted additional
  19 permissions described in the GCC Runtime Library Exception, version
  20 3.1, as published by the Free Software Foundation.
  21
  22 You should have received a copy of the GNU General Public License and
  23 a copy of the GCC Runtime Library Exception along with this program;
  24 see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  25 <http://www.gnu.org/licenses/>.  */
  26
  27 #define EFAULT  14
  28 #define EBUSY   16
  29 #define ENOSYS 251
  30
  31 #define _ASM_EFAULT "-14"
  32
  33 typedef unsigned char u8;
  34 typedef short unsigned int u16;
  35 typedef unsigned int u32;
  36 #ifdef __LP64__
  37 typedef long unsigned int u64;
  38 #else
  39 typedef long long unsigned int u64;
  40 #endif
  41
  42 /* PA-RISC 2.0 supports out-of-order execution for loads and stores.
  43    Thus, we need to synchonize memory accesses.  For more info, see:
  44    "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt.
  45
  46    We implement byte, short and int versions of each atomic operation
  47    using the kernel helper defined below.  There is no support for
  48    64-bit operations yet.  */
  49
  50 /* Determine kernel LWS function call (0=32-bit, 1=64-bit userspace).  */
  51 #define LWS_CAS (sizeof(long) == 4 ? 0 : 1)
  52
  53 /* Kernel helper for compare-and-exchange a 32-bit value.  */
  54 static inline long
  55 __kernel_cmpxchg (volatile void *mem, int oldval, int newval)
  56 {
  57   register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
  58   register int lws_old asm("r25") = oldval;
  59   register int lws_new asm("r24") = newval;
  60   register long lws_ret   asm("r28");
  61   register long lws_errno asm("r21");
  62   asm volatile (        "ble    0xb0(%%sr2, %%r0)       \n\t"
  63                         "ldi    %2, %%r20               \n\t"
  64                         "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t"
  65                         "iitlbp %%r0,(%%sr0, %%r0)      \n\t"
  66         : "=r" (lws_ret), "=r" (lws_errno)
  67         : "i" (LWS_CAS), "r" (lws_mem), "r" (lws_old), "r" (lws_new)
  68         : "r1", "r20", "r22", "r23", "r29", "r31", "memory"
  69   );
  70
  71   /* If the kernel LWS call succeeded (lws_errno == 0), lws_ret contains
  72      the old value from memory.  If this value is equal to OLDVAL, the
  73      new value was written to memory.  If not, return -EBUSY.  */
  74   if (!lws_errno && lws_ret != oldval)
  75     return -EBUSY;
  76
  77   return lws_errno;
  78 }
  79
  80 static inline long
  81 __kernel_cmpxchg2 (volatile void *mem, const void *oldval, const void *newval,
  82                    int val_size)
  83 {
  84   register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
  85   register unsigned long lws_old asm("r25") = (unsigned long) oldval;
  86   register unsigned long lws_new asm("r24") = (unsigned long) newval;
  87   register int lws_size asm("r23") = val_size;
  88   register long lws_ret   asm("r28");
  89   register long lws_errno asm("r21");
  90   asm volatile (        "ble    0xb0(%%sr2, %%r0)       \n\t"
  91                         "ldi    %6, %%r20               \n\t"
  92                         "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t"
  93                         "iitlbp %%r0,(%%sr0, %%r0)      \n\t"
  94         : "=r" (lws_ret), "=r" (lws_errno), "+r" (lws_mem),
  95           "+r" (lws_old), "+r" (lws_new), "+r" (lws_size)
  96         : "i" (2)
  97         : "r1", "r20", "r22", "r29", "r31", "fr4", "memory"
  98   );
  99
 100   /* If the kernel LWS call is successful, lws_ret contains 0.  */
 101   if (__builtin_expect (lws_ret == 0, 1))
 102     return 0;
 103
 104   /* If the kernel LWS call fails with no error, return -EBUSY */
 105   if (__builtin_expect (!lws_errno, 0))
 106     return -EBUSY;
 107
 108   return lws_errno;
 109 }
 110 #define HIDDEN __attribute__ ((visibility ("hidden")))
 111
 112 /* Big endian masks  */
 113 #define INVERT_MASK_1 24
 114 #define INVERT_MASK_2 16
 115
 116 #define MASK_1 0xffu
 117 #define MASK_2 0xffffu
 118
 119 /* Load value with an atomic processor load if possible.  */
 120 #define ATOMIC_LOAD(TYPE, WIDTH)                                        \
 121   static inline TYPE                                                    \
 122   atomic_load_##WIDTH (volatile void *ptr)                              \
 123   {                                                                     \
 124     return *(volatile TYPE *)ptr;                                       \
 125   }
 126
 127 #if defined(__LP64__) || defined(__SOFTFP__)
 128 ATOMIC_LOAD (u64, 8)
 129 #else
 130 static inline u64
 131 atomic_load_8 (volatile void *ptr)
 132 {
 133   u64 result;
 134   double tmp;
 135
 136   asm volatile ("{fldds|fldd} 0(%2),%1\n\t"
 137                 "{fstds|fstd} %1,-16(%%sp)\n\t"
 138                 "{ldws|ldw} -16(%%sp),%0\n\t"
 139                 "{ldws|ldw} -12(%%sp),%R0"
 140                 : "=r" (result), "=f" (tmp) : "r" (ptr): "memory");
 141   return result;
 142 }
 143 #endif
 144
 145 ATOMIC_LOAD (u32, 4)
 146 ATOMIC_LOAD (u16, 2)
 147 ATOMIC_LOAD (u8, 1)
 148
 149 #define FETCH_AND_OP_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX)          \
 150   TYPE HIDDEN                                                           \
 151   __sync_fetch_and_##OP##_##WIDTH (volatile void *ptr, TYPE val)        \
 152   {                                                                     \
 153     TYPE tmp, newval;                                                   \
 154     long failure;                                                       \
 155                                                                         \
 156     do {                                                                \
 157       tmp = atomic_load_##WIDTH ((volatile TYPE *)ptr);                 \
 158       newval = PFX_OP (tmp INF_OP val);                                 \
 159       failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX);          \
 160     } while (failure != 0);                                             \
 161                                                                         \
 162     return tmp;                                                         \
 163   }
 164
 165 FETCH_AND_OP_2 (add,   , +, u64, 8, 3)
 166 FETCH_AND_OP_2 (sub,   , -, u64, 8, 3)
 167 FETCH_AND_OP_2 (or,    , |, u64, 8, 3)
 168 FETCH_AND_OP_2 (and,   , &, u64, 8, 3)
 169 FETCH_AND_OP_2 (xor,   , ^, u64, 8, 3)
 170 FETCH_AND_OP_2 (nand, ~, &, u64, 8, 3)
 171
 172 FETCH_AND_OP_2 (add,   , +, u16, 2, 1)
 173 FETCH_AND_OP_2 (sub,   , -, u16, 2, 1)
 174 FETCH_AND_OP_2 (or,    , |, u16, 2, 1)
 175 FETCH_AND_OP_2 (and,   , &, u16, 2, 1)
 176 FETCH_AND_OP_2 (xor,   , ^, u16, 2, 1)
 177 FETCH_AND_OP_2 (nand, ~, &, u16, 2, 1)
 178
 179 FETCH_AND_OP_2 (add,   , +, u8, 1, 0)
 180 FETCH_AND_OP_2 (sub,   , -, u8, 1, 0)
 181 FETCH_AND_OP_2 (or,    , |, u8, 1, 0)
 182 FETCH_AND_OP_2 (and,   , &, u8, 1, 0)
 183 FETCH_AND_OP_2 (xor,   , ^, u8, 1, 0)
 184 FETCH_AND_OP_2 (nand, ~, &, u8, 1, 0)
 185
 186 #define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX)          \
 187   TYPE HIDDEN                                                           \
 188   __sync_##OP##_and_fetch_##WIDTH (volatile void *ptr, TYPE val)        \
 189   {                                                                     \
 190     TYPE tmp, newval;                                                   \
 191     long failure;                                                       \
 192                                                                         \
 193     do {                                                                \
 194       tmp = atomic_load_##WIDTH ((volatile TYPE *)ptr);                 \
 195       newval = PFX_OP (tmp INF_OP val);                                 \
 196       failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX);          \
 197     } while (failure != 0);                                             \
 198                                                                         \
 199     return PFX_OP (tmp INF_OP val);                                     \
 200   }
 201
 202 OP_AND_FETCH_2 (add,   , +, u64, 8, 3)
 203 OP_AND_FETCH_2 (sub,   , -, u64, 8, 3)
 204 OP_AND_FETCH_2 (or,    , |, u64, 8, 3)
 205 OP_AND_FETCH_2 (and,   , &, u64, 8, 3)
 206 OP_AND_FETCH_2 (xor,   , ^, u64, 8, 3)
 207 OP_AND_FETCH_2 (nand, ~, &, u64, 8, 3)
 208
 209 OP_AND_FETCH_2 (add,   , +, u16, 2, 1)
 210 OP_AND_FETCH_2 (sub,   , -, u16, 2, 1)
 211 OP_AND_FETCH_2 (or,    , |, u16, 2, 1)
 212 OP_AND_FETCH_2 (and,   , &, u16, 2, 1)
 213 OP_AND_FETCH_2 (xor,   , ^, u16, 2, 1)
 214 OP_AND_FETCH_2 (nand, ~, &, u16, 2, 1)
 215
 216 OP_AND_FETCH_2 (add,   , +, u8, 1, 0)
 217 OP_AND_FETCH_2 (sub,   , -, u8, 1, 0)
 218 OP_AND_FETCH_2 (or,    , |, u8, 1, 0)
 219 OP_AND_FETCH_2 (and,   , &, u8, 1, 0)
 220 OP_AND_FETCH_2 (xor,   , ^, u8, 1, 0)
 221 OP_AND_FETCH_2 (nand, ~, &, u8, 1, 0)
 222
 223 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP)                           \
 224   unsigned int HIDDEN                                                   \
 225   __sync_fetch_and_##OP##_4 (volatile void *ptr, unsigned int val)      \
 226   {                                                                     \
 227     unsigned int tmp;                                                   \
 228     long failure;                                                       \
 229                                                                         \
 230     do {                                                                \
 231       tmp = atomic_load_4 ((volatile unsigned int *)ptr);               \
 232       failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val));   \
 233     } while (failure != 0);                                             \
 234                                                                         \
 235     return tmp;                                                         \
 236   }
 237
 238 FETCH_AND_OP_WORD (add,   , +)
 239 FETCH_AND_OP_WORD (sub,   , -)
 240 FETCH_AND_OP_WORD (or,    , |)
 241 FETCH_AND_OP_WORD (and,   , &)
 242 FETCH_AND_OP_WORD (xor,   , ^)
 243 FETCH_AND_OP_WORD (nand, ~, &)
 244
 245 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP)                           \
 246   unsigned int HIDDEN                                                   \
 247   __sync_##OP##_and_fetch_4 (volatile void *ptr, unsigned int val)      \
 248   {                                                                     \
 249     unsigned int tmp;                                                   \
 250     long failure;                                                       \
 251                                                                         \
 252     do {                                                                \
 253       tmp = atomic_load_4 ((volatile unsigned int *)ptr);               \
 254       failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val));   \
 255     } while (failure != 0);                                             \
 256                                                                         \
 257     return PFX_OP (tmp INF_OP val);                                     \
 258   }
 259
 260 OP_AND_FETCH_WORD (add,   , +)
 261 OP_AND_FETCH_WORD (sub,   , -)
 262 OP_AND_FETCH_WORD (or,    , |)
 263 OP_AND_FETCH_WORD (and,   , &)
 264 OP_AND_FETCH_WORD (xor,   , ^)
 265 OP_AND_FETCH_WORD (nand, ~, &)
 266
 267 typedef unsigned char bool;
 268
 269 #define COMPARE_AND_SWAP_2(TYPE, WIDTH, INDEX)                          \
 270   TYPE HIDDEN                                                           \
 271   __sync_val_compare_and_swap_##WIDTH (volatile void *ptr, TYPE oldval, \
 272                                        TYPE newval)                     \
 273   {                                                                     \
 274     TYPE actual_oldval;                                                 \
 275     long fail;                                                          \
 276                                                                         \
 277     while (1)                                                           \
 278       {                                                                 \
 279         actual_oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr);     \
 280                                                                         \
 281         if (__builtin_expect (oldval != actual_oldval, 0))              \
 282           return actual_oldval;                                         \
 283                                                                         \
 284         fail = __kernel_cmpxchg2 (ptr, &actual_oldval, &newval, INDEX); \
 285                                                                         \
 286         if (__builtin_expect (!fail, 1))                                \
 287           return actual_oldval;                                         \
 288       }                                                                 \
 289   }                                                                     \
 290                                                                         \
 291   _Bool HIDDEN                                                          \
 292   __sync_bool_compare_and_swap_##WIDTH (volatile void *ptr,             \
 293                                         TYPE oldval, TYPE newval)       \
 294   {                                                                     \
 295     long failure = __kernel_cmpxchg2 (ptr, &oldval, &newval, INDEX);    \
 296     return (failure == 0);                                              \
 297   }
 298
 299 COMPARE_AND_SWAP_2 (u64, 8, 3)
 300 COMPARE_AND_SWAP_2 (u16, 2, 1)
 301 COMPARE_AND_SWAP_2 (u8, 1, 0)
 302
 303 unsigned int HIDDEN
 304 __sync_val_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
 305                                unsigned int newval)
 306 {
 307   long fail;
 308   unsigned int actual_oldval;
 309
 310   while (1)
 311     {
 312       actual_oldval = atomic_load_4 ((volatile unsigned int *)ptr);
 313
 314       if (__builtin_expect (oldval != actual_oldval, 0))
 315         return actual_oldval;
 316
 317       fail = __kernel_cmpxchg (ptr, actual_oldval, newval);
 318
 319       if (__builtin_expect (!fail, 1))
 320         return actual_oldval;
 321     }
 322 }
 323
 324 _Bool HIDDEN
 325 __sync_bool_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
 326                                 unsigned int newval)
 327 {
 328   long failure = __kernel_cmpxchg (ptr, oldval, newval);
 329   return (failure == 0);
 330 }
 331
 332 #define SYNC_LOCK_TEST_AND_SET_2(TYPE, WIDTH, INDEX)                    \
 333 TYPE HIDDEN                                                             \
 334   __sync_lock_test_and_set_##WIDTH (volatile void *ptr, TYPE val)       \
 335   {                                                                     \
 336     TYPE oldval;                                                        \
 337     long failure;                                                       \
 338                                                                         \
 339     do {                                                                \
 340       oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr);              \
 341       failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX);          \
 342     } while (failure != 0);                                             \
 343                                                                         \
 344     return oldval;                                                      \
 345   }
 346
 347 SYNC_LOCK_TEST_AND_SET_2 (u64, 8, 3)
 348 SYNC_LOCK_TEST_AND_SET_2 (u16, 2, 1)
 349 SYNC_LOCK_TEST_AND_SET_2 (u8, 1, 0)
 350
 351 u32 HIDDEN
 352 __sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val)
 353 {
 354   long failure;
 355   unsigned int oldval;
 356
 357   do {
 358     oldval = atomic_load_4 ((volatile unsigned int *)ptr);
 359     failure = __kernel_cmpxchg (ptr, oldval, val);
 360   } while (failure != 0);
 361
 362   return oldval;
 363 }
 364
 365 #define SYNC_LOCK_RELEASE_1(TYPE, WIDTH, INDEX)                 \
 366   void HIDDEN                                                   \
 367   __sync_lock_release_##WIDTH (volatile void *ptr)              \
 368   {                                                             \
 369     TYPE oldval, val = 0;                                       \
 370     long failure;                                               \
 371                                                                 \
 372     do {                                                        \
 373       oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr);      \
 374       failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX);  \
 375     } while (failure != 0);                                     \
 376   }
 377
 378 SYNC_LOCK_RELEASE_1 (u64, 8, 3)
 379 SYNC_LOCK_RELEASE_1 (u16, 2, 1)
 380 SYNC_LOCK_RELEASE_1 (u8, 1, 0)
 381
 382 void HIDDEN
 383 __sync_lock_release_4 (volatile void *ptr)
 384 {
 385   long failure;
 386   unsigned int oldval;
 387
 388   do {
 389     oldval = atomic_load_4 ((volatile unsigned int *)ptr);
 390     failure = __kernel_cmpxchg (ptr, oldval, 0);
 391   } while (failure != 0);
 392 }
 393
 394 #ifndef __LP64__
 395 #define SYNC_LOCK_LOAD_2(TYPE, WIDTH, INDEX)                            \
 396   TYPE __sync_lock_load_##WIDTH (volatile void *) HIDDEN;               \
 397   TYPE                                                                  \
 398   __sync_lock_load_##WIDTH (volatile void *ptr)                         \
 399   {                                                                     \
 400     TYPE oldval;                                                        \
 401     long failure;                                                       \
 402                                                                         \
 403     do {                                                                \
 404       oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr);              \
 405       failure = __kernel_cmpxchg2 (ptr, &oldval, &oldval, INDEX);       \
 406     } while (failure != 0);                                             \
 407                                                                         \
 408     return oldval;                                                      \
 409   }
 410
 411 SYNC_LOCK_LOAD_2 (u64, 8, 3)
 412 #endif