nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h

   1 /* Copyright (C) 2003, 2004, 2006-2008, 2009 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #include <time.h>
  24 #include <sys/param.h>
  25 #include <bits/pthreadtypes.h>
  26 #include <atomic.h>
  27 #include <kernel-features.h>
  28
  29 #define SYS_futex               238
  30 #define FUTEX_WAIT              0
  31 #define FUTEX_WAKE              1
  32 #define FUTEX_REQUEUE           3
  33 #define FUTEX_CMP_REQUEUE       4
  34 #define FUTEX_WAKE_OP           5
  35 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  36 #define FUTEX_LOCK_PI           6
  37 #define FUTEX_UNLOCK_PI         7
  38 #define FUTEX_TRYLOCK_PI        8
  39 #define FUTEX_WAIT_BITSET       9
  40 #define FUTEX_WAKE_BITSET       10
  41 #define FUTEX_PRIVATE_FLAG      128
  42 #define FUTEX_CLOCK_REALTIME    256
  43
  44 #define FUTEX_BITSET_MATCH_ANY  0xffffffff
  45
  46 /* Values for 'private' parameter of locking macros.  Yes, the
  47    definition seems to be backwards.  But it is not.  The bit will be
  48    reversed before passing to the system call.  */
  49 #define LLL_PRIVATE     0
  50 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  51
  52
  53 #if !defined NOT_IN_libc || defined IS_IN_rtld
  54 /* In libc.so or ld.so all futexes are private.  */
  55 # ifdef __ASSUME_PRIVATE_FUTEX
  56 #  define __lll_private_flag(fl, private) \
  57   ((fl) | FUTEX_PRIVATE_FLAG)
  58 # else
  59 #  define __lll_private_flag(fl, private) \
  60   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  61 # endif
  62 #else
  63 # ifdef __ASSUME_PRIVATE_FUTEX
  64 #  define __lll_private_flag(fl, private) \
  65   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  66 # else
  67 #  define __lll_private_flag(fl, private) \
  68   (__builtin_constant_p (private)                                             \
  69    ? ((private) == 0                                                          \
  70       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  71       : (fl))                                                                 \
  72    : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG)                                \
  73               & THREAD_GETMEM (THREAD_SELF, header.private_futex))))
  74 # endif
  75 #endif
  76
  77 #define lll_futex_wait(futex, val, private) \
  78   lll_futex_timed_wait (futex, val, NULL, private)
  79
  80 #define lll_futex_timed_wait(futex, val, timespec, private) \
  81   ({                                                                          \
  82     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
  83     register unsigned long int __r3 asm ("3")                                 \
  84       = __lll_private_flag (FUTEX_WAIT, private);                             \
  85     register unsigned long int __r4 asm ("4") = (unsigned long int) (val);    \
  86     register unsigned long int __r5 asm ("5") = (unsigned long int)(timespec);\
  87     register unsigned long int __result asm ("2");                            \
  88                                                                               \
  89     __asm __volatile ("svc %b1"                                               \
  90                       : "=d" (__result)                                       \
  91                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
  92                         "d" (__r4), "d" (__r5)                                \
  93                       : "cc", "memory" );                                     \
  94     __result;                                                                 \
  95   })
  96
  97
  98 #define lll_futex_wake(futex, nr, private) \
  99   ({                                                                          \
 100     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
 101     register unsigned long int __r3 asm ("3")                                 \
 102       = __lll_private_flag (FUTEX_WAKE, private);                             \
 103     register unsigned long int __r4 asm ("4") = (unsigned long int) (nr);     \
 104     register unsigned long int __result asm ("2");                            \
 105                                                                               \
 106     __asm __volatile ("svc %b1"                                               \
 107                       : "=d" (__result)                                       \
 108                       : "i" (SYS_futex), "0" (__r2), "d" (__r3), "d" (__r4)   \
 109                       : "cc", "memory" );                                     \
 110     __result;                                                                 \
 111   })
 112
 113
 114 #define lll_robust_dead(futexv, private) \
 115   do                                                                          \
 116     {                                                                         \
 117       int *__futexp = &(futexv);                                              \
 118                                                                               \
 119       atomic_or (__futexp, FUTEX_OWNER_DIED);                                 \
 120       lll_futex_wake (__futexp, 1, private);                                  \
 121     }                                                                         \
 122   while (0)
 123
 124
 125 /* Returns non-zero if error happened, zero if success.  */
 126 #define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \
 127   ({                                                                          \
 128     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
 129     register unsigned long int __r3 asm ("3")                                 \
 130       = __lll_private_flag (FUTEX_CMP_REQUEUE, private);                      \
 131     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);         \
 132     register unsigned long int __r5 asm ("5") = (long int) (nr_move);         \
 133     register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex);  \
 134     register unsigned long int __r7 asm ("7") = (int) (val);                  \
 135     register unsigned long __result asm ("2");                                \
 136                                                                               \
 137     __asm __volatile ("svc %b1"                                               \
 138                       : "=d" (__result)                                       \
 139                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
 140                         "d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7)        \
 141                       : "cc", "memory" );                                     \
 142     __result > -4096UL;                                                       \
 143   })
 144
 145
 146 /* Returns non-zero if error happened, zero if success.  */
 147 #define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \
 148   ({                                                                          \
 149     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
 150     register unsigned long int __r3 asm ("3")                                 \
 151       = __lll_private_flag (FUTEX_WAKE_OP, private);                          \
 152     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);         \
 153     register unsigned long int __r5 asm ("5") = (long int) (nr_wake2);        \
 154     register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \
 155     register unsigned long int __r7 asm ("7")                                 \
 156       = (int) FUTEX_OP_CLEAR_WAKE_IF_GT_ONE;                                  \
 157     register unsigned long __result asm ("2");                                \
 158                                                                               \
 159     __asm __volatile ("svc %b1"                                               \
 160                       : "=d" (__result)                                       \
 161                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
 162                         "d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7)        \
 163                       : "cc", "memory" );                                     \
 164     __result > -4096UL;                                                       \
 165   })
 166
 167
 168 #define lll_compare_and_swap(futex, oldval, newval, operation) \
 169   do {                                                                        \
 170     __typeof (futex) __futex = (futex);                                       \
 171     __asm __volatile ("   l   %1,%0\n"                                        \
 172                       "0: " operation "\n"                                    \
 173                       "   cs  %1,%2,%0\n"                                     \
 174                       "   jl  0b\n"                                           \
 175                       "1:"                                                    \
 176                       : "=Q" (*__futex), "=&d" (oldval), "=&d" (newval)       \
 177                       : "m" (*__futex) : "cc", "memory" );                    \
 178   } while (0)
 179
 180
 181 static inline int
 182 __attribute__ ((always_inline))
 183 __lll_trylock (int *futex)
 184 {
 185     unsigned int old;
 186
 187     __asm __volatile ("cs %0,%3,%1"
 188                        : "=d" (old), "=Q" (*futex)
 189                        : "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
 190     return old != 0;
 191 }
 192 #define lll_trylock(futex) __lll_trylock (&(futex))
 193
 194
 195 static inline int
 196 __attribute__ ((always_inline))
 197 __lll_cond_trylock (int *futex)
 198 {
 199     unsigned int old;
 200
 201     __asm __volatile ("cs %0,%3,%1"
 202                        : "=d" (old), "=Q" (*futex)
 203                        : "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
 204     return old != 0;
 205 }
 206 #define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))
 207
 208
 209 static inline int
 210 __attribute__ ((always_inline))
 211 __lll_robust_trylock (int *futex, int id)
 212 {
 213     unsigned int old;
 214
 215     __asm __volatile ("cs %0,%3,%1"
 216                        : "=d" (old), "=Q" (*futex)
 217                        : "0" (0), "d" (id), "m" (*futex) : "cc", "memory" );
 218     return old != 0;
 219 }
 220 #define lll_robust_trylock(futex, id) \
 221   __lll_robust_trylock (&(futex), id)
 222
 223
 224 extern void __lll_lock_wait_private (int *futex) attribute_hidden;
 225 extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
 226 extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;
 227
 228 static inline void
 229 __attribute__ ((always_inline))
 230 __lll_lock (int *futex, int private)
 231 {
 232   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
 233     {
 234       if (__builtin_constant_p (private) && private == LLL_PRIVATE)
 235         __lll_lock_wait_private (futex);
 236       else
 237         __lll_lock_wait (futex, private);
 238     }
 239 }
 240 #define lll_lock(futex, private) __lll_lock (&(futex), private)
 241
 242 static inline int
 243 __attribute__ ((always_inline))
 244 __lll_robust_lock (int *futex, int id, int private)
 245 {
 246   int result = 0;
 247   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
 248                         0))
 249     result = __lll_robust_lock_wait (futex, private);
 250   return result;
 251 }
 252 #define lll_robust_lock(futex, id, private) \
 253   __lll_robust_lock (&(futex), id, private)
 254
 255 static inline void
 256 __attribute__ ((always_inline))
 257 __lll_cond_lock (int *futex, int private)
 258 {
 259   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 2, 0), 0))
 260     __lll_lock_wait (futex, private);
 261 }
 262 #define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
 263
 264 #define lll_robust_cond_lock(futex, id, private) \
 265   __lll_robust_lock (&(futex), (id) | FUTEX_WAITERS, private)
 266
 267 extern int __lll_timedlock_wait
 268   (int *futex, const struct timespec *, int private) attribute_hidden;
 269 extern int __lll_robust_timedlock_wait
 270   (int *futex, const struct timespec *, int private) attribute_hidden;
 271
 272 static inline int
 273 __attribute__ ((always_inline))
 274 __lll_timedlock (int *futex, const struct timespec *abstime, int private)
 275 {
 276   int result = 0;
 277   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
 278     result = __lll_timedlock_wait (futex, abstime, private);
 279   return result;
 280 }
 281 #define lll_timedlock(futex, abstime, private) \
 282   __lll_timedlock (&(futex), abstime, private)
 283
 284 static inline int
 285 __attribute__ ((always_inline))
 286 __lll_robust_timedlock (int *futex, const struct timespec *abstime,
 287                         int id, int private)
 288 {
 289   int result = 0;
 290   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
 291                         0))
 292     result = __lll_robust_timedlock_wait (futex, abstime, private);
 293   return result;
 294 }
 295 #define lll_robust_timedlock(futex, abstime, id, private) \
 296   __lll_robust_timedlock (&(futex), abstime, id, private)
 297
 298
 299 #define __lll_unlock(futex, private) \
 300   (void)                                                                      \
 301     ({ int __oldval;                                                          \
 302        int __newval = 0;                                                      \
 303        int *__futexp = (futex);                                               \
 304                                                                               \
 305        lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
 306        if (__builtin_expect (__oldval > 1, 0))                                \
 307          lll_futex_wake (__futexp, 1, private);                               \
 308     })
 309 #define lll_unlock(futex, private) __lll_unlock(&(futex), private)
 310
 311
 312 #define __lll_robust_unlock(futex, private) \
 313   (void)                                                                      \
 314     ({ int __oldval;                                                          \
 315        int __newval = 0;                                                      \
 316        int *__futexp = (futex);                                               \
 317                                                                               \
 318        lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
 319        if (__builtin_expect (__oldval & FUTEX_WAITERS, 0))                    \
 320          lll_futex_wake (__futexp, 1, private);                               \
 321     })
 322 #define lll_robust_unlock(futex, private) \
 323   __lll_robust_unlock(&(futex), private)
 324
 325 #define lll_islocked(futex) \
 326   (futex != 0)
 327
 328
 329 /* Initializers for lock.  */
 330 #define LLL_LOCK_INITIALIZER            (0)
 331 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 332
 333 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
 334    wakeup when the clone terminates.  The memory location contains the
 335    thread ID while the clone is running and is reset to zero
 336    afterwards.  */
 337 #define __lll_wait_tid(ptid) \
 338   do                                                                          \
 339     {                                                                         \
 340       int __tid;                                                              \
 341                                                                               \
 342       while ((__tid = *ptid) != 0)                                            \
 343         lll_futex_wait (ptid, __tid, LLL_SHARED);                             \
 344     }                                                                         \
 345   while (0)
 346 #define lll_wait_tid(tid) __lll_wait_tid(&(tid))
 347
 348 extern int __lll_timedwait_tid (int *, const struct timespec *)
 349      attribute_hidden;
 350
 351 #define lll_timedwait_tid(tid, abstime) \
 352   ({                                                                          \
 353     int __res = 0;                                                            \
 354     if ((tid) != 0)                                                           \
 355       __res = __lll_timedwait_tid (&(tid), (abstime));                        \
 356     __res;                                                                    \
 357   })
 358
 359 #endif  /* lowlevellock.h */