sysdeps/unix/sysv/linux/i386/lowlevellock.h

   1 /* Copyright (C) 2002-2014 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #ifndef _LOWLEVELLOCK_H
  20 #define _LOWLEVELLOCK_H 1
  21
  22 #include <stap-probe.h>
  23
  24 #ifndef __ASSEMBLER__
  25 # include <time.h>
  26 # include <sys/param.h>
  27 # include <bits/pthreadtypes.h>
  28 # include <kernel-features.h>
  29 # include <tcb-offsets.h>
  30
  31 # ifndef LOCK_INSTR
  32 #  ifdef UP
  33 #   define LOCK_INSTR   /* nothing */
  34 #  else
  35 #   define LOCK_INSTR "lock;"
  36 #  endif
  37 # endif
  38 #else
  39 # ifndef LOCK
  40 #  ifdef UP
  41 #   define LOCK
  42 #  else
  43 #   define LOCK lock
  44 #  endif
  45 # endif
  46 #endif
  47
  48 #define SYS_futex               240
  49 #define FUTEX_WAIT              0
  50 #define FUTEX_WAKE              1
  51 #define FUTEX_CMP_REQUEUE       4
  52 #define FUTEX_WAKE_OP           5
  53 #define FUTEX_LOCK_PI           6
  54 #define FUTEX_UNLOCK_PI         7
  55 #define FUTEX_TRYLOCK_PI        8
  56 #define FUTEX_WAIT_BITSET       9
  57 #define FUTEX_WAKE_BITSET       10
  58 #define FUTEX_WAIT_REQUEUE_PI   11
  59 #define FUTEX_CMP_REQUEUE_PI    12
  60 #define FUTEX_PRIVATE_FLAG      128
  61 #define FUTEX_CLOCK_REALTIME    256
  62
  63 #define FUTEX_BITSET_MATCH_ANY  0xffffffff
  64
  65 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  66
  67 /* Values for 'private' parameter of locking macros.  Yes, the
  68    definition seems to be backwards.  But it is not.  The bit will be
  69    reversed before passing to the system call.  */
  70 #define LLL_PRIVATE     0
  71 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  72
  73
  74 #if !defined NOT_IN_libc || defined IS_IN_rtld
  75 /* In libc.so or ld.so all futexes are private.  */
  76 # ifdef __ASSUME_PRIVATE_FUTEX
  77 #  define __lll_private_flag(fl, private) \
  78   ((fl) | FUTEX_PRIVATE_FLAG)
  79 # else
  80 #  define __lll_private_flag(fl, private) \
  81   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  82 # endif
  83 #else
  84 # ifdef __ASSUME_PRIVATE_FUTEX
  85 #  define __lll_private_flag(fl, private) \
  86   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  87 # else
  88 #  define __lll_private_flag(fl, private) \
  89   (__builtin_constant_p (private)                                             \
  90    ? ((private) == 0                                                          \
  91       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  92       : (fl))                                                                 \
  93    : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG);                 \
  94         asm ("andl %%gs:%P1, %0" : "+r" (__fl)                                \
  95              : "i" (offsetof (struct pthread, header.private_futex)));        \
  96         __fl | (fl); }))
  97 # endif
  98 #endif
  99
 100 #ifndef __ASSEMBLER__
 101
 102 /* Initializer for compatibility lock.  */
 103 #define LLL_LOCK_INITIALIZER            (0)
 104 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 105 #define LLL_LOCK_INITIALIZER_WAITERS    (2)
 106
 107
 108 #ifdef PIC
 109 # define LLL_EBX_LOAD   "xchgl %2, %%ebx\n"
 110 # define LLL_EBX_REG    "D"
 111 #else
 112 # define LLL_EBX_LOAD
 113 # define LLL_EBX_REG    "b"
 114 #endif
 115
 116 #ifdef I386_USE_SYSENTER
 117 # ifdef SHARED
 118 #  define LLL_ENTER_KERNEL      "call *%%gs:%P6\n\t"
 119 # else
 120 #  define LLL_ENTER_KERNEL      "call *_dl_sysinfo\n\t"
 121 # endif
 122 #else
 123 # define LLL_ENTER_KERNEL       "int $0x80\n\t"
 124 #endif
 125
 126 /* Delay in spinlock loop.  */
 127 #define BUSY_WAIT_NOP   asm ("rep; nop")
 128
 129 #define lll_futex_wait(futex, val, private) \
 130   lll_futex_timed_wait (futex, val, NULL, private)
 131
 132
 133 #define lll_futex_timed_wait(futex, val, timeout, private) \
 134   ({                                                                          \
 135     int __status;                                                             \
 136     register __typeof (val) _val asm ("edx") = (val);                         \
 137     __asm __volatile (LLL_EBX_LOAD                                            \
 138                       LLL_ENTER_KERNEL                                        \
 139                       LLL_EBX_LOAD                                            \
 140                       : "=a" (__status)                                       \
 141                       : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
 142                         "c" (__lll_private_flag (FUTEX_WAIT, private)),       \
 143                         "d" (_val), "i" (offsetof (tcbhead_t, sysinfo))       \
 144                       : "memory");                                            \
 145     __status;                                                                 \
 146   })
 147
 148
 149 #define lll_futex_wake(futex, nr, private) \
 150   ({                                                                          \
 151     int __status;                                                             \
 152     register __typeof (nr) _nr asm ("edx") = (nr);                            \
 153     LIBC_PROBE (lll_futex_wake, 3, futex, nr, private);                       \
 154     __asm __volatile (LLL_EBX_LOAD                                            \
 155                       LLL_ENTER_KERNEL                                        \
 156                       LLL_EBX_LOAD                                            \
 157                       : "=a" (__status)                                       \
 158                       : "0" (SYS_futex), LLL_EBX_REG (futex),                 \
 159                         "c" (__lll_private_flag (FUTEX_WAKE, private)),       \
 160                         "d" (_nr),                                            \
 161                         "i" (0) /* phony, to align next arg's number */,      \
 162                         "i" (offsetof (tcbhead_t, sysinfo)));                 \
 163     __status;                                                                 \
 164   })
 165
 166
 167 /* NB: in the lll_trylock macro we simply return the value in %eax
 168    after the cmpxchg instruction.  In case the operation succeded this
 169    value is zero.  In case the operation failed, the cmpxchg instruction
 170    has loaded the current value of the memory work which is guaranteed
 171    to be nonzero.  */
 172 #if defined NOT_IN_libc || defined UP
 173 # define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
 174 #else
 175 # define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
 176                            "je 0f\n\t"                                        \
 177                            "lock\n"                                           \
 178                            "0:\tcmpxchgl %2, %1"
 179 #endif
 180
 181 #define lll_trylock(futex) \
 182   ({ int ret;                                                                 \
 183      __asm __volatile (__lll_trylock_asm                                      \
 184                        : "=a" (ret), "=m" (futex)                             \
 185                        : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
 186                          "0" (LLL_LOCK_INITIALIZER),                          \
 187                          "i" (MULTIPLE_THREADS_OFFSET)                        \
 188                        : "memory");                                           \
 189      ret; })
 190
 191
 192 #define lll_cond_trylock(futex) \
 193   ({ int ret;                                                                 \
 194      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"                           \
 195                        : "=a" (ret), "=m" (futex)                             \
 196                        : "r" (LLL_LOCK_INITIALIZER_WAITERS),                  \
 197                          "m" (futex), "0" (LLL_LOCK_INITIALIZER)              \
 198                        : "memory");                                           \
 199      ret; })
 200
 201 #if defined NOT_IN_libc || defined UP
 202 # define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
 203 #else
 204 # define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t"                         \
 205                               "je 0f\n\t"                                     \
 206                               "lock\n"                                        \
 207                               "0:\tcmpxchgl %1, %2\n\t"
 208 #endif
 209
 210 #define lll_lock(futex, private) \
 211   (void)                                                                      \
 212     ({ int ignore1, ignore2;                                                  \
 213        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 214          __asm __volatile (__lll_lock_asm_start                               \
 215                            "jz 18f\n\t"                               \
 216                            "1:\tleal %2, %%ecx\n"                             \
 217                            "2:\tcall __lll_lock_wait_private\n"               \
 218                            "18:"                                              \
 219                            : "=a" (ignore1), "=c" (ignore2), "=m" (futex)     \
 220                            : "0" (0), "1" (1), "m" (futex),                   \
 221                              "i" (MULTIPLE_THREADS_OFFSET)                    \
 222                            : "memory");                                       \
 223        else                                                                   \
 224          {                                                                    \
 225            int ignore3;                                                       \
 226            __asm __volatile (__lll_lock_asm_start                             \
 227                              "jz 18f\n\t"                                     \
 228                              "1:\tleal %2, %%edx\n"                           \
 229                              "0:\tmovl %8, %%ecx\n"                           \
 230                              "2:\tcall __lll_lock_wait\n"                     \
 231                              "18:"                                            \
 232                              : "=a" (ignore1), "=c" (ignore2),                \
 233                                "=m" (futex), "=&d" (ignore3)                  \
 234                              : "1" (1), "m" (futex),                          \
 235                                "i" (MULTIPLE_THREADS_OFFSET), "0" (0),        \
 236                                "g" ((int) (private))                          \
 237                              : "memory");                                     \
 238          }                                                                    \
 239     })
 240
 241 #define lll_robust_lock(futex, id, private) \
 242   ({ int result, ignore1, ignore2;                                            \
 243      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"                       \
 244                        "jz 18f\n\t"                                           \
 245                        "1:\tleal %2, %%edx\n"                                 \
 246                        "0:\tmovl %7, %%ecx\n"                                 \
 247                        "2:\tcall __lll_robust_lock_wait\n"                    \
 248                        "18:"                                                  \
 249                        : "=a" (result), "=c" (ignore1), "=m" (futex),         \
 250                          "=&d" (ignore2)                                      \
 251                        : "0" (0), "1" (id), "m" (futex), "g" ((int) (private))\
 252                        : "memory");                                           \
 253      result; })
 254
 255
 256 /* Special version of lll_lock which causes the unlock function to
 257    always wakeup waiters.  */
 258 #define lll_cond_lock(futex, private) \
 259   (void)                                                                      \
 260     ({ int ignore1, ignore2, ignore3;                                         \
 261        __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"                     \
 262                          "jz 18f\n\t"                                         \
 263                          "1:\tleal %2, %%edx\n"                               \
 264                          "0:\tmovl %7, %%ecx\n"                               \
 265                          "2:\tcall __lll_lock_wait\n"                         \
 266                          "18:"                                                \
 267                          : "=a" (ignore1), "=c" (ignore2), "=m" (futex),      \
 268                            "=&d" (ignore3)                                    \
 269                          : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
 270                          : "memory");                                         \
 271     })
 272
 273
 274 #define lll_robust_cond_lock(futex, id, private) \
 275   ({ int result, ignore1, ignore2;                                            \
 276      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"                       \
 277                        "jz 18f\n\t"                                           \
 278                        "1:\tleal %2, %%edx\n"                                 \
 279                        "0:\tmovl %7, %%ecx\n"                                 \
 280                        "2:\tcall __lll_robust_lock_wait\n"                    \
 281                        "18:"                                                  \
 282                        : "=a" (result), "=c" (ignore1), "=m" (futex),         \
 283                          "=&d" (ignore2)                                      \
 284                        : "0" (0), "1" (id | FUTEX_WAITERS), "m" (futex),      \
 285                          "g" ((int) (private))                                \
 286                        : "memory");                                           \
 287      result; })
 288
 289
 290 #define lll_timedlock(futex, timeout, private) \
 291   ({ int result, ignore1, ignore2, ignore3;                                   \
 292      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"                       \
 293                        "jz 18f\n\t"                                           \
 294                        "1:\tleal %3, %%ecx\n"                                 \
 295                        "0:\tmovl %8, %%edx\n"                                 \
 296                        "2:\tcall __lll_timedlock_wait\n"                      \
 297                        "18:"                                                  \
 298                        : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
 299                          "=m" (futex), "=S" (ignore3)                         \
 300                        : "0" (0), "1" (1), "m" (futex), "m" (timeout),        \
 301                          "4" ((int) (private))                                \
 302                        : "memory");                                           \
 303      result; })
 304
 305 extern int __lll_timedlock_elision (int *futex, short *adapt_count,
 306                                          const struct timespec *timeout,
 307                                          int private) attribute_hidden;
 308
 309 #define lll_timedlock_elision(futex, adapt_count, timeout, private)     \
 310   __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
 311
 312 #define lll_robust_timedlock(futex, timeout, id, private) \
 313   ({ int result, ignore1, ignore2, ignore3;                                   \
 314      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"                       \
 315                        "jz 18f\n\t"                                           \
 316                        "1:\tleal %3, %%ecx\n"                                 \
 317                        "0:\tmovl %8, %%edx\n"                                 \
 318                        "2:\tcall __lll_robust_timedlock_wait\n"               \
 319                        "18:"                                                  \
 320                        : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
 321                          "=m" (futex), "=S" (ignore3)                         \
 322                        : "0" (0), "1" (id), "m" (futex), "m" (timeout),       \
 323                          "4" ((int) (private))                                \
 324                        : "memory");                                           \
 325      result; })
 326
 327 #if defined NOT_IN_libc || defined UP
 328 # define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
 329 #else
 330 # define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t"                             \
 331                           "je 0f\n\t"                                         \
 332                           "lock\n"                                            \
 333                           "0:\tsubl $1,%0\n\t"
 334 #endif
 335
 336 #define lll_unlock(futex, private) \
 337   (void)                                                                      \
 338     ({ int ignore;                                                            \
 339        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 340          __asm __volatile (__lll_unlock_asm                                   \
 341                            "je 18f\n\t"                                       \
 342                            "1:\tleal %0, %%eax\n"                             \
 343                            "2:\tcall __lll_unlock_wake_private\n"             \
 344                            "18:"                                              \
 345                            : "=m" (futex), "=&a" (ignore)                     \
 346                            : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET)       \
 347                            : "memory");                                       \
 348        else                                                                   \
 349          {                                                                    \
 350            int ignore2;                                                       \
 351            __asm __volatile (__lll_unlock_asm                                 \
 352                              "je 18f\n\t"                                     \
 353                              "1:\tleal %0, %%eax\n"                           \
 354                              "0:\tmovl %5, %%ecx\n"                           \
 355                              "2:\tcall __lll_unlock_wake\n"                   \
 356                              "18:"                                            \
 357                              : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)  \
 358                              : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex),    \
 359                                "g" ((int) (private))                          \
 360                              : "memory");                                     \
 361          }                                                                    \
 362     })
 363
 364 #define lll_robust_unlock(futex, private) \
 365   (void)                                                                      \
 366     ({ int ignore, ignore2;                                                   \
 367        __asm __volatile (LOCK_INSTR "andl %3, %0\n\t"                         \
 368                          "je 18f\n\t"                                         \
 369                          "1:\tleal %0, %%eax\n"                               \
 370                          "0:\tmovl %5, %%ecx\n"                               \
 371                          "2:\tcall __lll_unlock_wake\n"                       \
 372                          "18:"                                                \
 373                          : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)      \
 374                          : "i" (FUTEX_WAITERS), "m" (futex),                  \
 375                            "g" ((int) (private))                              \
 376                          : "memory");                                         \
 377     })
 378
 379
 380 #define lll_islocked(futex) \
 381   (futex != LLL_LOCK_INITIALIZER)
 382
 383 /* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
 384    wakeup when the clone terminates.  The memory location contains the
 385    thread ID while the clone is running and is reset to zero
 386    afterwards.
 387
 388    The macro parameter must not have any side effect.  */
 389 #define lll_wait_tid(tid) \
 390   do {                                                                        \
 391     int __ignore;                                                             \
 392     register __typeof (tid) _tid asm ("edx") = (tid);                         \
 393     if (_tid != 0)                                                            \
 394       __asm __volatile (LLL_EBX_LOAD                                          \
 395                         "1:\tmovl %1, %%eax\n\t"                              \
 396                         LLL_ENTER_KERNEL                                      \
 397                         "cmpl $0, (%%ebx)\n\t"                                \
 398                         "jne 1b\n\t"                                          \
 399                         LLL_EBX_LOAD                                          \
 400                         : "=&a" (__ignore)                                    \
 401                         : "i" (SYS_futex), LLL_EBX_REG (&tid), "S" (0),       \
 402                           "c" (FUTEX_WAIT), "d" (_tid),                       \
 403                           "i" (offsetof (tcbhead_t, sysinfo))                 \
 404                         : "memory");                                          \
 405   } while (0)
 406
 407 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
 408      __attribute__ ((regparm (2))) attribute_hidden;
 409 #define lll_timedwait_tid(tid, abstime) \
 410   ({                                                                          \
 411     int __result = 0;                                                         \
 412     if (tid != 0)                                                             \
 413       {                                                                       \
 414         if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)           \
 415           __result = EINVAL;                                                  \
 416         else                                                                  \
 417           __result = __lll_timedwait_tid (&tid, abstime);                     \
 418       }                                                                       \
 419     __result; })
 420
 421 extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
 422   attribute_hidden;
 423
 424 extern int __lll_unlock_elision(int *lock, int private)
 425   attribute_hidden;
 426
 427 extern int __lll_trylock_elision(int *lock, short *adapt_count)
 428   attribute_hidden;
 429
 430 #define lll_lock_elision(futex, adapt_count, private) \
 431   __lll_lock_elision (&(futex), &(adapt_count), private)
 432 #define lll_unlock_elision(futex, private) \
 433   __lll_unlock_elision (&(futex), private)
 434 #define lll_trylock_elision(futex, adapt_count) \
 435   __lll_trylock_elision(&(futex), &(adapt_count))
 436
 437 #endif  /* !__ASSEMBLER__ */
 438
 439 #endif  /* lowlevellock.h */