nptl/pthread_mutex_timedlock.c

   1 /* Copyright (C) 2002-2017 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <time.h>
  22 #include <sys/param.h>
  23 #include <sys/time.h>
  24 #include "pthreadP.h"
  25 #include <atomic.h>
  26 #include <lowlevellock.h>
  27 #include <not-cancel.h>
  28
  29 #include <stap-probe.h>
  30
  31 #ifndef lll_timedlock_elision
  32 #define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c)
  33 #endif
  34
  35 #ifndef lll_trylock_elision
  36 #define lll_trylock_elision(a,t) lll_trylock(a)
  37 #endif
  38
  39 #ifndef FORCE_ELISION
  40 #define FORCE_ELISION(m, s)
  41 #endif
  42
  43 int
  44 pthread_mutex_timedlock (pthread_mutex_t *mutex,
  45                          const struct timespec *abstime)
  46 {
  47   int oldval;
  48   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  49   int result = 0;
  50
  51   LIBC_PROBE (mutex_timedlock_entry, 2, mutex, abstime);
  52
  53   /* We must not check ABSTIME here.  If the thread does not block
  54      abstime must not be checked for a valid value.  */
  55
  56   switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
  57                             PTHREAD_MUTEX_TIMED_NP))
  58     {
  59       /* Recursive mutex.  */
  60     case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
  61     case PTHREAD_MUTEX_RECURSIVE_NP:
  62       /* Check whether we already hold the mutex.  */
  63       if (mutex->__data.__owner == id)
  64         {
  65           /* Just bump the counter.  */
  66           if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
  67             /* Overflow of the counter.  */
  68             return EAGAIN;
  69
  70           ++mutex->__data.__count;
  71
  72           goto out;
  73         }
  74
  75       /* We have to get the mutex.  */
  76       result = lll_timedlock (mutex->__data.__lock, abstime,
  77                               PTHREAD_MUTEX_PSHARED (mutex));
  78
  79       if (result != 0)
  80         goto out;
  81
  82       /* Only locked once so far.  */
  83       mutex->__data.__count = 1;
  84       break;
  85
  86       /* Error checking mutex.  */
  87     case PTHREAD_MUTEX_ERRORCHECK_NP:
  88       /* Check whether we already hold the mutex.  */
  89       if (__glibc_unlikely (mutex->__data.__owner == id))
  90         return EDEADLK;
  91
  92       /* Don't do lock elision on an error checking mutex.  */
  93       goto simple;
  94
  95     case PTHREAD_MUTEX_TIMED_NP:
  96       FORCE_ELISION (mutex, goto elision);
  97     simple:
  98       /* Normal mutex.  */
  99       result = lll_timedlock (mutex->__data.__lock, abstime,
 100                               PTHREAD_MUTEX_PSHARED (mutex));
 101       break;
 102
 103     case PTHREAD_MUTEX_TIMED_ELISION_NP:
 104     elision: __attribute__((unused))
 105       /* Don't record ownership */
 106       return lll_timedlock_elision (mutex->__data.__lock,
 107                                     mutex->__data.__spins,
 108                                     abstime,
 109                                     PTHREAD_MUTEX_PSHARED (mutex));
 110
 111
 112     case PTHREAD_MUTEX_ADAPTIVE_NP:
 113       if (! __is_smp)
 114         goto simple;
 115
 116       if (lll_trylock (mutex->__data.__lock) != 0)
 117         {
 118           int cnt = 0;
 119           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
 120                              mutex->__data.__spins * 2 + 10);
 121           do
 122             {
 123               if (cnt++ >= max_cnt)
 124                 {
 125                   result = lll_timedlock (mutex->__data.__lock, abstime,
 126                                           PTHREAD_MUTEX_PSHARED (mutex));
 127                   break;
 128                 }
 129               atomic_spin_nop ();
 130             }
 131           while (lll_trylock (mutex->__data.__lock) != 0);
 132
 133           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 134         }
 135       break;
 136
 137     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 138     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 139     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 140     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 141       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 142                      &mutex->__data.__list.__next);
 143
 144       oldval = mutex->__data.__lock;
 145       /* This is set to FUTEX_WAITERS iff we might have shared the
 146          FUTEX_WAITERS flag with other threads, and therefore need to keep it
 147          set to avoid lost wake-ups.  We have the same requirement in the
 148          simple mutex algorithm.  */
 149       unsigned int assume_other_futex_waiters = 0;
 150       do
 151         {
 152         again:
 153           if ((oldval & FUTEX_OWNER_DIED) != 0)
 154             {
 155               /* The previous owner died.  Try locking the mutex.  */
 156               int newval = id | (oldval & FUTEX_WAITERS)
 157                   | assume_other_futex_waiters;
 158
 159               newval
 160                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 161                                                        newval, oldval);
 162               if (newval != oldval)
 163                 {
 164                   oldval = newval;
 165                   goto again;
 166                 }
 167
 168               /* We got the mutex.  */
 169               mutex->__data.__count = 1;
 170               /* But it is inconsistent unless marked otherwise.  */
 171               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 172
 173               ENQUEUE_MUTEX (mutex);
 174               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 175
 176               /* Note that we deliberately exit here.  If we fall
 177                  through to the end of the function __nusers would be
 178                  incremented which is not correct because the old
 179                  owner has to be discounted.  */
 180               return EOWNERDEAD;
 181             }
 182
 183           /* Check whether we already hold the mutex.  */
 184           if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 185             {
 186               int kind = PTHREAD_MUTEX_TYPE (mutex);
 187               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 188                 {
 189                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 190                                  NULL);
 191                   return EDEADLK;
 192                 }
 193
 194               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 195                 {
 196                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 197                                  NULL);
 198
 199                   /* Just bump the counter.  */
 200                   if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 201                     /* Overflow of the counter.  */
 202                     return EAGAIN;
 203
 204                   ++mutex->__data.__count;
 205
 206                   LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 207
 208                   return 0;
 209                 }
 210             }
 211
 212           result = lll_robust_timedlock (mutex->__data.__lock, abstime,
 213                                          id | assume_other_futex_waiters,
 214                                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 215           /* See above.  We set FUTEX_WAITERS and might have shared this flag
 216              with other threads; thus, we need to preserve it.  */
 217           assume_other_futex_waiters = FUTEX_WAITERS;
 218
 219           if (__builtin_expect (mutex->__data.__owner
 220                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 221             {
 222               /* This mutex is now not recoverable.  */
 223               mutex->__data.__count = 0;
 224               lll_unlock (mutex->__data.__lock,
 225                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 226               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 227               return ENOTRECOVERABLE;
 228             }
 229
 230           if (result == ETIMEDOUT || result == EINVAL)
 231             goto out;
 232
 233           oldval = result;
 234         }
 235       while ((oldval & FUTEX_OWNER_DIED) != 0);
 236
 237       mutex->__data.__count = 1;
 238       ENQUEUE_MUTEX (mutex);
 239       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 240       break;
 241
 242     /* The PI support requires the Linux futex system call.  If that's not
 243        available, pthread_mutex_init should never have allowed the type to
 244        be set.  So it will get the default case for an invalid type.  */
 245 #ifdef __NR_futex
 246     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 247     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 248     case PTHREAD_MUTEX_PI_NORMAL_NP:
 249     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 250     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 251     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 252     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 253     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 254       {
 255         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 256         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 257
 258         if (robust)
 259           /* Note: robust PI futexes are signaled by setting bit 0.  */
 260           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 261                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 262                                    | 1));
 263
 264         oldval = mutex->__data.__lock;
 265
 266         /* Check whether we already hold the mutex.  */
 267         if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 268           {
 269             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 270               {
 271                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 272                 return EDEADLK;
 273               }
 274
 275             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 276               {
 277                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 278
 279                 /* Just bump the counter.  */
 280                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 281                   /* Overflow of the counter.  */
 282                   return EAGAIN;
 283
 284                 ++mutex->__data.__count;
 285
 286                 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 287
 288                 return 0;
 289               }
 290           }
 291
 292         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 293                                                       id, 0);
 294
 295         if (oldval != 0)
 296           {
 297             /* The mutex is locked.  The kernel will now take care of
 298                everything.  The timeout value must be a relative value.
 299                Convert it.  */
 300             int private = (robust
 301                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 302                            : PTHREAD_MUTEX_PSHARED (mutex));
 303             INTERNAL_SYSCALL_DECL (__err);
 304
 305             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 306                                       __lll_private_flag (FUTEX_LOCK_PI,
 307                                                           private), 1,
 308                                       abstime);
 309             if (INTERNAL_SYSCALL_ERROR_P (e, __err))
 310               {
 311                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
 312                   return ETIMEDOUT;
 313
 314                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 315                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
 316                   {
 317                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 318                             || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 319                                 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 320                     /* ESRCH can happen only for non-robust PI mutexes where
 321                        the owner of the lock died.  */
 322                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
 323                             || !robust);
 324
 325                     /* Delay the thread until the timeout is reached.
 326                        Then return ETIMEDOUT.  */
 327                     struct timespec reltime;
 328                     struct timespec now;
 329
 330                     INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME,
 331                                       &now);
 332                     reltime.tv_sec = abstime->tv_sec - now.tv_sec;
 333                     reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
 334                     if (reltime.tv_nsec < 0)
 335                       {
 336                         reltime.tv_nsec += 1000000000;
 337                         --reltime.tv_sec;
 338                       }
 339                     if (reltime.tv_sec >= 0)
 340                       while (nanosleep_not_cancel (&reltime, &reltime) != 0)
 341                         continue;
 342
 343                     return ETIMEDOUT;
 344                   }
 345
 346                 return INTERNAL_SYSCALL_ERRNO (e, __err);
 347               }
 348
 349             oldval = mutex->__data.__lock;
 350
 351             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 352           }
 353
 354         if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
 355           {
 356             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 357
 358             /* We got the mutex.  */
 359             mutex->__data.__count = 1;
 360             /* But it is inconsistent unless marked otherwise.  */
 361             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 362
 363             ENQUEUE_MUTEX_PI (mutex);
 364             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 365
 366             /* Note that we deliberately exit here.  If we fall
 367                through to the end of the function __nusers would be
 368                incremented which is not correct because the old owner
 369                has to be discounted.  */
 370             return EOWNERDEAD;
 371           }
 372
 373         if (robust
 374             && __builtin_expect (mutex->__data.__owner
 375                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 376           {
 377             /* This mutex is now not recoverable.  */
 378             mutex->__data.__count = 0;
 379
 380             INTERNAL_SYSCALL_DECL (__err);
 381             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 382                               __lll_private_flag (FUTEX_UNLOCK_PI,
 383                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 384                               0, 0);
 385
 386             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 387             return ENOTRECOVERABLE;
 388           }
 389
 390         mutex->__data.__count = 1;
 391         if (robust)
 392           {
 393             ENQUEUE_MUTEX_PI (mutex);
 394             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 395           }
 396         }
 397       break;
 398 #endif  /* __NR_futex.  */
 399
 400     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 401     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 402     case PTHREAD_MUTEX_PP_NORMAL_NP:
 403     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 404       {
 405         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 406
 407         oldval = mutex->__data.__lock;
 408
 409         /* Check whether we already hold the mutex.  */
 410         if (mutex->__data.__owner == id)
 411           {
 412             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 413               return EDEADLK;
 414
 415             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 416               {
 417                 /* Just bump the counter.  */
 418                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 419                   /* Overflow of the counter.  */
 420                   return EAGAIN;
 421
 422                 ++mutex->__data.__count;
 423
 424                 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 425
 426                 return 0;
 427               }
 428           }
 429
 430         int oldprio = -1, ceilval;
 431         do
 432           {
 433             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 434                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 435
 436             if (__pthread_current_priority () > ceiling)
 437               {
 438                 result = EINVAL;
 439               failpp:
 440                 if (oldprio != -1)
 441                   __pthread_tpp_change_priority (oldprio, -1);
 442                 return result;
 443               }
 444
 445             result = __pthread_tpp_change_priority (oldprio, ceiling);
 446             if (result)
 447               return result;
 448
 449             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 450             oldprio = ceiling;
 451
 452             oldval
 453               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 454                                                      ceilval | 1, ceilval);
 455
 456             if (oldval == ceilval)
 457               break;
 458
 459             do
 460               {
 461                 oldval
 462                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 463                                                          ceilval | 2,
 464                                                          ceilval | 1);
 465
 466                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 467                   break;
 468
 469                 if (oldval != ceilval)
 470                   {
 471                     /* Reject invalid timeouts.  */
 472                     if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
 473                       {
 474                         result = EINVAL;
 475                         goto failpp;
 476                       }
 477
 478                     struct timeval tv;
 479                     struct timespec rt;
 480
 481                     /* Get the current time.  */
 482                     (void) __gettimeofday (&tv, NULL);
 483
 484                     /* Compute relative timeout.  */
 485                     rt.tv_sec = abstime->tv_sec - tv.tv_sec;
 486                     rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
 487                     if (rt.tv_nsec < 0)
 488                       {
 489                         rt.tv_nsec += 1000000000;
 490                         --rt.tv_sec;
 491                       }
 492
 493                     /* Already timed out?  */
 494                     if (rt.tv_sec < 0)
 495                       {
 496                         result = ETIMEDOUT;
 497                         goto failpp;
 498                       }
 499
 500                     lll_futex_timed_wait (&mutex->__data.__lock,
 501                                           ceilval | 2, &rt,
 502                                           PTHREAD_MUTEX_PSHARED (mutex));
 503                   }
 504               }
 505             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 506                                                         ceilval | 2, ceilval)
 507                    != ceilval);
 508           }
 509         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 510
 511         assert (mutex->__data.__owner == 0);
 512         mutex->__data.__count = 1;
 513       }
 514       break;
 515
 516     default:
 517       /* Correct code cannot set any other type.  */
 518       return EINVAL;
 519     }
 520
 521   if (result == 0)
 522     {
 523       /* Record the ownership.  */
 524       mutex->__data.__owner = id;
 525       ++mutex->__data.__nusers;
 526
 527       LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 528     }
 529
 530  out:
 531   return result;
 532 }