[thirdparty/glibc.git] / nptl / pthread_mutex_trylock.c

/* Copyright (C) 2002-2017 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <assert.h>
#include <errno.h>
#include <stdlib.h>
#include "pthreadP.h"
#include <lowlevellock.h>

#ifndef lll_trylock_elision
#define lll_trylock_elision(a,t) lll_trylock(a)
#endif

#ifndef FORCE_ELISION
#define FORCE_ELISION(m, s)
#endif

int
__pthread_mutex_trylock (pthread_mutex_t *mutex)
{
  int oldval;
  pid_t id = THREAD_GETMEM (THREAD_SELF, tid);

  switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
			    PTHREAD_MUTEX_TIMED_NP))
    {
      /* Recursive mutex.  */
    case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
    case PTHREAD_MUTEX_RECURSIVE_NP:
      /* Check whether we already hold the mutex.  */
      if (mutex->__data.__owner == id)
	{
	  /* Just bump the counter.  */
	  if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
	    /* Overflow of the counter.  */
	    return EAGAIN;

	  ++mutex->__data.__count;
	  return 0;
	}

      if (lll_trylock (mutex->__data.__lock) == 0)
	{
	  /* Record the ownership.  */
	  mutex->__data.__owner = id;
	  mutex->__data.__count = 1;
	  ++mutex->__data.__nusers;
	  return 0;
	}
      break;

    case PTHREAD_MUTEX_TIMED_ELISION_NP:
    elision: __attribute__((unused))
      if (lll_trylock_elision (mutex->__data.__lock,
			       mutex->__data.__elision) != 0)
	break;
      /* Don't record the ownership.  */
      return 0;

    case PTHREAD_MUTEX_TIMED_NP:
      FORCE_ELISION (mutex, goto elision);
      /*FALL THROUGH*/
    case PTHREAD_MUTEX_ADAPTIVE_NP:
    case PTHREAD_MUTEX_ERRORCHECK_NP:
      if (lll_trylock (mutex->__data.__lock) != 0)
	break;

      /* Record the ownership.  */
      mutex->__data.__owner = id;
      ++mutex->__data.__nusers;

      return 0;

    case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
    case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
    case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
    case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
		     &mutex->__data.__list.__next);

      oldval = mutex->__data.__lock;
      do
	{
	again:
	  if ((oldval & FUTEX_OWNER_DIED) != 0)
	    {
	      /* The previous owner died.  Try locking the mutex.  */
	      int newval = id | (oldval & FUTEX_WAITERS);

	      newval
		= atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
						       newval, oldval);

	      if (newval != oldval)
		{
		  oldval = newval;
		  goto again;
		}

	      /* We got the mutex.  */
	      mutex->__data.__count = 1;
	      /* But it is inconsistent unless marked otherwise.  */
	      mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;

	      ENQUEUE_MUTEX (mutex);
	      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);

	      /* Note that we deliberately exist here.  If we fall
		 through to the end of the function __nusers would be
		 incremented which is not correct because the old
		 owner has to be discounted.  */
	      return EOWNERDEAD;
	    }

	  /* Check whether we already hold the mutex.  */
	  if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
	    {
	      int kind = PTHREAD_MUTEX_TYPE (mutex);
	      if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
		{
		  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
				 NULL);
		  return EDEADLK;
		}

	      if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
		{
		  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
				 NULL);

		  /* Just bump the counter.  */
		  if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
		    /* Overflow of the counter.  */
		    return EAGAIN;

		  ++mutex->__data.__count;

		  return 0;
		}
	    }

	  oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
							id, 0);
	  if (oldval != 0 && (oldval & FUTEX_OWNER_DIED) == 0)
	    {
	      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);

	      return EBUSY;
	    }

	  if (__builtin_expect (mutex->__data.__owner
				== PTHREAD_MUTEX_NOTRECOVERABLE, 0))
	    {
	      /* This mutex is now not recoverable.  */
	      mutex->__data.__count = 0;
	      if (oldval == id)
		lll_unlock (mutex->__data.__lock,
			    PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
	      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	      return ENOTRECOVERABLE;
	    }
	}
      while ((oldval & FUTEX_OWNER_DIED) != 0);

      ENQUEUE_MUTEX (mutex);
      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);

      mutex->__data.__owner = id;
      ++mutex->__data.__nusers;
      mutex->__data.__count = 1;

      return 0;

    /* The PI support requires the Linux futex system call.  If that's not
       available, pthread_mutex_init should never have allowed the type to
       be set.  So it will get the default case for an invalid type.  */
#ifdef __NR_futex
    case PTHREAD_MUTEX_PI_RECURSIVE_NP:
    case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
    case PTHREAD_MUTEX_PI_NORMAL_NP:
    case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
    case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
    case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
    case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
    case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
      {
	int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
	int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;

	if (robust)
	  /* Note: robust PI futexes are signaled by setting bit 0.  */
	  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
			 (void *) (((uintptr_t) &mutex->__data.__list.__next)
				   | 1));

	oldval = mutex->__data.__lock;

	/* Check whether we already hold the mutex.  */
	if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
	  {
	    if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
	      {
		THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
		return EDEADLK;
	      }

	    if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
	      {
		THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);

		/* Just bump the counter.  */
		if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
		  /* Overflow of the counter.  */
		  return EAGAIN;

		++mutex->__data.__count;

		return 0;
	      }
	  }

	oldval
	  = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
						 id, 0);

	if (oldval != 0)
	  {
	    if ((oldval & FUTEX_OWNER_DIED) == 0)
	      {
		THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);

		return EBUSY;
	      }

	    assert (robust);

	    /* The mutex owner died.  The kernel will now take care of
	       everything.  */
	    int private = (robust
			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
			   : PTHREAD_MUTEX_PSHARED (mutex));
	    INTERNAL_SYSCALL_DECL (__err);
	    int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
				      __lll_private_flag (FUTEX_TRYLOCK_PI,
							  private), 0, 0);

	    if (INTERNAL_SYSCALL_ERROR_P (e, __err)
		&& INTERNAL_SYSCALL_ERRNO (e, __err) == EWOULDBLOCK)
	      {
		THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);

		return EBUSY;
	      }

	    oldval = mutex->__data.__lock;
	  }

	if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
	  {
	    atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);

	    /* We got the mutex.  */
	    mutex->__data.__count = 1;
	    /* But it is inconsistent unless marked otherwise.  */
	    mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;

	    ENQUEUE_MUTEX (mutex);
	    THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);

	    /* Note that we deliberately exit here.  If we fall
	       through to the end of the function __nusers would be
	       incremented which is not correct because the old owner
	       has to be discounted.  */
	    return EOWNERDEAD;
	  }

	if (robust
	    && __builtin_expect (mutex->__data.__owner
				 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
	  {
	    /* This mutex is now not recoverable.  */
	    mutex->__data.__count = 0;

	    INTERNAL_SYSCALL_DECL (__err);
	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
			      __lll_private_flag (FUTEX_UNLOCK_PI,
						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
			      0, 0);

	    THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	    return ENOTRECOVERABLE;
	  }

	if (robust)
	  {
	    ENQUEUE_MUTEX_PI (mutex);
	    THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	  }

	mutex->__data.__owner = id;
	++mutex->__data.__nusers;
	mutex->__data.__count = 1;

	return 0;
      }
#endif  /* __NR_futex.  */

    case PTHREAD_MUTEX_PP_RECURSIVE_NP:
    case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
    case PTHREAD_MUTEX_PP_NORMAL_NP:
    case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
      {
	int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;

	oldval = mutex->__data.__lock;

	/* Check whether we already hold the mutex.  */
	if (mutex->__data.__owner == id)
	  {
	    if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
	      return EDEADLK;

	    if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
	      {
		/* Just bump the counter.  */
		if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
		  /* Overflow of the counter.  */
		  return EAGAIN;

		++mutex->__data.__count;

		return 0;
	      }
	  }

	int oldprio = -1, ceilval;
	do
	  {
	    int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
			  >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;

	    if (__pthread_current_priority () > ceiling)
	      {
		if (oldprio != -1)
		  __pthread_tpp_change_priority (oldprio, -1);
		return EINVAL;
	      }

	    int retval = __pthread_tpp_change_priority (oldprio, ceiling);
	    if (retval)
	      return retval;

	    ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
	    oldprio = ceiling;

	    oldval
	      = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
						     ceilval | 1, ceilval);

	    if (oldval == ceilval)
	      break;
	  }
	while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);

	if (oldval != ceilval)
	  {
	    __pthread_tpp_change_priority (oldprio, -1);
	    break;
	  }

	assert (mutex->__data.__owner == 0);
	/* Record the ownership.  */
	mutex->__data.__owner = id;
	++mutex->__data.__nusers;
	mutex->__data.__count = 1;

	return 0;
      }
      break;

    default:
      /* Correct code cannot set any other type.  */
      return EINVAL;
    }

  return EBUSY;
}

#ifndef __pthread_mutex_trylock
#ifndef pthread_mutex_trylock
strong_alias (__pthread_mutex_trylock, pthread_mutex_trylock)
#endif
#endif
Commit	Line	Data
bfff8b1b	1	/* Copyright (C) 2002-2017 Free Software Foundation, Inc.
76a50749 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
76a50749	18
1bcfb5a5	19	#include <assert.h>
76a50749	20	#include <errno.h>
1bcfb5a5	21	#include <stdlib.h>
76a50749 UD	22	#include "pthreadP.h"
	23	#include <lowlevellock.h>
	24
e8c659d7 AK	25	#ifndef lll_trylock_elision
	26	#define lll_trylock_elision(a,t) lll_trylock(a)
	27	#endif
	28
b0a3c164 AS	29	#ifndef FORCE_ELISION
b0a3c164 AS	30	#define FORCE_ELISION(m, s)
e8c659d7 AK	31	#endif
e8c659d7 AK	32
76a50749	33	int
9dd346ff	34	__pthread_mutex_trylock (pthread_mutex_t *mutex)
76a50749	35	{
683040c3	36	int oldval;
1bcfb5a5	37	pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
76a50749	38
e8c659d7	39	switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
ae1ad3ae	40	PTHREAD_MUTEX_TIMED_NP))
76a50749 UD	41	{
76a50749 UD	42	/* Recursive mutex. */
e8c659d7	43	case PTHREAD_MUTEX_RECURSIVE_NP\|PTHREAD_MUTEX_ELISION_NP:
76a50749 UD	44	case PTHREAD_MUTEX_RECURSIVE_NP:
76a50749 UD	45	/* Check whether we already hold the mutex. */
9a7178d6	46	if (mutex->__data.__owner == id)
76a50749 UD	47	{
76a50749 UD	48	/* Just bump the counter. */
a1ffb40e	49	if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
0a37669a UD	50	/* Overflow of the counter. */
	51	return EAGAIN;
	52
76a50749 UD	53	++mutex->__data.__count;
	54	return 0;
	55	}
	56
e51deae7	57	if (lll_trylock (mutex->__data.__lock) == 0)
76a50749 UD	58	{
76a50749 UD	59	/* Record the ownership. */
9a7178d6	60	mutex->__data.__owner = id;
76a50749	61	mutex->__data.__count = 1;
61623643	62	++mutex->__data.__nusers;
76a50749 UD	63	return 0;
	64	}
	65	break;
	66
e8c659d7	67	case PTHREAD_MUTEX_TIMED_ELISION_NP:
b0a3c164	68	elision: __attribute__((unused))
e8c659d7 AK	69	if (lll_trylock_elision (mutex->__data.__lock,
e8c659d7 AK	70	mutex->__data.__elision) != 0)
887865fc	71	break;
075b9322	72	/* Don't record the ownership. */
e8c659d7 AK	73	return 0;
e8c659d7 AK	74
76a50749	75	case PTHREAD_MUTEX_TIMED_NP:
b0a3c164	76	FORCE_ELISION (mutex, goto elision);
e8c659d7	77	/FALL THROUGH/
76a50749	78	case PTHREAD_MUTEX_ADAPTIVE_NP:
e8c659d7	79	case PTHREAD_MUTEX_ERRORCHECK_NP:
e51deae7	80	if (lll_trylock (mutex->__data.__lock) != 0)
1bcfb5a5 UD	81	break;
	82
	83	/* Record the ownership. */
	84	mutex->__data.__owner = id;
	85	++mutex->__data.__nusers;
	86
	87	return 0;
	88
0f6699ea UD	89	case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
	90	case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
	91	case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
	92	case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
	93	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
	94	&mutex->__data.__list.__next);
	95
683040c3 UD	96	oldval = mutex->__data.__lock;
683040c3 UD	97	do
1bcfb5a5	98	{
0f6699ea	99	again:
683040c3 UD	100	if ((oldval & FUTEX_OWNER_DIED) != 0)
	101	{
	102	/* The previous owner died. Try locking the mutex. */
113ad5fc UD	103	int newval = id \| (oldval & FUTEX_WAITERS);
	104
	105	newval
0f6699ea	106	= atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
113ad5fc	107	newval, oldval);
0f6699ea UD	108
0f6699ea UD	109	if (newval != oldval)
683040c3	110	{
683040c3	111	oldval = newval;
0f6699ea	112	goto again;
683040c3 UD	113	}
	114
	115	/* We got the mutex. */
	116	mutex->__data.__count = 1;
	117	/* But it is inconsistent unless marked otherwise. */
	118	mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
	119
	120	ENQUEUE_MUTEX (mutex);
0f6699ea	121	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
683040c3 UD	122
	123	/* Note that we deliberately exist here. If we fall
	124	through to the end of the function __nusers would be
	125	incremented which is not correct because the old
	126	owner has to be discounted. */
	127	return EOWNERDEAD;
	128	}
	129
683040c3	130	/* Check whether we already hold the mutex. */
a1ffb40e	131	if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
683040c3	132	{
5bd8a249 UD	133	int kind = PTHREAD_MUTEX_TYPE (mutex);
5bd8a249 UD	134	if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
0f6699ea UD	135	{
	136	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
	137	NULL);
	138	return EDEADLK;
	139	}
683040c3	140
5bd8a249	141	if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
683040c3	142	{
0f6699ea UD	143	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
	144	NULL);
	145
683040c3	146	/* Just bump the counter. */
a1ffb40e	147	if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
683040c3 UD	148	/* Overflow of the counter. */
	149	return EAGAIN;
	150
	151	++mutex->__data.__count;
	152
	153	return 0;
	154	}
	155	}
	156
887865fc RM	157	oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
887865fc RM	158	id, 0);
683040c3	159	if (oldval != 0 && (oldval & FUTEX_OWNER_DIED) == 0)
0f6699ea UD	160	{
	161	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	162
	163	return EBUSY;
	164	}
683040c3	165
683040c3 UD	166	if (__builtin_expect (mutex->__data.__owner
	167	== PTHREAD_MUTEX_NOTRECOVERABLE, 0))
	168	{
	169	/* This mutex is now not recoverable. */
	170	mutex->__data.__count = 0;
	171	if (oldval == id)
5bd8a249 UD	172	lll_unlock (mutex->__data.__lock,
5bd8a249 UD	173	PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
0f6699ea	174	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
683040c3 UD	175	return ENOTRECOVERABLE;
683040c3 UD	176	}
1bcfb5a5	177	}
683040c3	178	while ((oldval & FUTEX_OWNER_DIED) != 0);
1bcfb5a5 UD	179
1bcfb5a5 UD	180	ENQUEUE_MUTEX (mutex);
0f6699ea	181	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
1bcfb5a5 UD	182
	183	mutex->__data.__owner = id;
	184	++mutex->__data.__nusers;
683040c3 UD	185	mutex->__data.__count = 1;
	186
	187	return 0;
1bcfb5a5	188
184ee940 RM	189	/* The PI support requires the Linux futex system call. If that's not
	190	available, pthread_mutex_init should never have allowed the type to
	191	be set. So it will get the default case for an invalid type. */
	192	#ifdef __NR_futex
df47504c UD	193	case PTHREAD_MUTEX_PI_RECURSIVE_NP:
	194	case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
	195	case PTHREAD_MUTEX_PI_NORMAL_NP:
	196	case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
	197	case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
	198	case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
	199	case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
	200	case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
	201	{
	202	int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
	203	int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
	204
	205	if (robust)
	206	/* Note: robust PI futexes are signaled by setting bit 0. */
	207	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
	208	(void *) (((uintptr_t) &mutex->__data.__list.__next)
	209	\| 1));
	210
	211	oldval = mutex->__data.__lock;
	212
	213	/* Check whether we already hold the mutex. */
a1ffb40e	214	if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
df47504c UD	215	{
	216	if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
	217	{
	218	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	219	return EDEADLK;
	220	}
	221
	222	if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
	223	{
	224	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	225
	226	/* Just bump the counter. */
a1ffb40e	227	if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
df47504c UD	228	/* Overflow of the counter. */
	229	return EAGAIN;
	230
	231	++mutex->__data.__count;
	232
	233	return 0;
	234	}
	235	}
	236
	237	oldval
	238	= atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
	239	id, 0);
	240
	241	if (oldval != 0)
	242	{
	243	if ((oldval & FUTEX_OWNER_DIED) == 0)
	244	{
	245	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	246
	247	return EBUSY;
	248	}
	249
	250	assert (robust);
	251
	252	/* The mutex owner died. The kernel will now take care of
	253	everything. */
efac1fce UD	254	int private = (robust
	255	? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
	256	: PTHREAD_MUTEX_PSHARED (mutex));
df47504c UD	257	INTERNAL_SYSCALL_DECL (__err);
df47504c UD	258	int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
efac1fce UD	259	__lll_private_flag (FUTEX_TRYLOCK_PI,
efac1fce UD	260	private), 0, 0);
df47504c UD	261
	262	if (INTERNAL_SYSCALL_ERROR_P (e, __err)
	263	&& INTERNAL_SYSCALL_ERRNO (e, __err) == EWOULDBLOCK)
	264	{
	265	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	266
	267	return EBUSY;
	268	}
	269
	270	oldval = mutex->__data.__lock;
	271	}
	272
a1ffb40e	273	if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
df47504c UD	274	{
	275	atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
	276
	277	/* We got the mutex. */
	278	mutex->__data.__count = 1;
	279	/* But it is inconsistent unless marked otherwise. */
	280	mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
	281
	282	ENQUEUE_MUTEX (mutex);
	283	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	284
	285	/* Note that we deliberately exit here. If we fall
	286	through to the end of the function __nusers would be
	287	incremented which is not correct because the old owner
	288	has to be discounted. */
	289	return EOWNERDEAD;
	290	}
	291
	292	if (robust
	293	&& __builtin_expect (mutex->__data.__owner
	294	== PTHREAD_MUTEX_NOTRECOVERABLE, 0))
	295	{
	296	/* This mutex is now not recoverable. */
	297	mutex->__data.__count = 0;
	298
	299	INTERNAL_SYSCALL_DECL (__err);
	300	INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
efac1fce UD	301	__lll_private_flag (FUTEX_UNLOCK_PI,
	302	PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
	303	0, 0);
df47504c UD	304
	305	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	306	return ENOTRECOVERABLE;
	307	}
	308
	309	if (robust)
	310	{
	311	ENQUEUE_MUTEX_PI (mutex);
	312	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
	313	}
	314
	315	mutex->__data.__owner = id;
	316	++mutex->__data.__nusers;
	317	mutex->__data.__count = 1;
	318
	319	return 0;
	320	}
184ee940	321	#endif /* __NR_futex. */
df47504c	322
f17efcb4 UD	323	case PTHREAD_MUTEX_PP_RECURSIVE_NP:
	324	case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
	325	case PTHREAD_MUTEX_PP_NORMAL_NP:
	326	case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
	327	{
	328	int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
	329
	330	oldval = mutex->__data.__lock;
	331
	332	/* Check whether we already hold the mutex. */
	333	if (mutex->__data.__owner == id)
	334	{
	335	if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
	336	return EDEADLK;
	337
	338	if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
	339	{
	340	/* Just bump the counter. */
a1ffb40e	341	if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
f17efcb4 UD	342	/* Overflow of the counter. */
	343	return EAGAIN;
	344
	345	++mutex->__data.__count;
	346
	347	return 0;
	348	}
	349	}
	350
	351	int oldprio = -1, ceilval;
	352	do
	353	{
	354	int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
	355	>> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
	356
	357	if (__pthread_current_priority () > ceiling)
	358	{
	359	if (oldprio != -1)
	360	__pthread_tpp_change_priority (oldprio, -1);
	361	return EINVAL;
	362	}
	363
	364	int retval = __pthread_tpp_change_priority (oldprio, ceiling);
	365	if (retval)
	366	return retval;
	367
	368	ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
	369	oldprio = ceiling;
	370
	371	oldval
	372	= atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
	373	ceilval \| 1, ceilval);
	374
	375	if (oldval == ceilval)
	376	break;
	377	}
	378	while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
	379
	380	if (oldval != ceilval)
	381	{
	382	__pthread_tpp_change_priority (oldprio, -1);
	383	break;
	384	}
	385
	386	assert (mutex->__data.__owner == 0);
	387	/* Record the ownership. */
	388	mutex->__data.__owner = id;
	389	++mutex->__data.__nusers;
	390	mutex->__data.__count = 1;
	391
	392	return 0;
	393	}
	394	break;
	395
dcc73a8d UD	396	default:
	397	/* Correct code cannot set any other type. */
	398	return EINVAL;
76a50749 UD	399	}
	400
	401	return EBUSY;
	402	}
e8c659d7 AK	403
	404	#ifndef __pthread_mutex_trylock
	405	#ifndef pthread_mutex_trylock
76a50749	406	strong_alias (__pthread_mutex_trylock, pthread_mutex_trylock)
e8c659d7 AK	407	#endif
e8c659d7 AK	408	#endif