[thirdparty/glibc.git] / nptl / pthread_mutex_unlock.c

/* Copyright (C) 2002-2017 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <assert.h>
#include <errno.h>
#include <stdlib.h>
#include "pthreadP.h"
#include <lowlevellock.h>
#include <stap-probe.h>

#ifndef lll_unlock_elision
#define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
#endif

static int
__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
     __attribute_noinline__;

int
attribute_hidden
__pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
{
  int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  if (__builtin_expect (type &
		~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
    return __pthread_mutex_unlock_full (mutex, decr);

  if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
      == PTHREAD_MUTEX_TIMED_NP)
    {
      /* Always reset the owner field.  */
    normal:
      mutex->__data.__owner = 0;
      if (decr)
	/* One less user.  */
	--mutex->__data.__nusers;

      /* Unlock.  */
      lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));

      LIBC_PROBE (mutex_release, 1, mutex);

      return 0;
    }
  else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
    {
      /* Don't reset the owner/users fields for elision.  */
      return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
				      PTHREAD_MUTEX_PSHARED (mutex));
    }
  else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
			      == PTHREAD_MUTEX_RECURSIVE_NP, 1))
    {
      /* Recursive mutex.  */
      if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	return EPERM;

      if (--mutex->__data.__count != 0)
	/* We still hold the mutex.  */
	return 0;
      goto normal;
    }
  else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
			      == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
    goto normal;
  else
    {
      /* Error checking mutex.  */
      assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
      if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
	  || ! lll_islocked (mutex->__data.__lock))
	return EPERM;
      goto normal;
    }
}


static int
__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
{
  int newowner = 0;
  int private;

  switch (PTHREAD_MUTEX_TYPE (mutex))
    {
    case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
      /* Recursive mutex.  */
      if ((mutex->__data.__lock & FUTEX_TID_MASK)
	  == THREAD_GETMEM (THREAD_SELF, tid)
	  && __builtin_expect (mutex->__data.__owner
			       == PTHREAD_MUTEX_INCONSISTENT, 0))
	{
	  if (--mutex->__data.__count != 0)
	    /* We still hold the mutex.  */
	    return ENOTRECOVERABLE;

	  goto notrecoverable;
	}

      if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	return EPERM;

      if (--mutex->__data.__count != 0)
	/* We still hold the mutex.  */
	return 0;

      goto robust;

    case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
    case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
    case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
      if ((mutex->__data.__lock & FUTEX_TID_MASK)
	  != THREAD_GETMEM (THREAD_SELF, tid)
	  || ! lll_islocked (mutex->__data.__lock))
	return EPERM;

      /* If the previous owner died and the caller did not succeed in
	 making the state consistent, mark the mutex as unrecoverable
	 and make all waiters.  */
      if (__builtin_expect (mutex->__data.__owner
			    == PTHREAD_MUTEX_INCONSISTENT, 0))
      notrecoverable:
	newowner = PTHREAD_MUTEX_NOTRECOVERABLE;

    robust:
      /* Remove mutex from the list.  */
      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
		     &mutex->__data.__list.__next);
      /* We must set op_pending before we dequeue the mutex.  Also see
	 comments at ENQUEUE_MUTEX.  */
      __asm ("" ::: "memory");
      DEQUEUE_MUTEX (mutex);

      mutex->__data.__owner = newowner;
      if (decr)
	/* One less user.  */
	--mutex->__data.__nusers;

      /* Unlock by setting the lock to 0 (not acquired); if the lock had
	 FUTEX_WAITERS set previously, then wake any waiters.
         The unlock operation must be the last access to the mutex to not
         violate the mutex destruction requirements (see __lll_unlock).  */
      private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
      if (__glibc_unlikely ((atomic_exchange_rel (&mutex->__data.__lock, 0)
			     & FUTEX_WAITERS) != 0))
	lll_futex_wake (&mutex->__data.__lock, 1, private);

      /* We must clear op_pending after we release the mutex.
	 FIXME However, this violates the mutex destruction requirements
	 because another thread could acquire the mutex, destroy it, and
	 reuse the memory for something else; then, if this thread crashes,
	 and the memory happens to have a value equal to the TID, the kernel
	 will believe it is still related to the mutex (which has been
	 destroyed already) and will modify some other random object.  */
      __asm ("" ::: "memory");
      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
      break;

    /* The PI support requires the Linux futex system call.  If that's not
       available, pthread_mutex_init should never have allowed the type to
       be set.  So it will get the default case for an invalid type.  */
#ifdef __NR_futex
    case PTHREAD_MUTEX_PI_RECURSIVE_NP:
      /* Recursive mutex.  */
      if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	return EPERM;

      if (--mutex->__data.__count != 0)
	/* We still hold the mutex.  */
	return 0;
      goto continue_pi_non_robust;

    case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
      /* Recursive mutex.  */
      if ((mutex->__data.__lock & FUTEX_TID_MASK)
	  == THREAD_GETMEM (THREAD_SELF, tid)
	  && __builtin_expect (mutex->__data.__owner
			       == PTHREAD_MUTEX_INCONSISTENT, 0))
	{
	  if (--mutex->__data.__count != 0)
	    /* We still hold the mutex.  */
	    return ENOTRECOVERABLE;

	  goto pi_notrecoverable;
	}

      if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	return EPERM;

      if (--mutex->__data.__count != 0)
	/* We still hold the mutex.  */
	return 0;

      goto continue_pi_robust;

    case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
    case PTHREAD_MUTEX_PI_NORMAL_NP:
    case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
    case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
    case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
    case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
      if ((mutex->__data.__lock & FUTEX_TID_MASK)
	  != THREAD_GETMEM (THREAD_SELF, tid)
	  || ! lll_islocked (mutex->__data.__lock))
	return EPERM;

      /* If the previous owner died and the caller did not succeed in
	 making the state consistent, mark the mutex as unrecoverable
	 and make all waiters.  */
      if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
	  && __builtin_expect (mutex->__data.__owner
			       == PTHREAD_MUTEX_INCONSISTENT, 0))
      pi_notrecoverable:
       newowner = PTHREAD_MUTEX_NOTRECOVERABLE;

      if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
	{
	continue_pi_robust:
	  /* Remove mutex from the list.
	     Note: robust PI futexes are signaled by setting bit 0.  */
	  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
			 (void *) (((uintptr_t) &mutex->__data.__list.__next)
				   | 1));
	  /* We must set op_pending before we dequeue the mutex.  Also see
	     comments at ENQUEUE_MUTEX.  */
	  __asm ("" ::: "memory");
	  DEQUEUE_MUTEX (mutex);
	}

    continue_pi_non_robust:
      mutex->__data.__owner = newowner;
      if (decr)
	/* One less user.  */
	--mutex->__data.__nusers;

      /* Unlock.  Load all necessary mutex data before releasing the mutex
	 to not violate the mutex destruction requirements (see
	 lll_unlock).  */
      int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
      private = (robust
		 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
		 : PTHREAD_MUTEX_PSHARED (mutex));
      /* Unlock the mutex using a CAS unless there are futex waiters or our
	 TID is not the value of __lock anymore, in which case we let the
	 kernel take care of the situation.  Use release MO in the CAS to
	 synchronize with acquire MO in lock acquisitions.  */
      int l = atomic_load_relaxed (&mutex->__data.__lock);
      do
	{
	  if (((l & FUTEX_WAITERS) != 0)
	      || (l != THREAD_GETMEM (THREAD_SELF, tid)))
	    {
	      INTERNAL_SYSCALL_DECL (__err);
	      INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
				__lll_private_flag (FUTEX_UNLOCK_PI, private));
	      break;
	    }
	}
      while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
						    &l, 0));

      /* This happens after the kernel releases the mutex but violates the
	 mutex destruction requirements; see comments in the code handling
	 PTHREAD_MUTEX_ROBUST_NORMAL_NP.  */
      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
      break;
#endif  /* __NR_futex.  */

    case PTHREAD_MUTEX_PP_RECURSIVE_NP:
      /* Recursive mutex.  */
      if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	return EPERM;

      if (--mutex->__data.__count != 0)
	/* We still hold the mutex.  */
	return 0;
      goto pp;

    case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
      /* Error checking mutex.  */
      if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
	  || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
	return EPERM;
      /* FALLTHROUGH */

    case PTHREAD_MUTEX_PP_NORMAL_NP:
    case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
      /* Always reset the owner field.  */
    pp:
      mutex->__data.__owner = 0;

      if (decr)
	/* One less user.  */
	--mutex->__data.__nusers;

      /* Unlock.  Use release MO in the CAS to synchronize with acquire MO in
	 lock acquisitions.  */
      int newval;
      int oldval = atomic_load_relaxed (&mutex->__data.__lock);
      do
	{
	  newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
	}
      while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
						    &oldval, newval));

      if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
	lll_futex_wake (&mutex->__data.__lock, 1,
			PTHREAD_MUTEX_PSHARED (mutex));

      int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;

      LIBC_PROBE (mutex_release, 1, mutex);

      return __pthread_tpp_change_priority (oldprio, -1);

    default:
      /* Correct code cannot set any other type.  */
      return EINVAL;
    }

  LIBC_PROBE (mutex_release, 1, mutex);
  return 0;
}


int
__pthread_mutex_unlock (pthread_mutex_t *mutex)
{
  return __pthread_mutex_unlock_usercnt (mutex, 1);
}
weak_alias (__pthread_mutex_unlock, pthread_mutex_unlock)
hidden_def (__pthread_mutex_unlock)
Commit	Line	Data
bfff8b1b	1	/* Copyright (C) 2002-2017 Free Software Foundation, Inc.
76a50749 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
76a50749	18
6de79a49	19	#include <assert.h>
76a50749	20	#include <errno.h>
1bcfb5a5	21	#include <stdlib.h>
76a50749 UD	22	#include "pthreadP.h"
76a50749 UD	23	#include <lowlevellock.h>
5acf7263	24	#include <stap-probe.h>
76a50749	25
e8c659d7	26	#ifndef lll_unlock_elision
e5e6bea2	27	#define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
e8c659d7 AK	28	#endif
e8c659d7 AK	29
6de79a49	30	static int
6de79a49 UD	31	__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
6de79a49 UD	32	__attribute_noinline__;
76a50749 UD	33
76a50749 UD	34	int
83b09837	35	attribute_hidden
9dd346ff	36	__pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
76a50749	37	{
e8c659d7 AK	38	int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
	39	if (__builtin_expect (type &
	40	~(PTHREAD_MUTEX_KIND_MASK_NP\|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
6de79a49	41	return __pthread_mutex_unlock_full (mutex, decr);
1bcfb5a5	42
6de79a49 UD	43	if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
	44	== PTHREAD_MUTEX_TIMED_NP)
	45	{
	46	/* Always reset the owner field. */
	47	normal:
	48	mutex->__data.__owner = 0;
	49	if (decr)
	50	/* One less user. */
	51	--mutex->__data.__nusers;
	52
	53	/* Unlock. */
	54	lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
5acf7263 RM	55
	56	LIBC_PROBE (mutex_release, 1, mutex);
	57
6de79a49 UD	58	return 0;
6de79a49 UD	59	}
a1ffb40e	60	else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
e8c659d7 AK	61	{
e8c659d7 AK	62	/* Don't reset the owner/users fields for elision. */
e5e6bea2	63	return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
e8c659d7 AK	64	PTHREAD_MUTEX_PSHARED (mutex));
	65	}
	66	else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
	67	== PTHREAD_MUTEX_RECURSIVE_NP, 1))
76a50749	68	{
76a50749	69	/* Recursive mutex. */
61623643	70	if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
76a50749 UD	71	return EPERM;
	72
	73	if (--mutex->__data.__count != 0)
	74	/* We still hold the mutex. */
	75	return 0;
683040c3	76	goto normal;
6de79a49	77	}
e8c659d7 AK	78	else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
e8c659d7 AK	79	== PTHREAD_MUTEX_ADAPTIVE_NP, 1))
6de79a49 UD	80	goto normal;
	81	else
	82	{
76a50749	83	/* Error checking mutex. */
6de79a49	84	assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
61623643	85	if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
e51deae7	86	\|\| ! lll_islocked (mutex->__data.__lock))
76a50749	87	return EPERM;
6de79a49 UD	88	goto normal;
	89	}
	90	}
76a50749	91
683040c3	92
6de79a49	93	static int
6de79a49 UD	94	__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
	95	{
	96	int newowner = 0;
65810f0e	97	int private;
dcc73a8d	98
6de79a49 UD	99	switch (PTHREAD_MUTEX_TYPE (mutex))
6de79a49 UD	100	{
0f6699ea	101	case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
1bcfb5a5	102	/* Recursive mutex. */
683040c3	103	if ((mutex->__data.__lock & FUTEX_TID_MASK)
0f6699ea UD	104	== THREAD_GETMEM (THREAD_SELF, tid)
	105	&& __builtin_expect (mutex->__data.__owner
	106	== PTHREAD_MUTEX_INCONSISTENT, 0))
1bcfb5a5 UD	107	{
	108	if (--mutex->__data.__count != 0)
	109	/* We still hold the mutex. */
	110	return ENOTRECOVERABLE;
	111
	112	goto notrecoverable;
	113	}
	114
	115	if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	116	return EPERM;
	117
	118	if (--mutex->__data.__count != 0)
	119	/* We still hold the mutex. */
	120	return 0;
	121
	122	goto robust;
	123
0f6699ea UD	124	case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
	125	case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
	126	case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
683040c3 UD	127	if ((mutex->__data.__lock & FUTEX_TID_MASK)
683040c3 UD	128	!= THREAD_GETMEM (THREAD_SELF, tid)
e51deae7	129	\|\| ! lll_islocked (mutex->__data.__lock))
1bcfb5a5 UD	130	return EPERM;
1bcfb5a5 UD	131
1bcfb5a5 UD	132	/* If the previous owner died and the caller did not succeed in
	133	making the state consistent, mark the mutex as unrecoverable
	134	and make all waiters. */
	135	if (__builtin_expect (mutex->__data.__owner
683040c3	136	== PTHREAD_MUTEX_INCONSISTENT, 0))
1bcfb5a5 UD	137	notrecoverable:
	138	newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
	139
	140	robust:
	141	/* Remove mutex from the list. */
0f6699ea UD	142	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
0f6699ea UD	143	&mutex->__data.__list.__next);
8f9450a0 TR	144	/* We must set op_pending before we dequeue the mutex. Also see
	145	comments at ENQUEUE_MUTEX. */
	146	__asm ("" ::: "memory");
1bcfb5a5	147	DEQUEUE_MUTEX (mutex);
683040c3 UD	148
	149	mutex->__data.__owner = newowner;
	150	if (decr)
	151	/* One less user. */
	152	--mutex->__data.__nusers;
	153
65810f0e TR	154	/* Unlock by setting the lock to 0 (not acquired); if the lock had
	155	FUTEX_WAITERS set previously, then wake any waiters.
	156	The unlock operation must be the last access to the mutex to not
	157	violate the mutex destruction requirements (see __lll_unlock). */
	158	private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
	159	if (__glibc_unlikely ((atomic_exchange_rel (&mutex->__data.__lock, 0)
	160	& FUTEX_WAITERS) != 0))
	161	lll_futex_wake (&mutex->__data.__lock, 1, private);
0f6699ea	162
8f9450a0 TR	163	/* We must clear op_pending after we release the mutex.
	164	FIXME However, this violates the mutex destruction requirements
	165	because another thread could acquire the mutex, destroy it, and
	166	reuse the memory for something else; then, if this thread crashes,
	167	and the memory happens to have a value equal to the TID, the kernel
	168	will believe it is still related to the mutex (which has been
	169	destroyed already) and will modify some other random object. */
	170	__asm ("" ::: "memory");
0f6699ea	171	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
1bcfb5a5 UD	172	break;
1bcfb5a5 UD	173
184ee940 RM	174	/* The PI support requires the Linux futex system call. If that's not
	175	available, pthread_mutex_init should never have allowed the type to
	176	be set. So it will get the default case for an invalid type. */
	177	#ifdef __NR_futex
df47504c UD	178	case PTHREAD_MUTEX_PI_RECURSIVE_NP:
	179	/* Recursive mutex. */
	180	if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	181	return EPERM;
	182
	183	if (--mutex->__data.__count != 0)
	184	/* We still hold the mutex. */
	185	return 0;
b0948ffd	186	goto continue_pi_non_robust;
df47504c UD	187
	188	case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
	189	/* Recursive mutex. */
	190	if ((mutex->__data.__lock & FUTEX_TID_MASK)
	191	== THREAD_GETMEM (THREAD_SELF, tid)
	192	&& __builtin_expect (mutex->__data.__owner
	193	== PTHREAD_MUTEX_INCONSISTENT, 0))
	194	{
	195	if (--mutex->__data.__count != 0)
	196	/* We still hold the mutex. */
	197	return ENOTRECOVERABLE;
	198
	199	goto pi_notrecoverable;
	200	}
	201
	202	if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	203	return EPERM;
	204
	205	if (--mutex->__data.__count != 0)
	206	/* We still hold the mutex. */
	207	return 0;
	208
b0948ffd	209	goto continue_pi_robust;
df47504c UD	210
	211	case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
	212	case PTHREAD_MUTEX_PI_NORMAL_NP:
	213	case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
	214	case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
	215	case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
	216	case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
	217	if ((mutex->__data.__lock & FUTEX_TID_MASK)
	218	!= THREAD_GETMEM (THREAD_SELF, tid)
e51deae7	219	\|\| ! lll_islocked (mutex->__data.__lock))
df47504c UD	220	return EPERM;
	221
	222	/* If the previous owner died and the caller did not succeed in
	223	making the state consistent, mark the mutex as unrecoverable
	224	and make all waiters. */
	225	if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
	226	&& __builtin_expect (mutex->__data.__owner
	227	== PTHREAD_MUTEX_INCONSISTENT, 0))
	228	pi_notrecoverable:
	229	newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
	230
df47504c UD	231	if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
df47504c UD	232	{
b0948ffd	233	continue_pi_robust:
df47504c UD	234	/* Remove mutex from the list.
	235	Note: robust PI futexes are signaled by setting bit 0. */
	236	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
	237	(void *) (((uintptr_t) &mutex->__data.__list.__next)
	238	\| 1));
8f9450a0 TR	239	/* We must set op_pending before we dequeue the mutex. Also see
	240	comments at ENQUEUE_MUTEX. */
	241	__asm ("" ::: "memory");
df47504c UD	242	DEQUEUE_MUTEX (mutex);
	243	}
	244
b0948ffd	245	continue_pi_non_robust:
df47504c UD	246	mutex->__data.__owner = newowner;
	247	if (decr)
	248	/* One less user. */
	249	--mutex->__data.__nusers;
	250
389fdf78 TR	251	/* Unlock. Load all necessary mutex data before releasing the mutex
	252	to not violate the mutex destruction requirements (see
	253	lll_unlock). */
	254	int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
65810f0e TR	255	private = (robust
	256	? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
	257	: PTHREAD_MUTEX_PSHARED (mutex));
76a0b73e TR	258	/* Unlock the mutex using a CAS unless there are futex waiters or our
	259	TID is not the value of __lock anymore, in which case we let the
	260	kernel take care of the situation. Use release MO in the CAS to
	261	synchronize with acquire MO in lock acquisitions. */
	262	int l = atomic_load_relaxed (&mutex->__data.__lock);
	263	do
df47504c	264	{
76a0b73e TR	265	if (((l & FUTEX_WAITERS) != 0)
	266	\|\| (l != THREAD_GETMEM (THREAD_SELF, tid)))
	267	{
	268	INTERNAL_SYSCALL_DECL (__err);
	269	INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
	270	__lll_private_flag (FUTEX_UNLOCK_PI, private));
	271	break;
	272	}
df47504c	273	}
76a0b73e TR	274	while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
76a0b73e TR	275	&l, 0));
df47504c	276
8f9450a0 TR	277	/* This happens after the kernel releases the mutex but violates the
	278	mutex destruction requirements; see comments in the code handling
	279	PTHREAD_MUTEX_ROBUST_NORMAL_NP. */
df47504c UD	280	THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
df47504c UD	281	break;
184ee940	282	#endif /* __NR_futex. */
df47504c	283
f17efcb4 UD	284	case PTHREAD_MUTEX_PP_RECURSIVE_NP:
	285	/* Recursive mutex. */
	286	if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
	287	return EPERM;
	288
	289	if (--mutex->__data.__count != 0)
	290	/* We still hold the mutex. */
	291	return 0;
	292	goto pp;
	293
	294	case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
	295	/* Error checking mutex. */
	296	if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
	297	\|\| (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
	298	return EPERM;
	299	/* FALLTHROUGH */
	300
	301	case PTHREAD_MUTEX_PP_NORMAL_NP:
	302	case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
	303	/* Always reset the owner field. */
	304	pp:
	305	mutex->__data.__owner = 0;
	306
	307	if (decr)
	308	/* One less user. */
	309	--mutex->__data.__nusers;
	310
76a0b73e TR	311	/* Unlock. Use release MO in the CAS to synchronize with acquire MO in
	312	lock acquisitions. */
	313	int newval;
	314	int oldval = atomic_load_relaxed (&mutex->__data.__lock);
f17efcb4 UD	315	do
f17efcb4 UD	316	{
f17efcb4 UD	317	newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
f17efcb4 UD	318	}
76a0b73e TR	319	while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
76a0b73e TR	320	&oldval, newval));
f17efcb4 UD	321
f17efcb4 UD	322	if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
835abc5c	323	lll_futex_wake (&mutex->__data.__lock, 1,
5bd8a249	324	PTHREAD_MUTEX_PSHARED (mutex));
f17efcb4 UD	325
f17efcb4 UD	326	int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
5acf7263 RM	327
	328	LIBC_PROBE (mutex_release, 1, mutex);
	329
f17efcb4 UD	330	return __pthread_tpp_change_priority (oldprio, -1);
f17efcb4 UD	331
dcc73a8d UD	332	default:
	333	/* Correct code cannot set any other type. */
	334	return EINVAL;
a334319f	335	}
0ecb606c	336
5acf7263	337	LIBC_PROBE (mutex_release, 1, mutex);
76a50749 UD	338	return 0;
76a50749 UD	339	}
61623643 UD	340
	341
	342	int
9dd346ff	343	__pthread_mutex_unlock (pthread_mutex_t *mutex)
61623643	344	{
8f31c0ef	345	return __pthread_mutex_unlock_usercnt (mutex, 1);
61623643	346	}
fa872e1b	347	weak_alias (__pthread_mutex_unlock, pthread_mutex_unlock)
4d17e683	348	hidden_def (__pthread_mutex_unlock)