[thirdparty/glibc.git] / sysdeps / unix / sysv / linux / s390 / lowlevellock.h

/* Copyright (C) 2003-2015 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#ifndef _LOWLEVELLOCK_H
#define _LOWLEVELLOCK_H	1

#include <time.h>
#include <sys/param.h>
#include <bits/pthreadtypes.h>
#include <atomic.h>
#include <kernel-features.h>

#define SYS_futex		238
#define FUTEX_WAIT		0
#define FUTEX_WAKE		1
#define FUTEX_REQUEUE		3
#define FUTEX_CMP_REQUEUE	4
#define FUTEX_WAKE_OP		5
#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE	((4 << 24) | 1)
#define FUTEX_LOCK_PI		6
#define FUTEX_UNLOCK_PI		7
#define FUTEX_TRYLOCK_PI	8
#define FUTEX_WAIT_BITSET	9
#define FUTEX_WAKE_BITSET	10
#define FUTEX_WAIT_REQUEUE_PI   11
#define FUTEX_CMP_REQUEUE_PI    12
#define FUTEX_PRIVATE_FLAG	128
#define FUTEX_CLOCK_REALTIME	256

#define FUTEX_BITSET_MATCH_ANY	0xffffffff

/* Values for 'private' parameter of locking macros.  Yes, the
   definition seems to be backwards.  But it is not.  The bit will be
   reversed before passing to the system call.  */
#define LLL_PRIVATE	0
#define LLL_SHARED	FUTEX_PRIVATE_FLAG


#if IS_IN (libc) || IS_IN (rtld)
/* In libc.so or ld.so all futexes are private.  */
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  ((fl) | FUTEX_PRIVATE_FLAG)
# else
#  define __lll_private_flag(fl, private) \
  ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
# endif
#else
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
# else
#  define __lll_private_flag(fl, private) \
  (__builtin_constant_p (private)					      \
   ? ((private) == 0							      \
      ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))	      \
      : (fl))								      \
   : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG)				      \
	      & THREAD_GETMEM (THREAD_SELF, header.private_futex))))
# endif
#endif

#define lll_futex_wait(futex, val, private) \
  lll_futex_timed_wait (futex, val, NULL, private)

#define lll_futex_timed_wait(futexp, val, timespec, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
									      \
    INTERNAL_SYSCALL (futex, __err, 4, (futexp),		      \
			      __lll_private_flag (FUTEX_WAIT, private),	      \
			      (val), (timespec));			      \
  })

#define lll_futex_timed_wait_bitset(futexp, val, timespec, clockbit, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    int __op = FUTEX_WAIT_BITSET | clockbit;				      \
									      \
    INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
			      __lll_private_flag (__op, private),	      \
			      (val), (timespec), NULL /* Unused.  */, 	      \
			      FUTEX_BITSET_MATCH_ANY);			      \
  })

#define lll_futex_wake(futexp, nr, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
									      \
    INTERNAL_SYSCALL (futex, __err, 4, (futexp),		      \
			      __lll_private_flag (FUTEX_WAKE, private),	      \
			      (nr), 0);					      \
  })


/* Returns non-zero if error happened, zero if success.  */
#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    long int __ret;							      \
									      \
    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
			      __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
			      (nr_wake), (nr_move), (mutex), (val));	      \
    INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
  })

/* Returns non-zero if error happened, zero if success.  */
#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    long int __ret;							      \
									      \
    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
			      __lll_private_flag (FUTEX_WAKE_OP, private),    \
			      (nr_wake), (nr_wake2), (futexp2),		      \
			      FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);		      \
    INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
  })

/* Priority Inheritance support.  */
#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
  lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)

#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit,      \
					mutex, private)			      \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    int __op = FUTEX_WAIT_REQUEUE_PI | clockbit;			      \
									      \
    INTERNAL_SYSCALL (futex, __err, 5, (futexp),			      \
		      __lll_private_flag (__op, private),		      \
		      (val), (timespec), mutex); 			      \
  })

#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv)  \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    long int __ret;							      \
									      \
    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
			      __lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
			      (nr_wake), (nr_move), (mutex), (val));	      \
    INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
  })

#define lll_compare_and_swap(futex, oldval, newval, operation) \
  do {									      \
    __typeof (futex) __futex = (futex);					      \
    __asm __volatile ("	  l   %1,%0\n"					      \
		      "0: " operation "\n"				      \
		      "	  cs  %1,%2,%0\n"				      \
		      "	  jl  0b\n"					      \
		      "1:"						      \
		      : "=Q" (*__futex), "=&d" (oldval), "=&d" (newval)	      \
		      : "m" (*__futex) : "cc", "memory" );		      \
  } while (0)


static inline int
__attribute__ ((always_inline))
__lll_trylock (int *futex)
{
    unsigned int old;

    __asm __volatile ("cs %0,%3,%1"
		       : "=d" (old), "=Q" (*futex)
		       : "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
    return old != 0;
}
#define lll_trylock(futex) __lll_trylock (&(futex))


static inline int
__attribute__ ((always_inline))
__lll_cond_trylock (int *futex)
{
    unsigned int old;

    __asm __volatile ("cs %0,%3,%1"
		       : "=d" (old), "=Q" (*futex)
		       : "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
    return old != 0;
}
#define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))


extern void __lll_lock_wait_private (int *futex) attribute_hidden;
extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;

static inline void
__attribute__ ((always_inline))
__lll_lock (int *futex, int private)
{
  if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 1, 0)))
    {
      if (__builtin_constant_p (private) && private == LLL_PRIVATE)
	__lll_lock_wait_private (futex);
      else
	__lll_lock_wait (futex, private);
    }
}
#define lll_lock(futex, private) __lll_lock (&(futex), private)

static inline int
__attribute__ ((always_inline))
__lll_robust_lock (int *futex, int id, int private)
{
  int result = 0;
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
			0))
    result = __lll_robust_lock_wait (futex, private);
  return result;
}
#define lll_robust_lock(futex, id, private) \
  __lll_robust_lock (&(futex), id, private)

static inline void
__attribute__ ((always_inline))
__lll_cond_lock (int *futex, int private)
{
  if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 2, 0)))
    __lll_lock_wait (futex, private);
}
#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)

#define lll_robust_cond_lock(futex, id, private) \
  __lll_robust_lock (&(futex), (id) | FUTEX_WAITERS, private)

extern int __lll_timedlock_wait
  (int *futex, const struct timespec *, int private) attribute_hidden;
extern int __lll_robust_timedlock_wait
  (int *futex, const struct timespec *, int private) attribute_hidden;

static inline int
__attribute__ ((always_inline))
__lll_timedlock (int *futex, const struct timespec *abstime, int private)
{
  int result = 0;
  if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 1, 0)))
    result = __lll_timedlock_wait (futex, abstime, private);
  return result;
}
#define lll_timedlock(futex, abstime, private) \
  __lll_timedlock (&(futex), abstime, private)

#ifdef ENABLE_LOCK_ELISION
extern int __lll_timedlock_elision
  (int *futex, short *adapt_count, const struct timespec *timeout, int private)
  attribute_hidden;

# define lll_timedlock_elision(futex, adapt_count, timeout, private)	\
  __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
#endif

static inline int
__attribute__ ((always_inline))
__lll_robust_timedlock (int *futex, const struct timespec *abstime,
			int id, int private)
{
  int result = 0;
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
			0))
    result = __lll_robust_timedlock_wait (futex, abstime, private);
  return result;
}
#define lll_robust_timedlock(futex, abstime, id, private) \
  __lll_robust_timedlock (&(futex), abstime, id, private)


#define __lll_unlock(futex, private) \
  (void)								      \
    ({ int __oldval;							      \
       int __newval = 0;						      \
       int *__futexp = (futex);						      \
									      \
       lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
       if (__glibc_unlikely (__oldval > 1))				      \
	 lll_futex_wake (__futexp, 1, private);				      \
    })
#define lll_unlock(futex, private) __lll_unlock(&(futex), private)


#define __lll_robust_unlock(futex, private) \
  (void)								      \
    ({ int __oldval;							      \
       int __newval = 0;						      \
       int *__futexp = (futex);						      \
									      \
       lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
       if (__glibc_unlikely (__oldval & FUTEX_WAITERS))			      \
	 lll_futex_wake (__futexp, 1, private);				      \
    })
#define lll_robust_unlock(futex, private) \
  __lll_robust_unlock(&(futex), private)

#define lll_islocked(futex) \
  (futex != 0)


/* Initializers for lock.  */
#define LLL_LOCK_INITIALIZER		(0)
#define LLL_LOCK_INITIALIZER_LOCKED	(1)

/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
   wakeup when the clone terminates.  The memory location contains the
   thread ID while the clone is running and is reset to zero
   afterwards.	*/
#define __lll_wait_tid(ptid) \
  do									      \
    {									      \
      int __tid;							      \
									      \
      while ((__tid = *ptid) != 0)					      \
	lll_futex_wait (ptid, __tid, LLL_SHARED);			      \
    }									      \
  while (0)
#define lll_wait_tid(tid) __lll_wait_tid(&(tid))

extern int __lll_timedwait_tid (int *, const struct timespec *)
     attribute_hidden;

#define lll_timedwait_tid(tid, abstime) \
  ({									      \
    int __res = 0;							      \
    if ((tid) != 0)							      \
      __res = __lll_timedwait_tid (&(tid), (abstime));			      \
    __res;								      \
  })

#ifdef ENABLE_LOCK_ELISION
extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
  attribute_hidden;

extern int __lll_unlock_elision(int *futex, int private)
  attribute_hidden;

extern int __lll_trylock_elision(int *futex, short *adapt_count)
  attribute_hidden;

# define lll_lock_elision(futex, adapt_count, private) \
  __lll_lock_elision (&(futex), &(adapt_count), private)
# define lll_unlock_elision(futex, private) \
  __lll_unlock_elision (&(futex), private)
# define lll_trylock_elision(futex, adapt_count) \
  __lll_trylock_elision(&(futex), &(adapt_count))
#endif

#endif	/* lowlevellock.h */
Commit	Line	Data
b168057a	1	/* Copyright (C) 2003-2015 Free Software Foundation, Inc.
a88c9263 UD	2	This file is part of the GNU C Library.
	3	Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
a88c9263 UD	18
	19	#ifndef _LOWLEVELLOCK_H
	20	#define _LOWLEVELLOCK_H 1
	21
	22	#include <time.h>
	23	#include <sys/param.h>
	24	#include <bits/pthreadtypes.h>
39358e8b	25	#include <atomic.h>
f8de5057	26	#include <kernel-features.h>
a88c9263 UD	27
	28	#define SYS_futex 238
	29	#define FUTEX_WAIT 0
	30	#define FUTEX_WAKE 1
284bdc42	31	#define FUTEX_REQUEUE 3
75fccede	32	#define FUTEX_CMP_REQUEUE 4
b9b8cf03 UD	33	#define FUTEX_WAKE_OP 5
b9b8cf03 UD	34	#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) \| 1)
d5ba53f9 UD	35	#define FUTEX_LOCK_PI 6
	36	#define FUTEX_UNLOCK_PI 7
	37	#define FUTEX_TRYLOCK_PI 8
da5ac135 UD	38	#define FUTEX_WAIT_BITSET 9
da5ac135 UD	39	#define FUTEX_WAKE_BITSET 10
8313cb99 SP	40	#define FUTEX_WAIT_REQUEUE_PI 11
8313cb99 SP	41	#define FUTEX_CMP_REQUEUE_PI 12
085a4412	42	#define FUTEX_PRIVATE_FLAG 128
7dd650d7 UD	43	#define FUTEX_CLOCK_REALTIME 256
	44
	45	#define FUTEX_BITSET_MATCH_ANY 0xffffffff
085a4412 UD	46
	47	/* Values for 'private' parameter of locking macros. Yes, the
	48	definition seems to be backwards. But it is not. The bit will be
	49	reversed before passing to the system call. */
	50	#define LLL_PRIVATE 0
	51	#define LLL_SHARED FUTEX_PRIVATE_FLAG
	52
	53
4f41c682	54	#if IS_IN (libc) \|\| IS_IN (rtld)
085a4412 UD	55	/* In libc.so or ld.so all futexes are private. */
	56	# ifdef __ASSUME_PRIVATE_FUTEX
	57	# define __lll_private_flag(fl, private) \
	58	((fl) \| FUTEX_PRIVATE_FLAG)
	59	# else
	60	# define __lll_private_flag(fl, private) \
	61	((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex))
	62	# endif
	63	#else
	64	# ifdef __ASSUME_PRIVATE_FUTEX
	65	# define __lll_private_flag(fl, private) \
	66	(((fl) \| FUTEX_PRIVATE_FLAG) ^ (private))
	67	# else
	68	# define __lll_private_flag(fl, private) \
	69	(__builtin_constant_p (private) \
	70	? ((private) == 0 \
	71	? ((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex)) \
	72	: (fl)) \
	73	: ((fl) \| (((private) ^ FUTEX_PRIVATE_FLAG) \
	74	& THREAD_GETMEM (THREAD_SELF, header.private_futex))))
da5ac135	75	# endif
085a4412	76	#endif
a88c9263	77
085a4412 UD	78	#define lll_futex_wait(futex, val, private) \
085a4412 UD	79	lll_futex_timed_wait (futex, val, NULL, private)
a88c9263	80
8ebac778	81	#define lll_futex_timed_wait(futexp, val, timespec, private) \
a88c9263	82	({ \
8ebac778	83	INTERNAL_SYSCALL_DECL (__err); \
a88c9263	84	\
8ebac778 SP	85	INTERNAL_SYSCALL (futex, __err, 4, (futexp), \
	86	__lll_private_flag (FUTEX_WAIT, private), \
	87	(val), (timespec)); \
a88c9263 UD	88	})
a88c9263 UD	89
8f861542 SP	90	#define lll_futex_timed_wait_bitset(futexp, val, timespec, clockbit, private) \
8f861542 SP	91	({ \
8ebac778 SP	92	INTERNAL_SYSCALL_DECL (__err); \
8ebac778 SP	93	int __op = FUTEX_WAIT_BITSET \| clockbit; \
8f861542	94	\
8ebac778 SP	95	INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
	96	__lll_private_flag (__op, private), \
	97	(val), (timespec), NULL /* Unused. */, \
	98	FUTEX_BITSET_MATCH_ANY); \
8f861542 SP	99	})
8f861542 SP	100
8ebac778	101	#define lll_futex_wake(futexp, nr, private) \
a88c9263	102	({ \
8ebac778	103	INTERNAL_SYSCALL_DECL (__err); \
a88c9263	104	\
8ebac778 SP	105	INTERNAL_SYSCALL (futex, __err, 4, (futexp), \
	106	__lll_private_flag (FUTEX_WAKE, private), \
	107	(nr), 0); \
a88c9263 UD	108	})
a88c9263 UD	109
f1740bc4	110
75fccede	111	/* Returns non-zero if error happened, zero if success. */
8ebac778	112	#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
284bdc42	113	({ \
8ebac778 SP	114	INTERNAL_SYSCALL_DECL (__err); \
8ebac778 SP	115	long int __ret; \
b9b8cf03	116	\
8ebac778 SP	117	__ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
	118	__lll_private_flag (FUTEX_CMP_REQUEUE, private),\
	119	(nr_wake), (nr_move), (mutex), (val)); \
	120	INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
b9b8cf03 UD	121	})
b9b8cf03 UD	122
b9b8cf03	123	/* Returns non-zero if error happened, zero if success. */
8ebac778	124	#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
b9b8cf03	125	({ \
8ebac778 SP	126	INTERNAL_SYSCALL_DECL (__err); \
8ebac778 SP	127	long int __ret; \
284bdc42	128	\
8ebac778 SP	129	__ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
	130	__lll_private_flag (FUTEX_WAKE_OP, private), \
	131	(nr_wake), (nr_wake2), (futexp2), \
	132	FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \
	133	INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
284bdc42 UD	134	})
284bdc42 UD	135
8313cb99 SP	136	/* Priority Inheritance support. */
	137	#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
	138	lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
	139
	140	#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit, \
	141	mutex, private) \
	142	({ \
	143	INTERNAL_SYSCALL_DECL (__err); \
	144	int __op = FUTEX_WAIT_REQUEUE_PI \| clockbit; \
	145	\
	146	INTERNAL_SYSCALL (futex, __err, 5, (futexp), \
	147	__lll_private_flag (__op, private), \
	148	(val), (timespec), mutex); \
	149	})
	150
	151	#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv) \
	152	({ \
	153	INTERNAL_SYSCALL_DECL (__err); \
	154	long int __ret; \
	155	\
	156	__ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
	157	__lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
	158	(nr_wake), (nr_move), (mutex), (val)); \
	159	INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
	160	})
	161
a88c9263 UD	162	#define lll_compare_and_swap(futex, oldval, newval, operation) \
	163	do { \
	164	__typeof (futex) __futex = (futex); \
	165	__asm __volatile (" l %1,%0\n" \
	166	"0: " operation "\n" \
	167	" cs %1,%2,%0\n" \
	168	" jl 0b\n" \
	169	"1:" \
	170	: "=Q" (*__futex), "=&d" (oldval), "=&d" (newval) \
f1847a84	171	: "m" (*__futex) : "cc", "memory" ); \
a88c9263 UD	172	} while (0)
	173
	174
	175	static inline int
	176	__attribute__ ((always_inline))
e51deae7	177	__lll_trylock (int *futex)
a88c9263 UD	178	{
	179	unsigned int old;
	180
	181	__asm __volatile ("cs %0,%3,%1"
	182	: "=d" (old), "=Q" (*futex)
f1847a84	183	: "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
a88c9263 UD	184	return old != 0;
a88c9263 UD	185	}
e51deae7	186	#define lll_trylock(futex) __lll_trylock (&(futex))
a88c9263 UD	187
a88c9263 UD	188
2c0b891a UD	189	static inline int
2c0b891a UD	190	__attribute__ ((always_inline))
e51deae7	191	__lll_cond_trylock (int *futex)
2c0b891a UD	192	{
	193	unsigned int old;
	194
	195	__asm __volatile ("cs %0,%3,%1"
	196	: "=d" (old), "=Q" (*futex)
f1847a84	197	: "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
2c0b891a UD	198	return old != 0;
2c0b891a UD	199	}
e51deae7	200	#define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))
2c0b891a UD	201
2c0b891a UD	202
e51deae7 UD	203	extern void __lll_lock_wait_private (int *futex) attribute_hidden;
	204	extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
	205	extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;
a88c9263 UD	206
	207	static inline void
	208	__attribute__ ((always_inline))
e51deae7	209	__lll_lock (int *futex, int private)
a88c9263	210	{
a1ffb40e	211	if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 1, 0)))
e51deae7 UD	212	{
	213	if (__builtin_constant_p (private) && private == LLL_PRIVATE)
	214	__lll_lock_wait_private (futex);
	215	else
	216	__lll_lock_wait (futex, private);
	217	}
a88c9263	218	}
e51deae7	219	#define lll_lock(futex, private) __lll_lock (&(futex), private)
a88c9263	220
f1740bc4 UD	221	static inline int
f1740bc4 UD	222	__attribute__ ((always_inline))
e51deae7	223	__lll_robust_lock (int *futex, int id, int private)
f1740bc4 UD	224	{
f1740bc4 UD	225	int result = 0;
e51deae7 UD	226	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
	227	0))
	228	result = __lll_robust_lock_wait (futex, private);
f1740bc4 UD	229	return result;
f1740bc4 UD	230	}
e51deae7 UD	231	#define lll_robust_lock(futex, id, private) \
e51deae7 UD	232	__lll_robust_lock (&(futex), id, private)
f1740bc4	233
ef276e64 JJ	234	static inline void
ef276e64 JJ	235	__attribute__ ((always_inline))
e51deae7	236	__lll_cond_lock (int *futex, int private)
ef276e64	237	{
a1ffb40e	238	if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 2, 0)))
e51deae7	239	__lll_lock_wait (futex, private);
ef276e64	240	}
e51deae7	241	#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
ef276e64	242
e51deae7 UD	243	#define lll_robust_cond_lock(futex, id, private) \
e51deae7 UD	244	__lll_robust_lock (&(futex), (id) \| FUTEX_WAITERS, private)
f1740bc4	245
39358e8b	246	extern int __lll_timedlock_wait
e51deae7	247	(int futex, const struct timespec , int private) attribute_hidden;
f1740bc4	248	extern int __lll_robust_timedlock_wait
e51deae7	249	(int futex, const struct timespec , int private) attribute_hidden;
a88c9263 UD	250
	251	static inline int
	252	__attribute__ ((always_inline))
e51deae7	253	__lll_timedlock (int futex, const struct timespec abstime, int private)
a88c9263	254	{
a88c9263	255	int result = 0;
a1ffb40e	256	if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 1, 0)))
e51deae7	257	result = __lll_timedlock_wait (futex, abstime, private);
a88c9263 UD	258	return result;
a88c9263 UD	259	}
e51deae7 UD	260	#define lll_timedlock(futex, abstime, private) \
e51deae7 UD	261	__lll_timedlock (&(futex), abstime, private)
a88c9263	262
5a414ff7 DV	263	#ifdef ENABLE_LOCK_ELISION
	264	extern int __lll_timedlock_elision
	265	(int futex, short adapt_count, const struct timespec *timeout, int private)
	266	attribute_hidden;
	267
	268	# define lll_timedlock_elision(futex, adapt_count, timeout, private) \
	269	__lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
	270	#endif
	271
f1740bc4 UD	272	static inline int
f1740bc4 UD	273	__attribute__ ((always_inline))
e51deae7 UD	274	__lll_robust_timedlock (int futex, const struct timespec abstime,
e51deae7 UD	275	int id, int private)
f1740bc4 UD	276	{
f1740bc4 UD	277	int result = 0;
e51deae7 UD	278	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
	279	0))
	280	result = __lll_robust_timedlock_wait (futex, abstime, private);
f1740bc4 UD	281	return result;
f1740bc4 UD	282	}
e51deae7 UD	283	#define lll_robust_timedlock(futex, abstime, id, private) \
e51deae7 UD	284	__lll_robust_timedlock (&(futex), abstime, id, private)
f1740bc4	285
a88c9263	286
702a9414 JJ	287	#define __lll_unlock(futex, private) \
	288	(void) \
	289	({ int __oldval; \
	290	int __newval = 0; \
	291	int *__futexp = (futex); \
	292	\
	293	lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2"); \
a1ffb40e	294	if (__glibc_unlikely (__oldval > 1)) \
702a9414 JJ	295	lll_futex_wake (__futexp, 1, private); \
702a9414 JJ	296	})
e51deae7	297	#define lll_unlock(futex, private) __lll_unlock(&(futex), private)
284bdc42	298
b8ba4a27	299
702a9414 JJ	300	#define __lll_robust_unlock(futex, private) \
	301	(void) \
	302	({ int __oldval; \
	303	int __newval = 0; \
	304	int *__futexp = (futex); \
	305	\
	306	lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2"); \
a1ffb40e	307	if (__glibc_unlikely (__oldval & FUTEX_WAITERS)) \
702a9414 JJ	308	lll_futex_wake (__futexp, 1, private); \
702a9414 JJ	309	})
e51deae7 UD	310	#define lll_robust_unlock(futex, private) \
e51deae7 UD	311	__lll_robust_unlock(&(futex), private)
a88c9263	312
e51deae7	313	#define lll_islocked(futex) \
a88c9263 UD	314	(futex != 0)
	315
	316
a88c9263	317	/* Initializers for lock. */
39358e8b UD	318	#define LLL_LOCK_INITIALIZER (0)
39358e8b UD	319	#define LLL_LOCK_INITIALIZER_LOCKED (1)
a88c9263	320
adcdc775	321	/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
a88c9263 UD	322	wakeup when the clone terminates. The memory location contains the
	323	thread ID while the clone is running and is reset to zero
	324	afterwards. */
702a9414 JJ	325	#define __lll_wait_tid(ptid) \
	326	do \
	327	{ \
	328	int __tid; \
	329	\
	330	while ((__tid = *ptid) != 0) \
	331	lll_futex_wait (ptid, __tid, LLL_SHARED); \
	332	} \
	333	while (0)
a88c9263 UD	334	#define lll_wait_tid(tid) __lll_wait_tid(&(tid))
a88c9263 UD	335
39358e8b	336	extern int __lll_timedwait_tid (int , const struct timespec )
a88c9263	337	attribute_hidden;
a88c9263	338
39358e8b UD	339	#define lll_timedwait_tid(tid, abstime) \
	340	({ \
	341	int __res = 0; \
	342	if ((tid) != 0) \
	343	__res = __lll_timedwait_tid (&(tid), (abstime)); \
	344	__res; \
	345	})
a88c9263	346
5a414ff7 DV	347	#ifdef ENABLE_LOCK_ELISION
	348	extern int __lll_lock_elision (int futex, short adapt_count, int private)
	349	attribute_hidden;
	350
	351	extern int __lll_unlock_elision(int *futex, int private)
	352	attribute_hidden;
	353
	354	extern int __lll_trylock_elision(int futex, short adapt_count)
	355	attribute_hidden;
	356
	357	# define lll_lock_elision(futex, adapt_count, private) \
	358	__lll_lock_elision (&(futex), &(adapt_count), private)
	359	# define lll_unlock_elision(futex, private) \
	360	__lll_unlock_elision (&(futex), private)
	361	# define lll_trylock_elision(futex, adapt_count) \
	362	__lll_trylock_elision(&(futex), &(adapt_count))
	363	#endif
	364
a88c9263	365	#endif /* lowlevellock.h */