[thirdparty/glibc.git] / nptl / sysdeps / unix / sysv / linux / s390 / lowlevellock.h

/* Copyright (C) 2003-2014 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#ifndef _LOWLEVELLOCK_H
#define _LOWLEVELLOCK_H	1

#include <time.h>
#include <sys/param.h>
#include <bits/pthreadtypes.h>
#include <atomic.h>
#include <kernel-features.h>

#define SYS_futex		238
#define FUTEX_WAIT		0
#define FUTEX_WAKE		1
#define FUTEX_REQUEUE		3
#define FUTEX_CMP_REQUEUE	4
#define FUTEX_WAKE_OP		5
#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE	((4 << 24) | 1)
#define FUTEX_LOCK_PI		6
#define FUTEX_UNLOCK_PI		7
#define FUTEX_TRYLOCK_PI	8
#define FUTEX_WAIT_BITSET	9
#define FUTEX_WAKE_BITSET	10
#define FUTEX_WAIT_REQUEUE_PI   11
#define FUTEX_CMP_REQUEUE_PI    12
#define FUTEX_PRIVATE_FLAG	128
#define FUTEX_CLOCK_REALTIME	256

#define FUTEX_BITSET_MATCH_ANY	0xffffffff

/* Values for 'private' parameter of locking macros.  Yes, the
   definition seems to be backwards.  But it is not.  The bit will be
   reversed before passing to the system call.  */
#define LLL_PRIVATE	0
#define LLL_SHARED	FUTEX_PRIVATE_FLAG


#if !defined NOT_IN_libc || defined IS_IN_rtld
/* In libc.so or ld.so all futexes are private.  */
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  ((fl) | FUTEX_PRIVATE_FLAG)
# else
#  define __lll_private_flag(fl, private) \
  ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
# endif
#else
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
# else
#  define __lll_private_flag(fl, private) \
  (__builtin_constant_p (private)					      \
   ? ((private) == 0							      \
      ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))	      \
      : (fl))								      \
   : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG)				      \
	      & THREAD_GETMEM (THREAD_SELF, header.private_futex))))
# endif
#endif

#define lll_futex_wait(futex, val, private) \
  lll_futex_timed_wait (futex, val, NULL, private)

#define lll_futex_timed_wait(futexp, val, timespec, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
									      \
    INTERNAL_SYSCALL (futex, __err, 4, (futexp),		      \
			      __lll_private_flag (FUTEX_WAIT, private),	      \
			      (val), (timespec));			      \
  })

#define lll_futex_timed_wait_bitset(futexp, val, timespec, clockbit, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    int __op = FUTEX_WAIT_BITSET | clockbit;				      \
									      \
    INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
			      __lll_private_flag (__op, private),	      \
			      (val), (timespec), NULL /* Unused.  */, 	      \
			      FUTEX_BITSET_MATCH_ANY);			      \
  })

#define lll_futex_wake(futexp, nr, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
									      \
    INTERNAL_SYSCALL (futex, __err, 4, (futexp),		      \
			      __lll_private_flag (FUTEX_WAKE, private),	      \
			      (nr), 0);					      \
  })

#define lll_robust_dead(futexv, private) \
  do									      \
    {									      \
      int *__futexp = &(futexv);					      \
									      \
      atomic_or (__futexp, FUTEX_OWNER_DIED);				      \
      lll_futex_wake (__futexp, 1, private);				      \
    }									      \
  while (0)


/* Returns non-zero if error happened, zero if success.  */
#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    long int __ret;							      \
									      \
    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
			      __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
			      (nr_wake), (nr_move), (mutex), (val));	      \
    INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
  })

/* Returns non-zero if error happened, zero if success.  */
#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    long int __ret;							      \
									      \
    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
			      __lll_private_flag (FUTEX_WAKE_OP, private),    \
			      (nr_wake), (nr_wake2), (futexp2),		      \
			      FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);		      \
    INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
  })

/* Priority Inheritance support.  */
#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
  lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)

#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit,      \
					mutex, private)			      \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    int __op = FUTEX_WAIT_REQUEUE_PI | clockbit;			      \
									      \
    INTERNAL_SYSCALL (futex, __err, 5, (futexp),			      \
		      __lll_private_flag (__op, private),		      \
		      (val), (timespec), mutex); 			      \
  })

#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv)  \
  ({									      \
    INTERNAL_SYSCALL_DECL (__err);					      \
    long int __ret;							      \
									      \
    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
			      __lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
			      (nr_wake), (nr_move), (mutex), (val));	      \
    INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
  })

#define lll_compare_and_swap(futex, oldval, newval, operation) \
  do {									      \
    __typeof (futex) __futex = (futex);					      \
    __asm __volatile ("	  l   %1,%0\n"					      \
		      "0: " operation "\n"				      \
		      "	  cs  %1,%2,%0\n"				      \
		      "	  jl  0b\n"					      \
		      "1:"						      \
		      : "=Q" (*__futex), "=&d" (oldval), "=&d" (newval)	      \
		      : "m" (*__futex) : "cc", "memory" );		      \
  } while (0)


static inline int
__attribute__ ((always_inline))
__lll_trylock (int *futex)
{
    unsigned int old;

    __asm __volatile ("cs %0,%3,%1"
		       : "=d" (old), "=Q" (*futex)
		       : "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
    return old != 0;
}
#define lll_trylock(futex) __lll_trylock (&(futex))


static inline int
__attribute__ ((always_inline))
__lll_cond_trylock (int *futex)
{
    unsigned int old;

    __asm __volatile ("cs %0,%3,%1"
		       : "=d" (old), "=Q" (*futex)
		       : "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
    return old != 0;
}
#define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))


static inline int
__attribute__ ((always_inline))
__lll_robust_trylock (int *futex, int id)
{
    unsigned int old;

    __asm __volatile ("cs %0,%3,%1"
		       : "=d" (old), "=Q" (*futex)
		       : "0" (0), "d" (id), "m" (*futex) : "cc", "memory" );
    return old != 0;
}
#define lll_robust_trylock(futex, id) \
  __lll_robust_trylock (&(futex), id)


extern void __lll_lock_wait_private (int *futex) attribute_hidden;
extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;

static inline void
__attribute__ ((always_inline))
__lll_lock (int *futex, int private)
{
  if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 1, 0)))
    {
      if (__builtin_constant_p (private) && private == LLL_PRIVATE)
	__lll_lock_wait_private (futex);
      else
	__lll_lock_wait (futex, private);
    }
}
#define lll_lock(futex, private) __lll_lock (&(futex), private)

static inline int
__attribute__ ((always_inline))
__lll_robust_lock (int *futex, int id, int private)
{
  int result = 0;
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
			0))
    result = __lll_robust_lock_wait (futex, private);
  return result;
}
#define lll_robust_lock(futex, id, private) \
  __lll_robust_lock (&(futex), id, private)

static inline void
__attribute__ ((always_inline))
__lll_cond_lock (int *futex, int private)
{
  if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 2, 0)))
    __lll_lock_wait (futex, private);
}
#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)

#define lll_robust_cond_lock(futex, id, private) \
  __lll_robust_lock (&(futex), (id) | FUTEX_WAITERS, private)

extern int __lll_timedlock_wait
  (int *futex, const struct timespec *, int private) attribute_hidden;
extern int __lll_robust_timedlock_wait
  (int *futex, const struct timespec *, int private) attribute_hidden;

static inline int
__attribute__ ((always_inline))
__lll_timedlock (int *futex, const struct timespec *abstime, int private)
{
  int result = 0;
  if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 1, 0)))
    result = __lll_timedlock_wait (futex, abstime, private);
  return result;
}
#define lll_timedlock(futex, abstime, private) \
  __lll_timedlock (&(futex), abstime, private)

static inline int
__attribute__ ((always_inline))
__lll_robust_timedlock (int *futex, const struct timespec *abstime,
			int id, int private)
{
  int result = 0;
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
			0))
    result = __lll_robust_timedlock_wait (futex, abstime, private);
  return result;
}
#define lll_robust_timedlock(futex, abstime, id, private) \
  __lll_robust_timedlock (&(futex), abstime, id, private)


#define __lll_unlock(futex, private) \
  (void)								      \
    ({ int __oldval;							      \
       int __newval = 0;						      \
       int *__futexp = (futex);						      \
									      \
       lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
       if (__glibc_unlikely (__oldval > 1))				      \
	 lll_futex_wake (__futexp, 1, private);				      \
    })
#define lll_unlock(futex, private) __lll_unlock(&(futex), private)


#define __lll_robust_unlock(futex, private) \
  (void)								      \
    ({ int __oldval;							      \
       int __newval = 0;						      \
       int *__futexp = (futex);						      \
									      \
       lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
       if (__glibc_unlikely (__oldval & FUTEX_WAITERS))			      \
	 lll_futex_wake (__futexp, 1, private);				      \
    })
#define lll_robust_unlock(futex, private) \
  __lll_robust_unlock(&(futex), private)

#define lll_islocked(futex) \
  (futex != 0)


/* Initializers for lock.  */
#define LLL_LOCK_INITIALIZER		(0)
#define LLL_LOCK_INITIALIZER_LOCKED	(1)

/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
   wakeup when the clone terminates.  The memory location contains the
   thread ID while the clone is running and is reset to zero
   afterwards.	*/
#define __lll_wait_tid(ptid) \
  do									      \
    {									      \
      int __tid;							      \
									      \
      while ((__tid = *ptid) != 0)					      \
	lll_futex_wait (ptid, __tid, LLL_SHARED);			      \
    }									      \
  while (0)
#define lll_wait_tid(tid) __lll_wait_tid(&(tid))

extern int __lll_timedwait_tid (int *, const struct timespec *)
     attribute_hidden;

#define lll_timedwait_tid(tid, abstime) \
  ({									      \
    int __res = 0;							      \
    if ((tid) != 0)							      \
      __res = __lll_timedwait_tid (&(tid), (abstime));			      \
    __res;								      \
  })

#endif	/* lowlevellock.h */
Commit	Line	Data
d4697bc9	1	/* Copyright (C) 2003-2014 Free Software Foundation, Inc.
a88c9263 UD	2	This file is part of the GNU C Library.
	3	Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
a88c9263 UD	18
	19	#ifndef _LOWLEVELLOCK_H
	20	#define _LOWLEVELLOCK_H 1
	21
	22	#include <time.h>
	23	#include <sys/param.h>
	24	#include <bits/pthreadtypes.h>
39358e8b	25	#include <atomic.h>
f8de5057	26	#include <kernel-features.h>
a88c9263 UD	27
	28	#define SYS_futex 238
	29	#define FUTEX_WAIT 0
	30	#define FUTEX_WAKE 1
284bdc42	31	#define FUTEX_REQUEUE 3
75fccede	32	#define FUTEX_CMP_REQUEUE 4
b9b8cf03 UD	33	#define FUTEX_WAKE_OP 5
b9b8cf03 UD	34	#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) \| 1)
d5ba53f9 UD	35	#define FUTEX_LOCK_PI 6
	36	#define FUTEX_UNLOCK_PI 7
	37	#define FUTEX_TRYLOCK_PI 8
da5ac135 UD	38	#define FUTEX_WAIT_BITSET 9
da5ac135 UD	39	#define FUTEX_WAKE_BITSET 10
8313cb99 SP	40	#define FUTEX_WAIT_REQUEUE_PI 11
8313cb99 SP	41	#define FUTEX_CMP_REQUEUE_PI 12
085a4412	42	#define FUTEX_PRIVATE_FLAG 128
7dd650d7 UD	43	#define FUTEX_CLOCK_REALTIME 256
	44
	45	#define FUTEX_BITSET_MATCH_ANY 0xffffffff
085a4412 UD	46
	47	/* Values for 'private' parameter of locking macros. Yes, the
	48	definition seems to be backwards. But it is not. The bit will be
	49	reversed before passing to the system call. */
	50	#define LLL_PRIVATE 0
	51	#define LLL_SHARED FUTEX_PRIVATE_FLAG
	52
	53
	54	#if !defined NOT_IN_libc \|\| defined IS_IN_rtld
	55	/* In libc.so or ld.so all futexes are private. */
	56	# ifdef __ASSUME_PRIVATE_FUTEX
	57	# define __lll_private_flag(fl, private) \
	58	((fl) \| FUTEX_PRIVATE_FLAG)
	59	# else
	60	# define __lll_private_flag(fl, private) \
	61	((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex))
	62	# endif
	63	#else
	64	# ifdef __ASSUME_PRIVATE_FUTEX
	65	# define __lll_private_flag(fl, private) \
	66	(((fl) \| FUTEX_PRIVATE_FLAG) ^ (private))
	67	# else
	68	# define __lll_private_flag(fl, private) \
	69	(__builtin_constant_p (private) \
	70	? ((private) == 0 \
	71	? ((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex)) \
	72	: (fl)) \
	73	: ((fl) \| (((private) ^ FUTEX_PRIVATE_FLAG) \
	74	& THREAD_GETMEM (THREAD_SELF, header.private_futex))))
da5ac135	75	# endif
085a4412	76	#endif
a88c9263	77
085a4412 UD	78	#define lll_futex_wait(futex, val, private) \
085a4412 UD	79	lll_futex_timed_wait (futex, val, NULL, private)
a88c9263	80
8ebac778	81	#define lll_futex_timed_wait(futexp, val, timespec, private) \
a88c9263	82	({ \
8ebac778	83	INTERNAL_SYSCALL_DECL (__err); \
a88c9263	84	\
8ebac778 SP	85	INTERNAL_SYSCALL (futex, __err, 4, (futexp), \
	86	__lll_private_flag (FUTEX_WAIT, private), \
	87	(val), (timespec)); \
a88c9263 UD	88	})
a88c9263 UD	89
8f861542 SP	90	#define lll_futex_timed_wait_bitset(futexp, val, timespec, clockbit, private) \
8f861542 SP	91	({ \
8ebac778 SP	92	INTERNAL_SYSCALL_DECL (__err); \
8ebac778 SP	93	int __op = FUTEX_WAIT_BITSET \| clockbit; \
8f861542	94	\
8ebac778 SP	95	INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
	96	__lll_private_flag (__op, private), \
	97	(val), (timespec), NULL /* Unused. */, \
	98	FUTEX_BITSET_MATCH_ANY); \
8f861542 SP	99	})
8f861542 SP	100
8ebac778	101	#define lll_futex_wake(futexp, nr, private) \
a88c9263	102	({ \
8ebac778	103	INTERNAL_SYSCALL_DECL (__err); \
a88c9263	104	\
8ebac778 SP	105	INTERNAL_SYSCALL (futex, __err, 4, (futexp), \
	106	__lll_private_flag (FUTEX_WAKE, private), \
	107	(nr), 0); \
a88c9263 UD	108	})
a88c9263 UD	109
e51deae7	110	#define lll_robust_dead(futexv, private) \
f1740bc4 UD	111	do \
	112	{ \
	113	int *__futexp = &(futexv); \
	114	\
	115	atomic_or (__futexp, FUTEX_OWNER_DIED); \
e51deae7	116	lll_futex_wake (__futexp, 1, private); \
f1740bc4 UD	117	} \
	118	while (0)
	119
	120
75fccede	121	/* Returns non-zero if error happened, zero if success. */
8ebac778	122	#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
284bdc42	123	({ \
8ebac778 SP	124	INTERNAL_SYSCALL_DECL (__err); \
8ebac778 SP	125	long int __ret; \
b9b8cf03	126	\
8ebac778 SP	127	__ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
	128	__lll_private_flag (FUTEX_CMP_REQUEUE, private),\
	129	(nr_wake), (nr_move), (mutex), (val)); \
	130	INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
b9b8cf03 UD	131	})
b9b8cf03 UD	132
b9b8cf03	133	/* Returns non-zero if error happened, zero if success. */
8ebac778	134	#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
b9b8cf03	135	({ \
8ebac778 SP	136	INTERNAL_SYSCALL_DECL (__err); \
8ebac778 SP	137	long int __ret; \
284bdc42	138	\
8ebac778 SP	139	__ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
	140	__lll_private_flag (FUTEX_WAKE_OP, private), \
	141	(nr_wake), (nr_wake2), (futexp2), \
	142	FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \
	143	INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
284bdc42 UD	144	})
284bdc42 UD	145
8313cb99 SP	146	/* Priority Inheritance support. */
	147	#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
	148	lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
	149
	150	#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit, \
	151	mutex, private) \
	152	({ \
	153	INTERNAL_SYSCALL_DECL (__err); \
	154	int __op = FUTEX_WAIT_REQUEUE_PI \| clockbit; \
	155	\
	156	INTERNAL_SYSCALL (futex, __err, 5, (futexp), \
	157	__lll_private_flag (__op, private), \
	158	(val), (timespec), mutex); \
	159	})
	160
	161	#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv) \
	162	({ \
	163	INTERNAL_SYSCALL_DECL (__err); \
	164	long int __ret; \
	165	\
	166	__ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
	167	__lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
	168	(nr_wake), (nr_move), (mutex), (val)); \
	169	INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
	170	})
	171
a88c9263 UD	172	#define lll_compare_and_swap(futex, oldval, newval, operation) \
	173	do { \
	174	__typeof (futex) __futex = (futex); \
	175	__asm __volatile (" l %1,%0\n" \
	176	"0: " operation "\n" \
	177	" cs %1,%2,%0\n" \
	178	" jl 0b\n" \
	179	"1:" \
	180	: "=Q" (*__futex), "=&d" (oldval), "=&d" (newval) \
f1847a84	181	: "m" (*__futex) : "cc", "memory" ); \
a88c9263 UD	182	} while (0)
	183
	184
	185	static inline int
	186	__attribute__ ((always_inline))
e51deae7	187	__lll_trylock (int *futex)
a88c9263 UD	188	{
	189	unsigned int old;
	190
	191	__asm __volatile ("cs %0,%3,%1"
	192	: "=d" (old), "=Q" (*futex)
f1847a84	193	: "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
a88c9263 UD	194	return old != 0;
a88c9263 UD	195	}
e51deae7	196	#define lll_trylock(futex) __lll_trylock (&(futex))
a88c9263 UD	197
a88c9263 UD	198
2c0b891a UD	199	static inline int
2c0b891a UD	200	__attribute__ ((always_inline))
e51deae7	201	__lll_cond_trylock (int *futex)
2c0b891a UD	202	{
	203	unsigned int old;
	204
	205	__asm __volatile ("cs %0,%3,%1"
	206	: "=d" (old), "=Q" (*futex)
f1847a84	207	: "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
2c0b891a UD	208	return old != 0;
2c0b891a UD	209	}
e51deae7	210	#define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))
2c0b891a UD	211
2c0b891a UD	212
f1740bc4 UD	213	static inline int
f1740bc4 UD	214	__attribute__ ((always_inline))
e51deae7	215	__lll_robust_trylock (int *futex, int id)
f1740bc4 UD	216	{
	217	unsigned int old;
	218
	219	__asm __volatile ("cs %0,%3,%1"
	220	: "=d" (old), "=Q" (*futex)
	221	: "0" (0), "d" (id), "m" (*futex) : "cc", "memory" );
	222	return old != 0;
	223	}
e51deae7 UD	224	#define lll_robust_trylock(futex, id) \
e51deae7 UD	225	__lll_robust_trylock (&(futex), id)
f1740bc4 UD	226
f1740bc4 UD	227
e51deae7 UD	228	extern void __lll_lock_wait_private (int *futex) attribute_hidden;
	229	extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
	230	extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;
a88c9263 UD	231
	232	static inline void
	233	__attribute__ ((always_inline))
e51deae7	234	__lll_lock (int *futex, int private)
a88c9263	235	{
a1ffb40e	236	if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 1, 0)))
e51deae7 UD	237	{
	238	if (__builtin_constant_p (private) && private == LLL_PRIVATE)
	239	__lll_lock_wait_private (futex);
	240	else
	241	__lll_lock_wait (futex, private);
	242	}
a88c9263	243	}
e51deae7	244	#define lll_lock(futex, private) __lll_lock (&(futex), private)
a88c9263	245
f1740bc4 UD	246	static inline int
f1740bc4 UD	247	__attribute__ ((always_inline))
e51deae7	248	__lll_robust_lock (int *futex, int id, int private)
f1740bc4 UD	249	{
f1740bc4 UD	250	int result = 0;
e51deae7 UD	251	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
	252	0))
	253	result = __lll_robust_lock_wait (futex, private);
f1740bc4 UD	254	return result;
f1740bc4 UD	255	}
e51deae7 UD	256	#define lll_robust_lock(futex, id, private) \
e51deae7 UD	257	__lll_robust_lock (&(futex), id, private)
f1740bc4	258
ef276e64 JJ	259	static inline void
ef276e64 JJ	260	__attribute__ ((always_inline))
e51deae7	261	__lll_cond_lock (int *futex, int private)
ef276e64	262	{
a1ffb40e	263	if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 2, 0)))
e51deae7	264	__lll_lock_wait (futex, private);
ef276e64	265	}
e51deae7	266	#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
ef276e64	267
e51deae7 UD	268	#define lll_robust_cond_lock(futex, id, private) \
e51deae7 UD	269	__lll_robust_lock (&(futex), (id) \| FUTEX_WAITERS, private)
f1740bc4	270
39358e8b	271	extern int __lll_timedlock_wait
e51deae7	272	(int futex, const struct timespec , int private) attribute_hidden;
f1740bc4	273	extern int __lll_robust_timedlock_wait
e51deae7	274	(int futex, const struct timespec , int private) attribute_hidden;
a88c9263 UD	275
	276	static inline int
	277	__attribute__ ((always_inline))
e51deae7	278	__lll_timedlock (int futex, const struct timespec abstime, int private)
a88c9263	279	{
a88c9263	280	int result = 0;
a1ffb40e	281	if (__glibc_unlikely (atomic_compare_and_exchange_bool_acq (futex, 1, 0)))
e51deae7	282	result = __lll_timedlock_wait (futex, abstime, private);
a88c9263 UD	283	return result;
a88c9263 UD	284	}
e51deae7 UD	285	#define lll_timedlock(futex, abstime, private) \
e51deae7 UD	286	__lll_timedlock (&(futex), abstime, private)
a88c9263	287
f1740bc4 UD	288	static inline int
f1740bc4 UD	289	__attribute__ ((always_inline))
e51deae7 UD	290	__lll_robust_timedlock (int futex, const struct timespec abstime,
e51deae7 UD	291	int id, int private)
f1740bc4 UD	292	{
f1740bc4 UD	293	int result = 0;
e51deae7 UD	294	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
	295	0))
	296	result = __lll_robust_timedlock_wait (futex, abstime, private);
f1740bc4 UD	297	return result;
f1740bc4 UD	298	}
e51deae7 UD	299	#define lll_robust_timedlock(futex, abstime, id, private) \
e51deae7 UD	300	__lll_robust_timedlock (&(futex), abstime, id, private)
f1740bc4	301
a88c9263	302
702a9414 JJ	303	#define __lll_unlock(futex, private) \
	304	(void) \
	305	({ int __oldval; \
	306	int __newval = 0; \
	307	int *__futexp = (futex); \
	308	\
	309	lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2"); \
a1ffb40e	310	if (__glibc_unlikely (__oldval > 1)) \
702a9414 JJ	311	lll_futex_wake (__futexp, 1, private); \
702a9414 JJ	312	})
e51deae7	313	#define lll_unlock(futex, private) __lll_unlock(&(futex), private)
284bdc42	314
b8ba4a27	315
702a9414 JJ	316	#define __lll_robust_unlock(futex, private) \
	317	(void) \
	318	({ int __oldval; \
	319	int __newval = 0; \
	320	int *__futexp = (futex); \
	321	\
	322	lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2"); \
a1ffb40e	323	if (__glibc_unlikely (__oldval & FUTEX_WAITERS)) \
702a9414 JJ	324	lll_futex_wake (__futexp, 1, private); \
702a9414 JJ	325	})
e51deae7 UD	326	#define lll_robust_unlock(futex, private) \
e51deae7 UD	327	__lll_robust_unlock(&(futex), private)
a88c9263	328
e51deae7	329	#define lll_islocked(futex) \
a88c9263 UD	330	(futex != 0)
	331
	332
a88c9263	333	/* Initializers for lock. */
39358e8b UD	334	#define LLL_LOCK_INITIALIZER (0)
39358e8b UD	335	#define LLL_LOCK_INITIALIZER_LOCKED (1)
a88c9263	336
adcdc775	337	/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
a88c9263 UD	338	wakeup when the clone terminates. The memory location contains the
	339	thread ID while the clone is running and is reset to zero
	340	afterwards. */
702a9414 JJ	341	#define __lll_wait_tid(ptid) \
	342	do \
	343	{ \
	344	int __tid; \
	345	\
	346	while ((__tid = *ptid) != 0) \
	347	lll_futex_wait (ptid, __tid, LLL_SHARED); \
	348	} \
	349	while (0)
a88c9263 UD	350	#define lll_wait_tid(tid) __lll_wait_tid(&(tid))
a88c9263 UD	351
39358e8b	352	extern int __lll_timedwait_tid (int , const struct timespec )
a88c9263	353	attribute_hidden;
a88c9263	354
39358e8b UD	355	#define lll_timedwait_tid(tid, abstime) \
	356	({ \
	357	int __res = 0; \
	358	if ((tid) != 0) \
	359	__res = __lll_timedwait_tid (&(tid), (abstime)); \
	360	__res; \
	361	})
a88c9263	362
a88c9263	363	#endif /* lowlevellock.h */