[thirdparty/glibc.git] / nptl / sysdeps / unix / sysv / linux / s390 / lowlevellock.h

/* Copyright (C) 2003, 2004, 2006-2008, 2009 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#ifndef _LOWLEVELLOCK_H
#define _LOWLEVELLOCK_H	1

#include <time.h>
#include <sys/param.h>
#include <bits/pthreadtypes.h>
#include <atomic.h>
#include <kernel-features.h>

#define SYS_futex		238
#define FUTEX_WAIT		0
#define FUTEX_WAKE		1
#define FUTEX_REQUEUE		3
#define FUTEX_CMP_REQUEUE	4
#define FUTEX_WAKE_OP		5
#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE	((4 << 24) | 1)
#define FUTEX_LOCK_PI		6
#define FUTEX_UNLOCK_PI		7
#define FUTEX_TRYLOCK_PI	8
#define FUTEX_WAIT_BITSET	9
#define FUTEX_WAKE_BITSET	10
#define FUTEX_PRIVATE_FLAG	128
#define FUTEX_CLOCK_REALTIME	256

#define FUTEX_BITSET_MATCH_ANY	0xffffffff

/* Values for 'private' parameter of locking macros.  Yes, the
   definition seems to be backwards.  But it is not.  The bit will be
   reversed before passing to the system call.  */
#define LLL_PRIVATE	0
#define LLL_SHARED	FUTEX_PRIVATE_FLAG


#if !defined NOT_IN_libc || defined IS_IN_rtld
/* In libc.so or ld.so all futexes are private.  */
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  ((fl) | FUTEX_PRIVATE_FLAG)
# else
#  define __lll_private_flag(fl, private) \
  ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
# endif
#else
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
# else
#  define __lll_private_flag(fl, private) \
  (__builtin_constant_p (private)					      \
   ? ((private) == 0							      \
      ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))	      \
      : (fl))								      \
   : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG)				      \
	      & THREAD_GETMEM (THREAD_SELF, header.private_futex))))
# endif
#endif

#define lll_futex_wait(futex, val, private) \
  lll_futex_timed_wait (futex, val, NULL, private)

#define lll_futex_timed_wait(futex, val, timespec, private) \
  ({									      \
    register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
    register unsigned long int __r3 asm ("3")				      \
      = __lll_private_flag (FUTEX_WAIT, private);			      \
    register unsigned long int __r4 asm ("4") = (unsigned long int) (val);    \
    register unsigned long int __r5 asm ("5") = (unsigned long int)(timespec);\
    register unsigned long int __result asm ("2");			      \
									      \
    __asm __volatile ("svc %b1"						      \
		      : "=d" (__result)					      \
		      : "i" (SYS_futex), "0" (__r2), "d" (__r3),	      \
			"d" (__r4), "d" (__r5)				      \
		      : "cc", "memory" );				      \
    __result;								      \
  })


#define lll_futex_wake(futex, nr, private) \
  ({									      \
    register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
    register unsigned long int __r3 asm ("3")				      \
      = __lll_private_flag (FUTEX_WAKE, private);			      \
    register unsigned long int __r4 asm ("4") = (unsigned long int) (nr);     \
    register unsigned long int __result asm ("2");			      \
									      \
    __asm __volatile ("svc %b1"						      \
		      : "=d" (__result)					      \
		      : "i" (SYS_futex), "0" (__r2), "d" (__r3), "d" (__r4)   \
		      : "cc", "memory" );				      \
    __result;								      \
  })


#define lll_robust_dead(futexv, private) \
  do									      \
    {									      \
      int *__futexp = &(futexv);					      \
									      \
      atomic_or (__futexp, FUTEX_OWNER_DIED);				      \
      lll_futex_wake (__futexp, 1, private);				      \
    }									      \
  while (0)


/* Returns non-zero if error happened, zero if success.  */
#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \
  ({									      \
    register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
    register unsigned long int __r3 asm ("3")				      \
      = __lll_private_flag (FUTEX_CMP_REQUEUE, private);		      \
    register unsigned long int __r4 asm ("4") = (long int) (nr_wake);	      \
    register unsigned long int __r5 asm ("5") = (long int) (nr_move);	      \
    register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex);  \
    register unsigned long int __r7 asm ("7") = (int) (val);		      \
    register unsigned long __result asm ("2");				      \
									      \
    __asm __volatile ("svc %b1"						      \
		      : "=d" (__result)					      \
		      : "i" (SYS_futex), "0" (__r2), "d" (__r3),	      \
			"d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7)	      \
		      : "cc", "memory" );				      \
    __result > -4096UL;							      \
  })


/* Returns non-zero if error happened, zero if success.  */
#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \
  ({									      \
    register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
    register unsigned long int __r3 asm ("3")				      \
      = __lll_private_flag (FUTEX_WAKE_OP, private);			      \
    register unsigned long int __r4 asm ("4") = (long int) (nr_wake);	      \
    register unsigned long int __r5 asm ("5") = (long int) (nr_wake2);	      \
    register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \
    register unsigned long int __r7 asm ("7")				      \
      = (int) FUTEX_OP_CLEAR_WAKE_IF_GT_ONE;				      \
    register unsigned long __result asm ("2");				      \
									      \
    __asm __volatile ("svc %b1"						      \
		      : "=d" (__result)					      \
		      : "i" (SYS_futex), "0" (__r2), "d" (__r3),	      \
			"d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7)	      \
		      : "cc", "memory" );				      \
    __result > -4096UL;							      \
  })


#define lll_compare_and_swap(futex, oldval, newval, operation) \
  do {									      \
    __typeof (futex) __futex = (futex);					      \
    __asm __volatile ("	  l   %1,%0\n"					      \
		      "0: " operation "\n"				      \
		      "	  cs  %1,%2,%0\n"				      \
		      "	  jl  0b\n"					      \
		      "1:"						      \
		      : "=Q" (*__futex), "=&d" (oldval), "=&d" (newval)	      \
		      : "m" (*__futex) : "cc", "memory" );		      \
  } while (0)


static inline int
__attribute__ ((always_inline))
__lll_trylock (int *futex)
{
    unsigned int old;

    __asm __volatile ("cs %0,%3,%1"
		       : "=d" (old), "=Q" (*futex)
		       : "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
    return old != 0;
}
#define lll_trylock(futex) __lll_trylock (&(futex))


static inline int
__attribute__ ((always_inline))
__lll_cond_trylock (int *futex)
{
    unsigned int old;

    __asm __volatile ("cs %0,%3,%1"
		       : "=d" (old), "=Q" (*futex)
		       : "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
    return old != 0;
}
#define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))


static inline int
__attribute__ ((always_inline))
__lll_robust_trylock (int *futex, int id)
{
    unsigned int old;

    __asm __volatile ("cs %0,%3,%1"
		       : "=d" (old), "=Q" (*futex)
		       : "0" (0), "d" (id), "m" (*futex) : "cc", "memory" );
    return old != 0;
}
#define lll_robust_trylock(futex, id) \
  __lll_robust_trylock (&(futex), id)


extern void __lll_lock_wait_private (int *futex) attribute_hidden;
extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;

static inline void
__attribute__ ((always_inline))
__lll_lock (int *futex, int private)
{
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
    {
      if (__builtin_constant_p (private) && private == LLL_PRIVATE)
	__lll_lock_wait_private (futex);
      else
	__lll_lock_wait (futex, private);
    }
}
#define lll_lock(futex, private) __lll_lock (&(futex), private)

static inline int
__attribute__ ((always_inline))
__lll_robust_lock (int *futex, int id, int private)
{
  int result = 0;
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
			0))
    result = __lll_robust_lock_wait (futex, private);
  return result;
}
#define lll_robust_lock(futex, id, private) \
  __lll_robust_lock (&(futex), id, private)

static inline void
__attribute__ ((always_inline))
__lll_cond_lock (int *futex, int private)
{
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 2, 0), 0))
    __lll_lock_wait (futex, private);
}
#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)

#define lll_robust_cond_lock(futex, id, private) \
  __lll_robust_lock (&(futex), (id) | FUTEX_WAITERS, private)

extern int __lll_timedlock_wait
  (int *futex, const struct timespec *, int private) attribute_hidden;
extern int __lll_robust_timedlock_wait
  (int *futex, const struct timespec *, int private) attribute_hidden;

static inline int
__attribute__ ((always_inline))
__lll_timedlock (int *futex, const struct timespec *abstime, int private)
{
  int result = 0;
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
    result = __lll_timedlock_wait (futex, abstime, private);
  return result;
}
#define lll_timedlock(futex, abstime, private) \
  __lll_timedlock (&(futex), abstime, private)

static inline int
__attribute__ ((always_inline))
__lll_robust_timedlock (int *futex, const struct timespec *abstime,
			int id, int private)
{
  int result = 0;
  if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
			0))
    result = __lll_robust_timedlock_wait (futex, abstime, private);
  return result;
}
#define lll_robust_timedlock(futex, abstime, id, private) \
  __lll_robust_timedlock (&(futex), abstime, id, private)


#define __lll_unlock(futex, private) \
  (void)								      \
    ({ int __oldval;							      \
       int __newval = 0;						      \
       int *__futexp = (futex);						      \
									      \
       lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
       if (__builtin_expect (__oldval > 1, 0))				      \
	 lll_futex_wake (__futexp, 1, private);				      \
    })
#define lll_unlock(futex, private) __lll_unlock(&(futex), private)


#define __lll_robust_unlock(futex, private) \
  (void)								      \
    ({ int __oldval;							      \
       int __newval = 0;						      \
       int *__futexp = (futex);						      \
									      \
       lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
       if (__builtin_expect (__oldval & FUTEX_WAITERS, 0))		      \
	 lll_futex_wake (__futexp, 1, private);				      \
    })
#define lll_robust_unlock(futex, private) \
  __lll_robust_unlock(&(futex), private)

#define lll_islocked(futex) \
  (futex != 0)


/* Initializers for lock.  */
#define LLL_LOCK_INITIALIZER		(0)
#define LLL_LOCK_INITIALIZER_LOCKED	(1)

/* The kernel notifies a process with uses CLONE_CLEARTID via futex
   wakeup when the clone terminates.  The memory location contains the
   thread ID while the clone is running and is reset to zero
   afterwards.	*/
#define __lll_wait_tid(ptid) \
  do									      \
    {									      \
      int __tid;							      \
									      \
      while ((__tid = *ptid) != 0)					      \
	lll_futex_wait (ptid, __tid, LLL_SHARED);			      \
    }									      \
  while (0)
#define lll_wait_tid(tid) __lll_wait_tid(&(tid))

extern int __lll_timedwait_tid (int *, const struct timespec *)
     attribute_hidden;

#define lll_timedwait_tid(tid, abstime) \
  ({									      \
    int __res = 0;							      \
    if ((tid) != 0)							      \
      __res = __lll_timedwait_tid (&(tid), (abstime));			      \
    __res;								      \
  })

#endif	/* lowlevellock.h */
Commit	Line	Data
7dd650d7	1	/* Copyright (C) 2003, 2004, 2006-2008, 2009 Free Software Foundation, Inc.
a88c9263 UD	2	This file is part of the GNU C Library.
	3	Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
a88c9263 UD	18
	19	#ifndef _LOWLEVELLOCK_H
	20	#define _LOWLEVELLOCK_H 1
	21
	22	#include <time.h>
	23	#include <sys/param.h>
	24	#include <bits/pthreadtypes.h>
39358e8b	25	#include <atomic.h>
f8de5057	26	#include <kernel-features.h>
a88c9263 UD	27
	28	#define SYS_futex 238
	29	#define FUTEX_WAIT 0
	30	#define FUTEX_WAKE 1
284bdc42	31	#define FUTEX_REQUEUE 3
75fccede	32	#define FUTEX_CMP_REQUEUE 4
b9b8cf03 UD	33	#define FUTEX_WAKE_OP 5
b9b8cf03 UD	34	#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) \| 1)
d5ba53f9 UD	35	#define FUTEX_LOCK_PI 6
	36	#define FUTEX_UNLOCK_PI 7
	37	#define FUTEX_TRYLOCK_PI 8
da5ac135 UD	38	#define FUTEX_WAIT_BITSET 9
da5ac135 UD	39	#define FUTEX_WAKE_BITSET 10
085a4412	40	#define FUTEX_PRIVATE_FLAG 128
7dd650d7 UD	41	#define FUTEX_CLOCK_REALTIME 256
	42
	43	#define FUTEX_BITSET_MATCH_ANY 0xffffffff
085a4412 UD	44
	45	/* Values for 'private' parameter of locking macros. Yes, the
	46	definition seems to be backwards. But it is not. The bit will be
	47	reversed before passing to the system call. */
	48	#define LLL_PRIVATE 0
	49	#define LLL_SHARED FUTEX_PRIVATE_FLAG
	50
	51
	52	#if !defined NOT_IN_libc \|\| defined IS_IN_rtld
	53	/* In libc.so or ld.so all futexes are private. */
	54	# ifdef __ASSUME_PRIVATE_FUTEX
	55	# define __lll_private_flag(fl, private) \
	56	((fl) \| FUTEX_PRIVATE_FLAG)
	57	# else
	58	# define __lll_private_flag(fl, private) \
	59	((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex))
	60	# endif
	61	#else
	62	# ifdef __ASSUME_PRIVATE_FUTEX
	63	# define __lll_private_flag(fl, private) \
	64	(((fl) \| FUTEX_PRIVATE_FLAG) ^ (private))
	65	# else
	66	# define __lll_private_flag(fl, private) \
	67	(__builtin_constant_p (private) \
	68	? ((private) == 0 \
	69	? ((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex)) \
	70	: (fl)) \
	71	: ((fl) \| (((private) ^ FUTEX_PRIVATE_FLAG) \
	72	& THREAD_GETMEM (THREAD_SELF, header.private_futex))))
da5ac135	73	# endif
085a4412	74	#endif
a88c9263	75
085a4412 UD	76	#define lll_futex_wait(futex, val, private) \
085a4412 UD	77	lll_futex_timed_wait (futex, val, NULL, private)
a88c9263	78
085a4412	79	#define lll_futex_timed_wait(futex, val, timespec, private) \
a88c9263 UD	80	({ \
a88c9263 UD	81	register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \
085a4412 UD	82	register unsigned long int __r3 asm ("3") \
085a4412 UD	83	= __lll_private_flag (FUTEX_WAIT, private); \
a88c9263 UD	84	register unsigned long int __r4 asm ("4") = (unsigned long int) (val); \
	85	register unsigned long int __r5 asm ("5") = (unsigned long int)(timespec);\
	86	register unsigned long int __result asm ("2"); \
	87	\
	88	__asm __volatile ("svc %b1" \
	89	: "=d" (__result) \
	90	: "i" (SYS_futex), "0" (__r2), "d" (__r3), \
	91	"d" (__r4), "d" (__r5) \
	92	: "cc", "memory" ); \
	93	__result; \
	94	})
	95
	96
085a4412	97	#define lll_futex_wake(futex, nr, private) \
a88c9263 UD	98	({ \
a88c9263 UD	99	register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \
085a4412	100	register unsigned long int __r3 asm ("3") \
5bd8a249	101	= __lll_private_flag (FUTEX_WAKE, private); \
a88c9263 UD	102	register unsigned long int __r4 asm ("4") = (unsigned long int) (nr); \
	103	register unsigned long int __result asm ("2"); \
	104	\
	105	__asm __volatile ("svc %b1" \
	106	: "=d" (__result) \
	107	: "i" (SYS_futex), "0" (__r2), "d" (__r3), "d" (__r4) \
	108	: "cc", "memory" ); \
	109	__result; \
	110	})
	111
	112
e51deae7	113	#define lll_robust_dead(futexv, private) \
f1740bc4 UD	114	do \
	115	{ \
	116	int *__futexp = &(futexv); \
	117	\
	118	atomic_or (__futexp, FUTEX_OWNER_DIED); \
e51deae7	119	lll_futex_wake (__futexp, 1, private); \
f1740bc4 UD	120	} \
	121	while (0)
	122
	123
75fccede	124	/* Returns non-zero if error happened, zero if success. */
5bd8a249	125	#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \
284bdc42	126	({ \
75fccede	127	register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \
5bd8a249 UD	128	register unsigned long int __r3 asm ("3") \
5bd8a249 UD	129	= __lll_private_flag (FUTEX_CMP_REQUEUE, private); \
75fccede UD	130	register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \
	131	register unsigned long int __r5 asm ("5") = (long int) (nr_move); \
	132	register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex); \
539842a4	133	register unsigned long int __r7 asm ("7") = (int) (val); \
b9b8cf03 UD	134	register unsigned long __result asm ("2"); \
	135	\
	136	__asm __volatile ("svc %b1" \
	137	: "=d" (__result) \
	138	: "i" (SYS_futex), "0" (__r2), "d" (__r3), \
	139	"d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7) \
	140	: "cc", "memory" ); \
	141	__result > -4096UL; \
	142	})
	143
	144
	145	/* Returns non-zero if error happened, zero if success. */
5bd8a249	146	#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \
b9b8cf03 UD	147	({ \
b9b8cf03 UD	148	register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \
5bd8a249 UD	149	register unsigned long int __r3 asm ("3") \
5bd8a249 UD	150	= __lll_private_flag (FUTEX_WAKE_OP, private); \
b9b8cf03 UD	151	register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \
	152	register unsigned long int __r5 asm ("5") = (long int) (nr_wake2); \
	153	register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \
	154	register unsigned long int __r7 asm ("7") \
	155	= (int) FUTEX_OP_CLEAR_WAKE_IF_GT_ONE; \
75fccede	156	register unsigned long __result asm ("2"); \
284bdc42 UD	157	\
	158	__asm __volatile ("svc %b1" \
	159	: "=d" (__result) \
	160	: "i" (SYS_futex), "0" (__r2), "d" (__r3), \
34dd1fb3	161	"d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7) \
284bdc42	162	: "cc", "memory" ); \
75fccede	163	__result > -4096UL; \
284bdc42 UD	164	})
	165
	166
a88c9263 UD	167	#define lll_compare_and_swap(futex, oldval, newval, operation) \
	168	do { \
	169	__typeof (futex) __futex = (futex); \
	170	__asm __volatile (" l %1,%0\n" \
	171	"0: " operation "\n" \
	172	" cs %1,%2,%0\n" \
	173	" jl 0b\n" \
	174	"1:" \
	175	: "=Q" (*__futex), "=&d" (oldval), "=&d" (newval) \
f1847a84	176	: "m" (*__futex) : "cc", "memory" ); \
a88c9263 UD	177	} while (0)
	178
	179
	180	static inline int
	181	__attribute__ ((always_inline))
e51deae7	182	__lll_trylock (int *futex)
a88c9263 UD	183	{
	184	unsigned int old;
	185
	186	__asm __volatile ("cs %0,%3,%1"
	187	: "=d" (old), "=Q" (*futex)
f1847a84	188	: "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
a88c9263 UD	189	return old != 0;
a88c9263 UD	190	}
e51deae7	191	#define lll_trylock(futex) __lll_trylock (&(futex))
a88c9263 UD	192
a88c9263 UD	193
2c0b891a UD	194	static inline int
2c0b891a UD	195	__attribute__ ((always_inline))
e51deae7	196	__lll_cond_trylock (int *futex)
2c0b891a UD	197	{
	198	unsigned int old;
	199
	200	__asm __volatile ("cs %0,%3,%1"
	201	: "=d" (old), "=Q" (*futex)
f1847a84	202	: "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
2c0b891a UD	203	return old != 0;
2c0b891a UD	204	}
e51deae7	205	#define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))
2c0b891a UD	206
2c0b891a UD	207
f1740bc4 UD	208	static inline int
f1740bc4 UD	209	__attribute__ ((always_inline))
e51deae7	210	__lll_robust_trylock (int *futex, int id)
f1740bc4 UD	211	{
	212	unsigned int old;
	213
	214	__asm __volatile ("cs %0,%3,%1"
	215	: "=d" (old), "=Q" (*futex)
	216	: "0" (0), "d" (id), "m" (*futex) : "cc", "memory" );
	217	return old != 0;
	218	}
e51deae7 UD	219	#define lll_robust_trylock(futex, id) \
e51deae7 UD	220	__lll_robust_trylock (&(futex), id)
f1740bc4 UD	221
f1740bc4 UD	222
e51deae7 UD	223	extern void __lll_lock_wait_private (int *futex) attribute_hidden;
	224	extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
	225	extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;
a88c9263 UD	226
	227	static inline void
	228	__attribute__ ((always_inline))
e51deae7	229	__lll_lock (int *futex, int private)
a88c9263	230	{
e51deae7 UD	231	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
	232	{
	233	if (__builtin_constant_p (private) && private == LLL_PRIVATE)
	234	__lll_lock_wait_private (futex);
	235	else
	236	__lll_lock_wait (futex, private);
	237	}
a88c9263	238	}
e51deae7	239	#define lll_lock(futex, private) __lll_lock (&(futex), private)
a88c9263	240
f1740bc4 UD	241	static inline int
f1740bc4 UD	242	__attribute__ ((always_inline))
e51deae7	243	__lll_robust_lock (int *futex, int id, int private)
f1740bc4 UD	244	{
f1740bc4 UD	245	int result = 0;
e51deae7 UD	246	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
	247	0))
	248	result = __lll_robust_lock_wait (futex, private);
f1740bc4 UD	249	return result;
f1740bc4 UD	250	}
e51deae7 UD	251	#define lll_robust_lock(futex, id, private) \
e51deae7 UD	252	__lll_robust_lock (&(futex), id, private)
f1740bc4	253
ef276e64 JJ	254	static inline void
ef276e64 JJ	255	__attribute__ ((always_inline))
e51deae7	256	__lll_cond_lock (int *futex, int private)
ef276e64	257	{
e51deae7 UD	258	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 2, 0), 0))
e51deae7 UD	259	__lll_lock_wait (futex, private);
ef276e64	260	}
e51deae7	261	#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
ef276e64	262
e51deae7 UD	263	#define lll_robust_cond_lock(futex, id, private) \
e51deae7 UD	264	__lll_robust_lock (&(futex), (id) \| FUTEX_WAITERS, private)
f1740bc4	265
39358e8b	266	extern int __lll_timedlock_wait
e51deae7	267	(int futex, const struct timespec , int private) attribute_hidden;
f1740bc4	268	extern int __lll_robust_timedlock_wait
e51deae7	269	(int futex, const struct timespec , int private) attribute_hidden;
a88c9263 UD	270
	271	static inline int
	272	__attribute__ ((always_inline))
e51deae7	273	__lll_timedlock (int futex, const struct timespec abstime, int private)
a88c9263	274	{
a88c9263	275	int result = 0;
e51deae7 UD	276	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
e51deae7 UD	277	result = __lll_timedlock_wait (futex, abstime, private);
a88c9263 UD	278	return result;
a88c9263 UD	279	}
e51deae7 UD	280	#define lll_timedlock(futex, abstime, private) \
e51deae7 UD	281	__lll_timedlock (&(futex), abstime, private)
a88c9263	282
f1740bc4 UD	283	static inline int
f1740bc4 UD	284	__attribute__ ((always_inline))
e51deae7 UD	285	__lll_robust_timedlock (int futex, const struct timespec abstime,
e51deae7 UD	286	int id, int private)
f1740bc4 UD	287	{
f1740bc4 UD	288	int result = 0;
e51deae7 UD	289	if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
	290	0))
	291	result = __lll_robust_timedlock_wait (futex, abstime, private);
f1740bc4 UD	292	return result;
f1740bc4 UD	293	}
e51deae7 UD	294	#define lll_robust_timedlock(futex, abstime, id, private) \
e51deae7 UD	295	__lll_robust_timedlock (&(futex), abstime, id, private)
f1740bc4	296
a88c9263	297
702a9414 JJ	298	#define __lll_unlock(futex, private) \
	299	(void) \
	300	({ int __oldval; \
	301	int __newval = 0; \
	302	int *__futexp = (futex); \
	303	\
	304	lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2"); \
	305	if (__builtin_expect (__oldval > 1, 0)) \
	306	lll_futex_wake (__futexp, 1, private); \
	307	})
e51deae7	308	#define lll_unlock(futex, private) __lll_unlock(&(futex), private)
284bdc42	309
b8ba4a27	310
702a9414 JJ	311	#define __lll_robust_unlock(futex, private) \
	312	(void) \
	313	({ int __oldval; \
	314	int __newval = 0; \
	315	int *__futexp = (futex); \
	316	\
	317	lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2"); \
	318	if (__builtin_expect (__oldval & FUTEX_WAITERS, 0)) \
	319	lll_futex_wake (__futexp, 1, private); \
	320	})
e51deae7 UD	321	#define lll_robust_unlock(futex, private) \
e51deae7 UD	322	__lll_robust_unlock(&(futex), private)
a88c9263	323
e51deae7	324	#define lll_islocked(futex) \
a88c9263 UD	325	(futex != 0)
	326
	327
a88c9263	328	/* Initializers for lock. */
39358e8b UD	329	#define LLL_LOCK_INITIALIZER (0)
39358e8b UD	330	#define LLL_LOCK_INITIALIZER_LOCKED (1)
a88c9263	331
a88c9263 UD	332	/* The kernel notifies a process with uses CLONE_CLEARTID via futex
	333	wakeup when the clone terminates. The memory location contains the
	334	thread ID while the clone is running and is reset to zero
	335	afterwards. */
702a9414 JJ	336	#define __lll_wait_tid(ptid) \
	337	do \
	338	{ \
	339	int __tid; \
	340	\
	341	while ((__tid = *ptid) != 0) \
	342	lll_futex_wait (ptid, __tid, LLL_SHARED); \
	343	} \
	344	while (0)
a88c9263 UD	345	#define lll_wait_tid(tid) __lll_wait_tid(&(tid))
a88c9263 UD	346
39358e8b	347	extern int __lll_timedwait_tid (int , const struct timespec )
a88c9263	348	attribute_hidden;
a88c9263	349
39358e8b UD	350	#define lll_timedwait_tid(tid, abstime) \
	351	({ \
	352	int __res = 0; \
	353	if ((tid) != 0) \
	354	__res = __lll_timedwait_tid (&(tid), (abstime)); \
	355	__res; \
	356	})
a88c9263	357
a88c9263	358	#endif /* lowlevellock.h */