[thirdparty/glibc.git] / sysdeps / unix / sysv / linux / i386 / lowlevellock.h

/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#ifndef _LOWLEVELLOCK_H
#define _LOWLEVELLOCK_H	1

#include <stap-probe.h>

#ifndef __ASSEMBLER__
# include <time.h>
# include <sys/param.h>
# include <bits/pthreadtypes.h>
# include <kernel-features.h>
# include <tcb-offsets.h>

# ifndef LOCK_INSTR
#  ifdef UP
#   define LOCK_INSTR	/* nothing */
#  else
#   define LOCK_INSTR "lock;"
#  endif
# endif
#else
# ifndef LOCK
#  ifdef UP
#   define LOCK
#  else
#   define LOCK lock
#  endif
# endif
#endif

#include <lowlevellock-futex.h>

/* XXX Remove when no assembler code uses futexes anymore.  */
#define SYS_futex		__NR_futex

#ifndef __ASSEMBLER__

/* Initializer for compatibility lock.  */
#define LLL_LOCK_INITIALIZER		(0)
#define LLL_LOCK_INITIALIZER_LOCKED	(1)
#define LLL_LOCK_INITIALIZER_WAITERS	(2)


/* NB: in the lll_trylock macro we simply return the value in %eax
   after the cmpxchg instruction.  In case the operation succeded this
   value is zero.  In case the operation failed, the cmpxchg instruction
   has loaded the current value of the memory work which is guaranteed
   to be nonzero.  */
#if !IS_IN (libc) || defined UP
# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
#else
# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
			   "je 0f\n\t"					      \
			   "lock\n"					      \
			   "0:\tcmpxchgl %2, %1"
#endif

#define lll_trylock(futex) \
  ({ int ret;								      \
     __asm __volatile (__lll_trylock_asm				      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
			 "0" (LLL_LOCK_INITIALIZER),			      \
			 "i" (MULTIPLE_THREADS_OFFSET)			      \
		       : "memory");					      \
     ret; })


#define lll_cond_trylock(futex) \
  ({ int ret;								      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (LLL_LOCK_INITIALIZER_WAITERS),		      \
			 "m" (futex), "0" (LLL_LOCK_INITIALIZER)	      \
		       : "memory");					      \
     ret; })

#if !IS_IN (libc) || defined UP
# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
#else
# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t"			      \
			      "je 0f\n\t"				      \
			      "lock\n"					      \
			      "0:\tcmpxchgl %1, %2\n\t"
#endif

#define lll_lock(futex, private) \
  (void)								      \
    ({ int ignore1, ignore2;						      \
       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
	 __asm __volatile (__lll_lock_asm_start				      \
			   "jz 18f\n\t"				      \
			   "1:\tleal %2, %%ecx\n"			      \
			   "2:\tcall __lll_lock_wait_private\n" 	      \
			   "18:"					      \
			   : "=a" (ignore1), "=c" (ignore2), "=m" (futex)     \
			   : "0" (0), "1" (1), "m" (futex),		      \
			     "i" (MULTIPLE_THREADS_OFFSET)		      \
			   : "memory");					      \
       else								      \
	 {								      \
	   int ignore3;							      \
	   __asm __volatile (__lll_lock_asm_start			      \
			     "jz 18f\n\t"			 	      \
			     "1:\tleal %2, %%edx\n"			      \
			     "0:\tmovl %8, %%ecx\n"			      \
			     "2:\tcall __lll_lock_wait\n"		      \
			     "18:"					      \
			     : "=a" (ignore1), "=c" (ignore2),		      \
			       "=m" (futex), "=&d" (ignore3) 		      \
			     : "1" (1), "m" (futex),			      \
			       "i" (MULTIPLE_THREADS_OFFSET), "0" (0),	      \
			       "g" ((int) (private))			      \
			     : "memory");				      \
	 }								      \
    })

#define lll_robust_lock(futex, id, private) \
  ({ int result, ignore1, ignore2;					      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"			      \
		       "jz 18f\n\t"					      \
		       "1:\tleal %2, %%edx\n"				      \
		       "0:\tmovl %7, %%ecx\n"				      \
		       "2:\tcall __lll_robust_lock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=m" (futex),	      \
			 "=&d" (ignore2)				      \
		       : "0" (0), "1" (id), "m" (futex), "g" ((int) (private))\
		       : "memory");					      \
     result; })


/* Special version of lll_lock which causes the unlock function to
   always wakeup waiters.  */
#define lll_cond_lock(futex, private) \
  (void)								      \
    ({ int ignore1, ignore2, ignore3;					      \
       __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
			 "jz 18f\n\t"					      \
			 "1:\tleal %2, %%edx\n"				      \
			 "0:\tmovl %7, %%ecx\n"				      \
			 "2:\tcall __lll_lock_wait\n"			      \
			 "18:"						      \
			 : "=a" (ignore1), "=c" (ignore2), "=m" (futex),      \
			   "=&d" (ignore3)				      \
			 : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
			 : "memory");					      \
    })


#define lll_robust_cond_lock(futex, id, private) \
  ({ int result, ignore1, ignore2;					      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"			      \
		       "jz 18f\n\t"					      \
		       "1:\tleal %2, %%edx\n"				      \
		       "0:\tmovl %7, %%ecx\n"				      \
		       "2:\tcall __lll_robust_lock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=m" (futex),	      \
			 "=&d" (ignore2)				      \
		       : "0" (0), "1" (id | FUTEX_WAITERS), "m" (futex),      \
			 "g" ((int) (private))				      \
		       : "memory");					      \
     result; })


#define lll_timedlock(futex, timeout, private) \
  ({ int result, ignore1, ignore2, ignore3;				      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
		       "jz 18f\n\t"					      \
		       "1:\tleal %3, %%ecx\n"				      \
		       "0:\tmovl %8, %%edx\n"				      \
		       "2:\tcall __lll_timedlock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
			 "=m" (futex), "=S" (ignore3)			      \
		       : "0" (0), "1" (1), "m" (futex), "m" (timeout),	      \
			 "4" ((int) (private))				      \
		       : "memory");					      \
     result; })

extern int __lll_timedlock_elision (int *futex, short *adapt_count,
					 const struct timespec *timeout,
					 int private) attribute_hidden;

#define lll_timedlock_elision(futex, adapt_count, timeout, private)	\
  __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)

#define lll_robust_timedlock(futex, timeout, id, private) \
  ({ int result, ignore1, ignore2, ignore3;				      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
		       "jz 18f\n\t"			   		      \
		       "1:\tleal %3, %%ecx\n"				      \
		       "0:\tmovl %8, %%edx\n"				      \
		       "2:\tcall __lll_robust_timedlock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
			 "=m" (futex), "=S" (ignore3)			      \
		       : "0" (0), "1" (id), "m" (futex), "m" (timeout),	      \
			 "4" ((int) (private))				      \
		       : "memory");					      \
     result; })

#if !IS_IN (libc) || defined UP
# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
#else
# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t"			      \
			  "je 0f\n\t"					      \
			  "lock\n"					      \
			  "0:\tsubl $1,%0\n\t"
#endif

#define lll_unlock(futex, private) \
  (void)								      \
    ({ int ignore;							      \
       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
	 __asm __volatile (__lll_unlock_asm				      \
			   "je 18f\n\t"					      \
			   "1:\tleal %0, %%eax\n"			      \
			   "2:\tcall __lll_unlock_wake_private\n"	      \
			   "18:"					      \
			   : "=m" (futex), "=&a" (ignore)		      \
			   : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET)	      \
			   : "memory");					      \
       else								      \
	 {								      \
	   int ignore2;							      \
	   __asm __volatile (__lll_unlock_asm				      \
			     "je 18f\n\t"				      \
			     "1:\tleal %0, %%eax\n"			      \
			     "0:\tmovl %5, %%ecx\n"			      \
			     "2:\tcall __lll_unlock_wake\n"		      \
			     "18:"					      \
			     : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)  \
			     : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex),    \
			       "g" ((int) (private))			      \
			     : "memory");				      \
	 }								      \
    })

#define lll_robust_unlock(futex, private) \
  (void)								      \
    ({ int ignore, ignore2;						      \
       __asm __volatile (LOCK_INSTR "andl %3, %0\n\t"			      \
			 "je 18f\n\t"					      \
			 "1:\tleal %0, %%eax\n"				      \
			 "0:\tmovl %5, %%ecx\n"				      \
			 "2:\tcall __lll_unlock_wake\n"			      \
			 "18:"						      \
			 : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)      \
			 : "i" (FUTEX_WAITERS), "m" (futex),		      \
			   "g" ((int) (private))			      \
			 : "memory");					      \
    })


#define lll_islocked(futex) \
  (futex != LLL_LOCK_INITIALIZER)

/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
   wake-up when the clone terminates.  The memory location contains the
   thread ID while the clone is running and is reset to zero by the kernel
   afterwards.  The kernel up to version 3.16.3 does not use the private futex
   operations for futex wake-up when the clone terminates.  */
#define lll_wait_tid(tid) \
  do {					\
    __typeof (tid) __tid;		\
    while ((__tid = (tid)) != 0)	\
      lll_futex_wait (&(tid), __tid, LLL_SHARED);\
  } while (0)

extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
     __attribute__ ((regparm (2))) attribute_hidden;

/* As lll_wait_tid, but with a timeout.  If the timeout occurs then return
   ETIMEDOUT.  If ABSTIME is invalid, return EINVAL.
   XXX Note that this differs from the generic version in that we do the
   error checking here and not in __lll_timedwait_tid.  */
#define lll_timedwait_tid(tid, abstime) \
  ({									      \
    int __result = 0;							      \
    if ((tid) != 0)							      \
      {									      \
	if ((abstime)->tv_nsec < 0 || (abstime)->tv_nsec >= 1000000000)	      \
	  __result = EINVAL;						      \
	else								      \
	  __result = __lll_timedwait_tid (&(tid), (abstime));		      \
      }									      \
    __result; })


extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
  attribute_hidden;

extern int __lll_unlock_elision(int *lock, int private)
  attribute_hidden;

extern int __lll_trylock_elision(int *lock, short *adapt_count)
  attribute_hidden;

#define lll_lock_elision(futex, adapt_count, private) \
  __lll_lock_elision (&(futex), &(adapt_count), private)
#define lll_unlock_elision(futex, adapt_count, private) \
  __lll_unlock_elision (&(futex), private)
#define lll_trylock_elision(futex, adapt_count) \
  __lll_trylock_elision(&(futex), &(adapt_count))

#endif  /* !__ASSEMBLER__ */

#endif	/* lowlevellock.h */
Commit	Line	Data
f7a9f785	1	/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
76a50749 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
76a50749 UD	18
	19	#ifndef _LOWLEVELLOCK_H
	20	#define _LOWLEVELLOCK_H 1
	21
17557282 RH	22	#include <stap-probe.h>
17557282 RH	23
e51deae7 UD	24	#ifndef __ASSEMBLER__
	25	# include <time.h>
	26	# include <sys/param.h>
	27	# include <bits/pthreadtypes.h>
	28	# include <kernel-features.h>
	29	# include <tcb-offsets.h>
	30
	31	# ifndef LOCK_INSTR
	32	# ifdef UP
	33	# define LOCK_INSTR /* nothing */
	34	# else
	35	# define LOCK_INSTR "lock;"
	36	# endif
	37	# endif
	38	#else
	39	# ifndef LOCK
	40	# ifdef UP
	41	# define LOCK
	42	# else
	43	# define LOCK lock
	44	# endif
76a50749 UD	45	# endif
	46	#endif
	47
a4a43a90 TR	48	#include <lowlevellock-futex.h>
	49
	50	/* XXX Remove when no assembler code uses futexes anymore. */
a9fe4c5a	51	#define SYS_futex __NR_futex
1168be45	52
e51deae7	53	#ifndef __ASSEMBLER__
1168be45	54
76a50749	55	/* Initializer for compatibility lock. */
e51deae7 UD	56	#define LLL_LOCK_INITIALIZER (0)
	57	#define LLL_LOCK_INITIALIZER_LOCKED (1)
	58	#define LLL_LOCK_INITIALIZER_WAITERS (2)
76a50749 UD	59
76a50749 UD	60
e51deae7	61	/* NB: in the lll_trylock macro we simply return the value in %eax
e408880b UD	62	after the cmpxchg instruction. In case the operation succeded this
	63	value is zero. In case the operation failed, the cmpxchg instruction
	64	has loaded the current value of the memory work which is guaranteed
	65	to be nonzero. */
4f41c682	66	#if !IS_IN (libc) \|\| defined UP
e51deae7 UD	67	# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
	68	#else
	69	# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
	70	"je 0f\n\t" \
	71	"lock\n" \
	72	"0:\tcmpxchgl %2, %1"
	73	#endif
	74
	75	#define lll_trylock(futex) \
e408880b	76	({ int ret; \
e51deae7	77	__asm __volatile (__lll_trylock_asm \
76a50749	78	: "=a" (ret), "=m" (futex) \
e51deae7 UD	79	: "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \
	80	"0" (LLL_LOCK_INITIALIZER), \
	81	"i" (MULTIPLE_THREADS_OFFSET) \
7a5cdb30	82	: "memory"); \
76a50749 UD	83	ret; })
76a50749 UD	84
683040c3	85
e51deae7	86	#define lll_cond_trylock(futex) \
2c0b891a UD	87	({ int ret; \
	88	__asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
	89	: "=a" (ret), "=m" (futex) \
e51deae7 UD	90	: "r" (LLL_LOCK_INITIALIZER_WAITERS), \
e51deae7 UD	91	"m" (futex), "0" (LLL_LOCK_INITIALIZER) \
2c0b891a UD	92	: "memory"); \
	93	ret; })
	94
4f41c682	95	#if !IS_IN (libc) \|\| defined UP
e51deae7 UD	96	# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
	97	#else
	98	# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t" \
	99	"je 0f\n\t" \
	100	"lock\n" \
	101	"0:\tcmpxchgl %1, %2\n\t"
	102	#endif
2c0b891a	103
e51deae7 UD	104	#define lll_lock(futex, private) \
	105	(void) \
	106	({ int ignore1, ignore2; \
	107	if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
	108	__asm __volatile (__lll_lock_asm_start \
df5b85da	109	"jz 18f\n\t" \
e51deae7 UD	110	"1:\tleal %2, %%ecx\n" \
e51deae7 UD	111	"2:\tcall __lll_lock_wait_private\n" \
e51deae7 UD	112	"18:" \
	113	: "=a" (ignore1), "=c" (ignore2), "=m" (futex) \
	114	: "0" (0), "1" (1), "m" (futex), \
	115	"i" (MULTIPLE_THREADS_OFFSET) \
	116	: "memory"); \
	117	else \
	118	{ \
	119	int ignore3; \
	120	__asm __volatile (__lll_lock_asm_start \
df5b85da	121	"jz 18f\n\t" \
e51deae7 UD	122	"1:\tleal %2, %%edx\n" \
	123	"0:\tmovl %8, %%ecx\n" \
	124	"2:\tcall __lll_lock_wait\n" \
e51deae7 UD	125	"18:" \
	126	: "=a" (ignore1), "=c" (ignore2), \
	127	"=m" (futex), "=&d" (ignore3) \
	128	: "1" (1), "m" (futex), \
	129	"i" (MULTIPLE_THREADS_OFFSET), "0" (0), \
2458c748	130	"g" ((int) (private)) \
e51deae7 UD	131	: "memory"); \
	132	} \
	133	})
	134
	135	#define lll_robust_lock(futex, id, private) \
	136	({ int result, ignore1, ignore2; \
683040c3	137	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
df5b85da	138	"jz 18f\n\t" \
e51deae7 UD	139	"1:\tleal %2, %%edx\n" \
	140	"0:\tmovl %7, %%ecx\n" \
	141	"2:\tcall __lll_robust_lock_wait\n" \
cd248c3f	142	"18:" \
e51deae7 UD	143	: "=a" (result), "=c" (ignore1), "=m" (futex), \
e51deae7 UD	144	"=&d" (ignore2) \
2458c748	145	: "0" (0), "1" (id), "m" (futex), "g" ((int) (private))\
683040c3 UD	146	: "memory"); \
	147	result; })
	148
	149
e51deae7	150	/* Special version of lll_lock which causes the unlock function to
69431c9a	151	always wakeup waiters. */
e51deae7 UD	152	#define lll_cond_lock(futex, private) \
	153	(void) \
	154	({ int ignore1, ignore2, ignore3; \
	155	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
df5b85da	156	"jz 18f\n\t" \
e51deae7 UD	157	"1:\tleal %2, %%edx\n" \
	158	"0:\tmovl %7, %%ecx\n" \
	159	"2:\tcall __lll_lock_wait\n" \
e51deae7 UD	160	"18:" \
	161	: "=a" (ignore1), "=c" (ignore2), "=m" (futex), \
	162	"=&d" (ignore3) \
2458c748	163	: "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
e51deae7 UD	164	: "memory"); \
	165	})
	166
	167
	168	#define lll_robust_cond_lock(futex, id, private) \
	169	({ int result, ignore1, ignore2; \
683040c3	170	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
df5b85da	171	"jz 18f\n\t" \
e51deae7 UD	172	"1:\tleal %2, %%edx\n" \
	173	"0:\tmovl %7, %%ecx\n" \
	174	"2:\tcall __lll_robust_lock_wait\n" \
cd248c3f	175	"18:" \
e51deae7 UD	176	: "=a" (result), "=c" (ignore1), "=m" (futex), \
	177	"=&d" (ignore2) \
	178	: "0" (0), "1" (id \| FUTEX_WAITERS), "m" (futex), \
2458c748	179	"g" ((int) (private)) \
683040c3 UD	180	: "memory"); \
	181	result; })
	182
	183
e51deae7 UD	184	#define lll_timedlock(futex, timeout, private) \
e51deae7 UD	185	({ int result, ignore1, ignore2, ignore3; \
71451de2	186	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
df5b85da	187	"jz 18f\n\t" \
cd248c3f	188	"1:\tleal %3, %%ecx\n" \
e51deae7 UD	189	"0:\tmovl %8, %%edx\n" \
e51deae7 UD	190	"2:\tcall __lll_timedlock_wait\n" \
cd248c3f	191	"18:" \
71451de2	192	: "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
e51deae7 UD	193	"=m" (futex), "=S" (ignore3) \
e51deae7 UD	194	: "0" (0), "1" (1), "m" (futex), "m" (timeout), \
2458c748	195	"4" ((int) (private)) \
7a5cdb30	196	: "memory"); \
76a50749 UD	197	result; })
76a50749 UD	198
1cdbe579 AK	199	extern int __lll_timedlock_elision (int futex, short adapt_count,
	200	const struct timespec *timeout,
	201	int private) attribute_hidden;
	202
	203	#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
	204	__lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
76a50749	205
e51deae7 UD	206	#define lll_robust_timedlock(futex, timeout, id, private) \
e51deae7 UD	207	({ int result, ignore1, ignore2, ignore3; \
683040c3	208	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
df5b85da	209	"jz 18f\n\t" \
cd248c3f	210	"1:\tleal %3, %%ecx\n" \
e51deae7 UD	211	"0:\tmovl %8, %%edx\n" \
e51deae7 UD	212	"2:\tcall __lll_robust_timedlock_wait\n" \
cd248c3f	213	"18:" \
683040c3	214	: "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
e51deae7 UD	215	"=m" (futex), "=S" (ignore3) \
e51deae7 UD	216	: "0" (0), "1" (id), "m" (futex), "m" (timeout), \
2458c748	217	"4" ((int) (private)) \
683040c3 UD	218	: "memory"); \
	219	result; })
	220
4f41c682	221	#if !IS_IN (libc) \|\| defined UP
e51deae7	222	# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
14e7aece	223	#else
e51deae7 UD	224	# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t" \
	225	"je 0f\n\t" \
	226	"lock\n" \
	227	"0:\tsubl $1,%0\n\t"
14e7aece	228	#endif
76a50749	229
e51deae7 UD	230	#define lll_unlock(futex, private) \
	231	(void) \
	232	({ int ignore; \
	233	if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
	234	__asm __volatile (__lll_unlock_asm \
df5b85da	235	"je 18f\n\t" \
e51deae7 UD	236	"1:\tleal %0, %%eax\n" \
e51deae7 UD	237	"2:\tcall __lll_unlock_wake_private\n" \
e51deae7 UD	238	"18:" \
	239	: "=m" (futex), "=&a" (ignore) \
	240	: "m" (futex), "i" (MULTIPLE_THREADS_OFFSET) \
	241	: "memory"); \
	242	else \
	243	{ \
	244	int ignore2; \
	245	__asm __volatile (__lll_unlock_asm \
df5b85da	246	"je 18f\n\t" \
e51deae7 UD	247	"1:\tleal %0, %%eax\n" \
	248	"0:\tmovl %5, %%ecx\n" \
	249	"2:\tcall __lll_unlock_wake\n" \
e51deae7 UD	250	"18:" \
	251	: "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
	252	: "i" (MULTIPLE_THREADS_OFFSET), "m" (futex), \
2458c748	253	"g" ((int) (private)) \
e51deae7 UD	254	: "memory"); \
	255	} \
	256	})
	257
	258	#define lll_robust_unlock(futex, private) \
	259	(void) \
	260	({ int ignore, ignore2; \
	261	__asm __volatile (LOCK_INSTR "andl %3, %0\n\t" \
df5b85da	262	"je 18f\n\t" \
e51deae7 UD	263	"1:\tleal %0, %%eax\n" \
	264	"0:\tmovl %5, %%ecx\n" \
	265	"2:\tcall __lll_unlock_wake\n" \
e51deae7 UD	266	"18:" \
e51deae7 UD	267	: "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
2458c748 AJ	268	: "i" (FUTEX_WAITERS), "m" (futex), \
2458c748 AJ	269	"g" ((int) (private)) \
e51deae7 UD	270	: "memory"); \
	271	})
	272
	273
76a50749	274	#define lll_islocked(futex) \
71451de2	275	(futex != LLL_LOCK_INITIALIZER)
76a50749	276
adcdc775	277	/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
a4a43a90 TR	278	wake-up when the clone terminates. The memory location contains the
	279	thread ID while the clone is running and is reset to zero by the kernel
	280	afterwards. The kernel up to version 3.16.3 does not use the private futex
	281	operations for futex wake-up when the clone terminates. */
76a50749	282	#define lll_wait_tid(tid) \
a4a43a90 TR	283	do { \
	284	__typeof (tid) __tid; \
	285	while ((__tid = (tid)) != 0) \
	286	lll_futex_wait (&(tid), __tid, LLL_SHARED);\
76a50749 UD	287	} while (0)
	288
	289	extern int __lll_timedwait_tid (int tid, const struct timespec abstime)
	290	__attribute__ ((regparm (2))) attribute_hidden;
a4a43a90 TR	291
	292	/* As lll_wait_tid, but with a timeout. If the timeout occurs then return
	293	ETIMEDOUT. If ABSTIME is invalid, return EINVAL.
	294	XXX Note that this differs from the generic version in that we do the
	295	error checking here and not in __lll_timedwait_tid. */
76a50749 UD	296	#define lll_timedwait_tid(tid, abstime) \
	297	({ \
	298	int __result = 0; \
a4a43a90	299	if ((tid) != 0) \
76a50749	300	{ \
a4a43a90	301	if ((abstime)->tv_nsec < 0 \|\| (abstime)->tv_nsec >= 1000000000) \
76a50749 UD	302	__result = EINVAL; \
76a50749 UD	303	else \
a4a43a90	304	__result = __lll_timedwait_tid (&(tid), (abstime)); \
76a50749 UD	305	} \
	306	__result; })
	307
a4a43a90	308
1cdbe579 AK	309	extern int __lll_lock_elision (int futex, short adapt_count, int private)
	310	attribute_hidden;
	311
	312	extern int __lll_unlock_elision(int *lock, int private)
	313	attribute_hidden;
	314
	315	extern int __lll_trylock_elision(int lock, short adapt_count)
	316	attribute_hidden;
	317
	318	#define lll_lock_elision(futex, adapt_count, private) \
	319	__lll_lock_elision (&(futex), &(adapt_count), private)
5ef19339	320	#define lll_unlock_elision(futex, adapt_count, private) \
1cdbe579 AK	321	__lll_unlock_elision (&(futex), private)
	322	#define lll_trylock_elision(futex, adapt_count) \
	323	__lll_trylock_elision(&(futex), &(adapt_count))
	324
e51deae7	325	#endif /* !__ASSEMBLER__ */
76a50749 UD	326
76a50749 UD	327	#endif /* lowlevellock.h */