[thirdparty/glibc.git] / sysdeps / unix / sysv / linux / i386 / lowlevellock.h

/* Copyright (C) 2002-2018 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#ifndef _LOWLEVELLOCK_H
#define _LOWLEVELLOCK_H	1

#include <stap-probe.h>

#ifndef __ASSEMBLER__
# include <time.h>
# include <sys/param.h>
# include <bits/pthreadtypes.h>
# include <kernel-features.h>
# include <tcb-offsets.h>

# ifndef LOCK_INSTR
#  ifdef UP
#   define LOCK_INSTR	/* nothing */
#  else
#   define LOCK_INSTR "lock;"
#  endif
# endif
#else
# ifndef LOCK
#  ifdef UP
#   define LOCK
#  else
#   define LOCK lock
#  endif
# endif
#endif

#include <lowlevellock-futex.h>

/* XXX Remove when no assembler code uses futexes anymore.  */
#define SYS_futex		__NR_futex

#ifndef __ASSEMBLER__

/* Initializer for compatibility lock.  */
#define LLL_LOCK_INITIALIZER		(0)
#define LLL_LOCK_INITIALIZER_LOCKED	(1)
#define LLL_LOCK_INITIALIZER_WAITERS	(2)


/* NB: in the lll_trylock macro we simply return the value in %eax
   after the cmpxchg instruction.  In case the operation succeded this
   value is zero.  In case the operation failed, the cmpxchg instruction
   has loaded the current value of the memory work which is guaranteed
   to be nonzero.  */
#if !IS_IN (libc) || defined UP
# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
#else
# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
			   "je 0f\n\t"					      \
			   "lock\n"					      \
			   "0:\tcmpxchgl %2, %1"
#endif

#define lll_trylock(futex) \
  ({ int ret;								      \
     __asm __volatile (__lll_trylock_asm				      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
			 "0" (LLL_LOCK_INITIALIZER),			      \
			 "i" (MULTIPLE_THREADS_OFFSET)			      \
		       : "memory");					      \
     ret; })


#define lll_cond_trylock(futex) \
  ({ int ret;								      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (LLL_LOCK_INITIALIZER_WAITERS),		      \
			 "m" (futex), "0" (LLL_LOCK_INITIALIZER)	      \
		       : "memory");					      \
     ret; })

#if !IS_IN (libc) || defined UP
# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
#else
# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t"			      \
			      "je 0f\n\t"				      \
			      "lock\n"					      \
			      "0:\tcmpxchgl %1, %2\n\t"
#endif

#define lll_lock(futex, private) \
  (void)								      \
    ({ int ignore1, ignore2;						      \
       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
	 __asm __volatile (__lll_lock_asm_start				      \
			   "jz 18f\n\t"				      \
			   "1:\tleal %2, %%ecx\n"			      \
			   "2:\tcall __lll_lock_wait_private\n" 	      \
			   "18:"					      \
			   : "=a" (ignore1), "=c" (ignore2), "=m" (futex)     \
			   : "0" (0), "1" (1), "m" (futex),		      \
			     "i" (MULTIPLE_THREADS_OFFSET)		      \
			   : "memory");					      \
       else								      \
	 {								      \
	   int ignore3;							      \
	   __asm __volatile (__lll_lock_asm_start			      \
			     "jz 18f\n\t"			 	      \
			     "1:\tleal %2, %%edx\n"			      \
			     "0:\tmovl %8, %%ecx\n"			      \
			     "2:\tcall __lll_lock_wait\n"		      \
			     "18:"					      \
			     : "=a" (ignore1), "=c" (ignore2),		      \
			       "=m" (futex), "=&d" (ignore3) 		      \
			     : "1" (1), "m" (futex),			      \
			       "i" (MULTIPLE_THREADS_OFFSET), "0" (0),	      \
			       "g" ((int) (private))			      \
			     : "memory");				      \
	 }								      \
    })


/* Special version of lll_lock which causes the unlock function to
   always wakeup waiters.  */
#define lll_cond_lock(futex, private) \
  (void)								      \
    ({ int ignore1, ignore2, ignore3;					      \
       __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
			 "jz 18f\n\t"					      \
			 "1:\tleal %2, %%edx\n"				      \
			 "0:\tmovl %7, %%ecx\n"				      \
			 "2:\tcall __lll_lock_wait\n"			      \
			 "18:"						      \
			 : "=a" (ignore1), "=c" (ignore2), "=m" (futex),      \
			   "=&d" (ignore3)				      \
			 : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
			 : "memory");					      \
    })


#define lll_timedlock(futex, timeout, private) \
  ({ int result, ignore1, ignore2, ignore3;				      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
		       "jz 18f\n\t"					      \
		       "1:\tleal %3, %%ecx\n"				      \
		       "0:\tmovl %8, %%edx\n"				      \
		       "2:\tcall __lll_timedlock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
			 "=m" (futex), "=S" (ignore3)			      \
		       : "0" (0), "1" (1), "m" (futex), "m" (timeout),	      \
			 "4" ((int) (private))				      \
		       : "memory");					      \
     result; })

extern int __lll_timedlock_elision (int *futex, short *adapt_count,
					 const struct timespec *timeout,
					 int private) attribute_hidden;

#define lll_timedlock_elision(futex, adapt_count, timeout, private)	\
  __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)

#if !IS_IN (libc) || defined UP
# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
#else
# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t"			      \
			  "je 0f\n\t"					      \
			  "lock\n"					      \
			  "0:\tsubl $1,%0\n\t"
#endif

#define lll_unlock(futex, private) \
  (void)								      \
    ({ int ignore;							      \
       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
	 __asm __volatile (__lll_unlock_asm				      \
			   "je 18f\n\t"					      \
			   "1:\tleal %0, %%eax\n"			      \
			   "2:\tcall __lll_unlock_wake_private\n"	      \
			   "18:"					      \
			   : "=m" (futex), "=&a" (ignore)		      \
			   : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET)	      \
			   : "memory");					      \
       else								      \
	 {								      \
	   int ignore2;							      \
	   __asm __volatile (__lll_unlock_asm				      \
			     "je 18f\n\t"				      \
			     "1:\tleal %0, %%eax\n"			      \
			     "0:\tmovl %5, %%ecx\n"			      \
			     "2:\tcall __lll_unlock_wake\n"		      \
			     "18:"					      \
			     : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)  \
			     : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex),    \
			       "g" ((int) (private))			      \
			     : "memory");				      \
	 }								      \
    })


#define lll_islocked(futex) \
  (futex != LLL_LOCK_INITIALIZER)

/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
   wake-up when the clone terminates.  The memory location contains the
   thread ID while the clone is running and is reset to zero by the kernel
   afterwards.  The kernel up to version 3.16.3 does not use the private futex
   operations for futex wake-up when the clone terminates.  */
#define lll_wait_tid(tid) \
  do {					\
    __typeof (tid) __tid;		\
    while ((__tid = (tid)) != 0)	\
      lll_futex_wait (&(tid), __tid, LLL_SHARED);\
  } while (0)

extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
     __attribute__ ((regparm (2))) attribute_hidden;

/* As lll_wait_tid, but with a timeout.  If the timeout occurs then return
   ETIMEDOUT.  If ABSTIME is invalid, return EINVAL.
   XXX Note that this differs from the generic version in that we do the
   error checking here and not in __lll_timedwait_tid.  */
#define lll_timedwait_tid(tid, abstime) \
  ({									      \
    int __result = 0;							      \
    if ((tid) != 0)							      \
      __result = __lll_timedwait_tid (&(tid), (abstime));		      \
    __result; })


extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
  attribute_hidden;

extern int __lll_unlock_elision(int *lock, int private)
  attribute_hidden;

extern int __lll_trylock_elision(int *lock, short *adapt_count)
  attribute_hidden;

#define lll_lock_elision(futex, adapt_count, private) \
  __lll_lock_elision (&(futex), &(adapt_count), private)
#define lll_unlock_elision(futex, adapt_count, private) \
  __lll_unlock_elision (&(futex), private)
#define lll_trylock_elision(futex, adapt_count) \
  __lll_trylock_elision(&(futex), &(adapt_count))

#endif  /* !__ASSEMBLER__ */

#endif	/* lowlevellock.h */
Commit	Line	Data
688903eb	1	/* Copyright (C) 2002-2018 Free Software Foundation, Inc.
76a50749 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
76a50749 UD	18
	19	#ifndef _LOWLEVELLOCK_H
	20	#define _LOWLEVELLOCK_H 1
	21
17557282 RH	22	#include <stap-probe.h>
17557282 RH	23
e51deae7 UD	24	#ifndef __ASSEMBLER__
	25	# include <time.h>
	26	# include <sys/param.h>
	27	# include <bits/pthreadtypes.h>
	28	# include <kernel-features.h>
	29	# include <tcb-offsets.h>
	30
	31	# ifndef LOCK_INSTR
	32	# ifdef UP
	33	# define LOCK_INSTR /* nothing */
	34	# else
	35	# define LOCK_INSTR "lock;"
	36	# endif
	37	# endif
	38	#else
	39	# ifndef LOCK
	40	# ifdef UP
	41	# define LOCK
	42	# else
	43	# define LOCK lock
	44	# endif
76a50749 UD	45	# endif
	46	#endif
	47
a4a43a90 TR	48	#include <lowlevellock-futex.h>
	49
	50	/* XXX Remove when no assembler code uses futexes anymore. */
a9fe4c5a	51	#define SYS_futex __NR_futex
1168be45	52
e51deae7	53	#ifndef __ASSEMBLER__
1168be45	54
76a50749	55	/* Initializer for compatibility lock. */
e51deae7 UD	56	#define LLL_LOCK_INITIALIZER (0)
	57	#define LLL_LOCK_INITIALIZER_LOCKED (1)
	58	#define LLL_LOCK_INITIALIZER_WAITERS (2)
76a50749 UD	59
76a50749 UD	60
e51deae7	61	/* NB: in the lll_trylock macro we simply return the value in %eax
e408880b UD	62	after the cmpxchg instruction. In case the operation succeded this
	63	value is zero. In case the operation failed, the cmpxchg instruction
	64	has loaded the current value of the memory work which is guaranteed
	65	to be nonzero. */
4f41c682	66	#if !IS_IN (libc) \|\| defined UP
e51deae7 UD	67	# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
	68	#else
	69	# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
	70	"je 0f\n\t" \
	71	"lock\n" \
	72	"0:\tcmpxchgl %2, %1"
	73	#endif
	74
	75	#define lll_trylock(futex) \
e408880b	76	({ int ret; \
e51deae7	77	__asm __volatile (__lll_trylock_asm \
76a50749	78	: "=a" (ret), "=m" (futex) \
e51deae7 UD	79	: "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \
	80	"0" (LLL_LOCK_INITIALIZER), \
	81	"i" (MULTIPLE_THREADS_OFFSET) \
7a5cdb30	82	: "memory"); \
76a50749 UD	83	ret; })
76a50749 UD	84
683040c3	85
e51deae7	86	#define lll_cond_trylock(futex) \
2c0b891a UD	87	({ int ret; \
	88	__asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
	89	: "=a" (ret), "=m" (futex) \
e51deae7 UD	90	: "r" (LLL_LOCK_INITIALIZER_WAITERS), \
e51deae7 UD	91	"m" (futex), "0" (LLL_LOCK_INITIALIZER) \
2c0b891a UD	92	: "memory"); \
	93	ret; })
	94
4f41c682	95	#if !IS_IN (libc) \|\| defined UP
e51deae7 UD	96	# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
	97	#else
	98	# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t" \
	99	"je 0f\n\t" \
	100	"lock\n" \
	101	"0:\tcmpxchgl %1, %2\n\t"
	102	#endif
2c0b891a	103
e51deae7 UD	104	#define lll_lock(futex, private) \
	105	(void) \
	106	({ int ignore1, ignore2; \
	107	if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
	108	__asm __volatile (__lll_lock_asm_start \
df5b85da	109	"jz 18f\n\t" \
e51deae7 UD	110	"1:\tleal %2, %%ecx\n" \
e51deae7 UD	111	"2:\tcall __lll_lock_wait_private\n" \
e51deae7 UD	112	"18:" \
	113	: "=a" (ignore1), "=c" (ignore2), "=m" (futex) \
	114	: "0" (0), "1" (1), "m" (futex), \
	115	"i" (MULTIPLE_THREADS_OFFSET) \
	116	: "memory"); \
	117	else \
	118	{ \
	119	int ignore3; \
	120	__asm __volatile (__lll_lock_asm_start \
df5b85da	121	"jz 18f\n\t" \
e51deae7 UD	122	"1:\tleal %2, %%edx\n" \
	123	"0:\tmovl %8, %%ecx\n" \
	124	"2:\tcall __lll_lock_wait\n" \
e51deae7 UD	125	"18:" \
	126	: "=a" (ignore1), "=c" (ignore2), \
	127	"=m" (futex), "=&d" (ignore3) \
	128	: "1" (1), "m" (futex), \
	129	"i" (MULTIPLE_THREADS_OFFSET), "0" (0), \
2458c748	130	"g" ((int) (private)) \
e51deae7 UD	131	: "memory"); \
	132	} \
	133	})
	134
683040c3	135
e51deae7	136	/* Special version of lll_lock which causes the unlock function to
69431c9a	137	always wakeup waiters. */
e51deae7 UD	138	#define lll_cond_lock(futex, private) \
	139	(void) \
	140	({ int ignore1, ignore2, ignore3; \
	141	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
df5b85da	142	"jz 18f\n\t" \
e51deae7 UD	143	"1:\tleal %2, %%edx\n" \
	144	"0:\tmovl %7, %%ecx\n" \
	145	"2:\tcall __lll_lock_wait\n" \
e51deae7 UD	146	"18:" \
	147	: "=a" (ignore1), "=c" (ignore2), "=m" (futex), \
	148	"=&d" (ignore3) \
2458c748	149	: "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
e51deae7 UD	150	: "memory"); \
	151	})
	152
	153
e51deae7 UD	154	#define lll_timedlock(futex, timeout, private) \
e51deae7 UD	155	({ int result, ignore1, ignore2, ignore3; \
71451de2	156	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
df5b85da	157	"jz 18f\n\t" \
cd248c3f	158	"1:\tleal %3, %%ecx\n" \
e51deae7 UD	159	"0:\tmovl %8, %%edx\n" \
e51deae7 UD	160	"2:\tcall __lll_timedlock_wait\n" \
cd248c3f	161	"18:" \
71451de2	162	: "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
e51deae7 UD	163	"=m" (futex), "=S" (ignore3) \
e51deae7 UD	164	: "0" (0), "1" (1), "m" (futex), "m" (timeout), \
2458c748	165	"4" ((int) (private)) \
7a5cdb30	166	: "memory"); \
76a50749 UD	167	result; })
76a50749 UD	168
1cdbe579 AK	169	extern int __lll_timedlock_elision (int futex, short adapt_count,
	170	const struct timespec *timeout,
	171	int private) attribute_hidden;
	172
	173	#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
	174	__lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
76a50749	175
4f41c682	176	#if !IS_IN (libc) \|\| defined UP
e51deae7	177	# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
14e7aece	178	#else
e51deae7 UD	179	# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t" \
	180	"je 0f\n\t" \
	181	"lock\n" \
	182	"0:\tsubl $1,%0\n\t"
14e7aece	183	#endif
76a50749	184
e51deae7 UD	185	#define lll_unlock(futex, private) \
	186	(void) \
	187	({ int ignore; \
	188	if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
	189	__asm __volatile (__lll_unlock_asm \
df5b85da	190	"je 18f\n\t" \
e51deae7 UD	191	"1:\tleal %0, %%eax\n" \
e51deae7 UD	192	"2:\tcall __lll_unlock_wake_private\n" \
e51deae7 UD	193	"18:" \
	194	: "=m" (futex), "=&a" (ignore) \
	195	: "m" (futex), "i" (MULTIPLE_THREADS_OFFSET) \
	196	: "memory"); \
	197	else \
	198	{ \
	199	int ignore2; \
	200	__asm __volatile (__lll_unlock_asm \
df5b85da	201	"je 18f\n\t" \
e51deae7 UD	202	"1:\tleal %0, %%eax\n" \
	203	"0:\tmovl %5, %%ecx\n" \
	204	"2:\tcall __lll_unlock_wake\n" \
e51deae7 UD	205	"18:" \
	206	: "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
	207	: "i" (MULTIPLE_THREADS_OFFSET), "m" (futex), \
2458c748	208	"g" ((int) (private)) \
e51deae7 UD	209	: "memory"); \
	210	} \
	211	})
	212
e51deae7	213
76a50749	214	#define lll_islocked(futex) \
71451de2	215	(futex != LLL_LOCK_INITIALIZER)
76a50749	216
adcdc775	217	/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
a4a43a90 TR	218	wake-up when the clone terminates. The memory location contains the
	219	thread ID while the clone is running and is reset to zero by the kernel
	220	afterwards. The kernel up to version 3.16.3 does not use the private futex
	221	operations for futex wake-up when the clone terminates. */
76a50749	222	#define lll_wait_tid(tid) \
a4a43a90 TR	223	do { \
	224	__typeof (tid) __tid; \
	225	while ((__tid = (tid)) != 0) \
	226	lll_futex_wait (&(tid), __tid, LLL_SHARED);\
76a50749 UD	227	} while (0)
	228
	229	extern int __lll_timedwait_tid (int tid, const struct timespec abstime)
	230	__attribute__ ((regparm (2))) attribute_hidden;
a4a43a90 TR	231
	232	/* As lll_wait_tid, but with a timeout. If the timeout occurs then return
	233	ETIMEDOUT. If ABSTIME is invalid, return EINVAL.
	234	XXX Note that this differs from the generic version in that we do the
	235	error checking here and not in __lll_timedwait_tid. */
76a50749 UD	236	#define lll_timedwait_tid(tid, abstime) \
	237	({ \
	238	int __result = 0; \
a4a43a90	239	if ((tid) != 0) \
4735850f	240	__result = __lll_timedwait_tid (&(tid), (abstime)); \
76a50749 UD	241	__result; })
76a50749 UD	242
a4a43a90	243
1cdbe579 AK	244	extern int __lll_lock_elision (int futex, short adapt_count, int private)
	245	attribute_hidden;
	246
	247	extern int __lll_unlock_elision(int *lock, int private)
	248	attribute_hidden;
	249
	250	extern int __lll_trylock_elision(int lock, short adapt_count)
	251	attribute_hidden;
	252
	253	#define lll_lock_elision(futex, adapt_count, private) \
	254	__lll_lock_elision (&(futex), &(adapt_count), private)
5ef19339	255	#define lll_unlock_elision(futex, adapt_count, private) \
1cdbe579 AK	256	__lll_unlock_elision (&(futex), private)
	257	#define lll_trylock_elision(futex, adapt_count) \
	258	__lll_trylock_elision(&(futex), &(adapt_count))
	259
e51deae7	260	#endif /* !__ASSEMBLER__ */
76a50749 UD	261
76a50749 UD	262	#endif /* lowlevellock.h */