[thirdparty/glibc.git] / sysdeps / unix / sysv / linux / i386 / i486 / lowlevellock.S

/* Copyright (C) 2002-2015 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <sysdep.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
#include <lowlevellock.h>

#include <stap-probe.h>

	.text

#ifdef __ASSUME_PRIVATE_FUTEX
# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
	movl	$(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
	movl	$(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
# define LOAD_FUTEX_WAIT(reg) \
	xorl	$(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
# define LOAD_FUTEX_WAIT_ABS(reg) \
	xorl	$(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
# define LOAD_FUTEX_WAKE(reg) \
	xorl	$(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
#else
# if FUTEX_WAIT == 0
#  define LOAD_PRIVATE_FUTEX_WAIT(reg) \
	movl	%gs:PRIVATE_FUTEX, reg
# else
#  define LOAD_PRIVATE_FUTEX_WAIT(reg) \
	movl	%gs:PRIVATE_FUTEX, reg ; \
	orl	$FUTEX_WAIT, reg
# endif
# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
	movl	%gs:PRIVATE_FUTEX, reg ; \
	orl	$FUTEX_WAKE, reg
# if FUTEX_WAIT == 0
#  define LOAD_FUTEX_WAIT(reg) \
	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
	andl	%gs:PRIVATE_FUTEX, reg
# else
#  define LOAD_FUTEX_WAIT(reg) \
	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
	andl	%gs:PRIVATE_FUTEX, reg ; \
	orl	$FUTEX_WAIT, reg
# endif
# define LOAD_FUTEX_WAIT_ABS(reg) \
	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
	andl	%gs:PRIVATE_FUTEX, reg ; \
	orl	$FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg
# define LOAD_FUTEX_WAKE(reg) \
	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
	andl	%gs:PRIVATE_FUTEX, reg ; \
	orl	$FUTEX_WAKE, reg
#endif

	.globl	__lll_lock_wait_private
	.type	__lll_lock_wait_private,@function
	.hidden	__lll_lock_wait_private
	.align	16
__lll_lock_wait_private:
	cfi_startproc
	pushl	%edx
	cfi_adjust_cfa_offset(4)
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	pushl	%esi
	cfi_adjust_cfa_offset(4)
	cfi_offset(%edx, -8)
	cfi_offset(%ebx, -12)
	cfi_offset(%esi, -16)

	movl	$2, %edx
	movl	%ecx, %ebx
	xorl	%esi, %esi	/* No timeout.  */
	LOAD_PRIVATE_FUTEX_WAIT (%ecx)

	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
	jne 2f

1:	LIBC_PROBE (lll_lock_wait_private, 1, %ebx)
	movl	$SYS_futex, %eax
	ENTER_KERNEL

2:	movl	%edx, %eax
	xchgl	%eax, (%ebx)	/* NB:	 lock is implied */

	testl	%eax, %eax
	jnz	1b

	popl	%esi
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%esi)
	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	popl	%edx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edx)
	ret
	cfi_endproc
	.size	__lll_lock_wait_private,.-__lll_lock_wait_private

#if !IS_IN (libc)
	.globl	__lll_lock_wait
	.type	__lll_lock_wait,@function
	.hidden	__lll_lock_wait
	.align	16
__lll_lock_wait:
	cfi_startproc
	pushl	%edx
	cfi_adjust_cfa_offset(4)
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	pushl	%esi
	cfi_adjust_cfa_offset(4)
	cfi_offset(%edx, -8)
	cfi_offset(%ebx, -12)
	cfi_offset(%esi, -16)

	movl	%edx, %ebx
	movl	$2, %edx
	xorl	%esi, %esi	/* No timeout.  */
	LOAD_FUTEX_WAIT (%ecx)

	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
	jne 2f

1:	movl	$SYS_futex, %eax
	ENTER_KERNEL

2:	movl	%edx, %eax
	xchgl	%eax, (%ebx)	/* NB:	 lock is implied */

	testl	%eax, %eax
	jnz	1b

	popl	%esi
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%esi)
	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	popl	%edx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edx)
	ret
	cfi_endproc
	.size	__lll_lock_wait,.-__lll_lock_wait

	/*      %ecx: futex
		%esi: flags
		%edx: timeout
		%eax: futex value
	*/
	.globl	__lll_timedlock_wait
	.type	__lll_timedlock_wait,@function
	.hidden	__lll_timedlock_wait
	.align	16
__lll_timedlock_wait:
	cfi_startproc
	pushl	%ebp
	cfi_adjust_cfa_offset(4)
	cfi_rel_offset(%ebp, 0)
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	cfi_rel_offset(%ebx, 0)

# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
#  ifdef PIC
	LOAD_PIC_REG (bx)
	cmpl	$0, __have_futex_clock_realtime@GOTOFF(%ebx)
#  else
	cmpl	$0, __have_futex_clock_realtime
#  endif
	je	.Lreltmo
# endif

	cmpl	$0, (%edx)
	js	8f

	movl	%ecx, %ebx
	movl	%esi, %ecx
	movl	%edx, %esi
	movl	$0xffffffff, %ebp
	LOAD_FUTEX_WAIT_ABS (%ecx)

	movl	$2, %edx
	cmpl	%edx, %eax
	jne	2f

1:	movl	$SYS_futex, %eax
	movl	$2, %edx
	ENTER_KERNEL

2:	xchgl	%edx, (%ebx)	/* NB:   lock is implied */

	testl	%edx, %edx
	jz	3f

	cmpl	$-ETIMEDOUT, %eax
	je	4f
	cmpl	$-EINVAL, %eax
	jne	1b
4:	movl	%eax, %edx
	negl	%edx

3:	movl	%edx, %eax
7:	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	popl	%ebp
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebp)
	ret

8:	movl	$ETIMEDOUT, %eax
	jmp	7b

# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
.Lreltmo:
	/* Check for a valid timeout value.  */
	cmpl	$1000000000, 4(%edx)
	jae	3f

	pushl	%esi
	cfi_adjust_cfa_offset(4)
	cfi_rel_offset(%esi, 0)
	pushl	%edi
	cfi_adjust_cfa_offset(4)
	cfi_rel_offset(%edi, 0)

	/* Stack frame for the timespec and timeval structs.  */
	subl	$8, %esp
	cfi_adjust_cfa_offset(8)

	movl	%ecx, %ebp
	movl	%edx, %edi

	movl	$2, %edx
	xchgl	%edx, (%ebp)

	test	%edx, %edx
	je	6f

1:
	/* Get current time.  */
	movl	%esp, %ebx
	xorl	%ecx, %ecx
	movl	$__NR_gettimeofday, %eax
	ENTER_KERNEL

	/* Compute relative timeout.  */
	movl	4(%esp), %eax
	movl	$1000, %edx
	mul	%edx		/* Milli seconds to nano seconds.  */
	movl	(%edi), %ecx
	movl	4(%edi), %edx
	subl	(%esp), %ecx
	subl	%eax, %edx
	jns	4f
	addl	$1000000000, %edx
	subl	$1, %ecx
4:	testl	%ecx, %ecx
	js	2f		/* Time is already up.  */

	/* Store relative timeout.  */
	movl	%ecx, (%esp)
	movl	%edx, 4(%esp)

	/* Futex call.  */
	movl	%ebp, %ebx
	movl	$2, %edx
	movl	%esp, %esi
	movl	16(%esp), %ecx
	LOAD_FUTEX_WAIT (%ecx)
	movl	$SYS_futex, %eax
	ENTER_KERNEL

	/* NB: %edx == 2 */
	xchgl	%edx, (%ebp)

	testl	%edx, %edx
	je	6f

	cmpl	$-ETIMEDOUT, %eax
	jne	1b
2:	movl	$ETIMEDOUT, %edx

6:	addl	$8, %esp
	cfi_adjust_cfa_offset(-8)
	popl	%edi
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edi)
	popl	%esi
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%esi)
7:	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	popl	%ebp
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebp)
	movl	%edx, %eax
	ret

3:	movl	$EINVAL, %edx
	jmp	7b
# endif
	cfi_endproc
	.size	__lll_timedlock_wait,.-__lll_timedlock_wait
#endif

	.globl	__lll_unlock_wake_private
	.type	__lll_unlock_wake_private,@function
	.hidden	__lll_unlock_wake_private
	.align	16
__lll_unlock_wake_private:
	cfi_startproc
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	pushl	%ecx
	cfi_adjust_cfa_offset(4)
	pushl	%edx
	cfi_adjust_cfa_offset(4)
	cfi_offset(%ebx, -8)
	cfi_offset(%ecx, -12)
	cfi_offset(%edx, -16)

	movl	%eax, %ebx
	movl	$0, (%eax)
	LOAD_PRIVATE_FUTEX_WAKE (%ecx)
	movl	$1, %edx	/* Wake one thread.  */
	movl	$SYS_futex, %eax
	ENTER_KERNEL

	popl	%edx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edx)
	popl	%ecx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ecx)
	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	ret
	cfi_endproc
	.size	__lll_unlock_wake_private,.-__lll_unlock_wake_private

#if !IS_IN (libc)
	.globl	__lll_unlock_wake
	.type	__lll_unlock_wake,@function
	.hidden	__lll_unlock_wake
	.align	16
__lll_unlock_wake:
	cfi_startproc
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	pushl	%ecx
	cfi_adjust_cfa_offset(4)
	pushl	%edx
	cfi_adjust_cfa_offset(4)
	cfi_offset(%ebx, -8)
	cfi_offset(%ecx, -12)
	cfi_offset(%edx, -16)

	movl	%eax, %ebx
	movl	$0, (%eax)
	LOAD_FUTEX_WAKE (%ecx)
	movl	$1, %edx	/* Wake one thread.  */
	movl	$SYS_futex, %eax
	ENTER_KERNEL

	popl	%edx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edx)
	popl	%ecx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ecx)
	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	ret
	cfi_endproc
	.size	__lll_unlock_wake,.-__lll_unlock_wake

	.globl	__lll_timedwait_tid
	.type	__lll_timedwait_tid,@function
	.hidden	__lll_timedwait_tid
	.align	16
__lll_timedwait_tid:
	pushl	%edi
	pushl	%esi
	pushl	%ebx
	pushl	%ebp

	movl	%eax, %ebp
	movl	%edx, %edi
	subl	$8, %esp

	/* Get current time.  */
2:	movl	%esp, %ebx
	xorl	%ecx, %ecx
	movl	$__NR_gettimeofday, %eax
	ENTER_KERNEL

	/* Compute relative timeout.  */
	movl	4(%esp), %eax
	movl	$1000, %edx
	mul	%edx		/* Milli seconds to nano seconds.  */
	movl	(%edi), %ecx
	movl	4(%edi), %edx
	subl	(%esp), %ecx
	subl	%eax, %edx
	jns	5f
	addl	$1000000000, %edx
	subl	$1, %ecx
5:	testl	%ecx, %ecx
	js	6f		/* Time is already up.  */

	movl	%ecx, (%esp)	/* Store relative timeout.  */
	movl	%edx, 4(%esp)

	movl	(%ebp), %edx
	testl	%edx, %edx
	jz	4f

	movl	%esp, %esi
	/* XXX The kernel so far uses global futex for the wakeup at
	   all times.  */
	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
	movl	%ebp, %ebx
	movl	$SYS_futex, %eax
	ENTER_KERNEL

	cmpl	$0, (%ebx)
	jne	1f
4:	xorl	%eax, %eax

3:	addl	$8, %esp
	popl	%ebp
	popl	%ebx
	popl	%esi
	popl	%edi
	ret

1:	cmpl	$-ETIMEDOUT, %eax
	jne	2b
6:	movl	$ETIMEDOUT, %eax
	jmp	3b
	.size	__lll_timedwait_tid,.-__lll_timedwait_tid
#endif
Commit	Line	Data
b168057a	1	/* Copyright (C) 2002-2015 Free Software Foundation, Inc.
76a50749 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
76a50749 UD	18
76a50749 UD	19	#include <sysdep.h>
326132db	20	#include <pthread-errnos.h>
e51deae7 UD	21	#include <kernel-features.h>
e51deae7 UD	22	#include <lowlevellock.h>
76a50749	23
17557282 RH	24	#include <stap-probe.h>
17557282 RH	25
76a50749 UD	26	.text
76a50749 UD	27
e51deae7 UD	28	#ifdef __ASSUME_PRIVATE_FUTEX
	29	# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
	30	movl $(FUTEX_WAIT \| FUTEX_PRIVATE_FLAG), reg
	31	# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
	32	movl $(FUTEX_WAKE \| FUTEX_PRIVATE_FLAG), reg
	33	# define LOAD_FUTEX_WAIT(reg) \
	34	xorl $(FUTEX_WAIT \| FUTEX_PRIVATE_FLAG), reg
45f77606 UD	35	# define LOAD_FUTEX_WAIT_ABS(reg) \
45f77606 UD	36	xorl $(FUTEX_WAIT_BITSET \| FUTEX_PRIVATE_FLAG \| FUTEX_CLOCK_REALTIME), reg
e51deae7 UD	37	# define LOAD_FUTEX_WAKE(reg) \
	38	xorl $(FUTEX_WAKE \| FUTEX_PRIVATE_FLAG), reg
	39	#else
	40	# if FUTEX_WAIT == 0
	41	# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
	42	movl %gs:PRIVATE_FUTEX, reg
9356d063	43	# else
e51deae7 UD	44	# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
	45	movl %gs:PRIVATE_FUTEX, reg ; \
	46	orl $FUTEX_WAIT, reg
9356d063	47	# endif
e51deae7 UD	48	# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
	49	movl %gs:PRIVATE_FUTEX, reg ; \
	50	orl $FUTEX_WAKE, reg
5a8075b1 UD	51	# if FUTEX_WAIT == 0
5a8075b1 UD	52	# define LOAD_FUTEX_WAIT(reg) \
e51deae7 UD	53	xorl $FUTEX_PRIVATE_FLAG, reg ; \
e51deae7 UD	54	andl %gs:PRIVATE_FUTEX, reg
5a8075b1 UD	55	# else
5a8075b1 UD	56	# define LOAD_FUTEX_WAIT(reg) \
e51deae7 UD	57	xorl $FUTEX_PRIVATE_FLAG, reg ; \
	58	andl %gs:PRIVATE_FUTEX, reg ; \
	59	orl $FUTEX_WAIT, reg
5a8075b1	60	# endif
45f77606 UD	61	# define LOAD_FUTEX_WAIT_ABS(reg) \
	62	xorl $FUTEX_PRIVATE_FLAG, reg ; \
	63	andl %gs:PRIVATE_FUTEX, reg ; \
1828530f	64	orl $FUTEX_WAIT_BITSET \| FUTEX_CLOCK_REALTIME, reg
5a8075b1	65	# define LOAD_FUTEX_WAKE(reg) \
e51deae7 UD	66	xorl $FUTEX_PRIVATE_FLAG, reg ; \
	67	andl %gs:PRIVATE_FUTEX, reg ; \
	68	orl $FUTEX_WAKE, reg
5a8075b1	69	#endif
76a50749	70
e51deae7 UD	71	.globl __lll_lock_wait_private
	72	.type __lll_lock_wait_private,@function
	73	.hidden __lll_lock_wait_private
76a50749	74	.align 16
e51deae7	75	__lll_lock_wait_private:
cd248c3f	76	cfi_startproc
76a50749	77	pushl %edx
cd248c3f	78	cfi_adjust_cfa_offset(4)
c0df57e1	79	pushl %ebx
cd248c3f	80	cfi_adjust_cfa_offset(4)
c0df57e1	81	pushl %esi
cd248c3f UD	82	cfi_adjust_cfa_offset(4)
	83	cfi_offset(%edx, -8)
	84	cfi_offset(%ebx, -12)
	85	cfi_offset(%esi, -16)
76a50749	86
c0df57e1	87	movl $2, %edx
76a50749 UD	88	movl %ecx, %ebx
76a50749 UD	89	xorl %esi, %esi /* No timeout. */
e51deae7	90	LOAD_PRIVATE_FUTEX_WAIT (%ecx)
71451de2	91
a8fd5a02	92	cmpl %edx, %eax /* NB: %edx == 2 */
ebddb424	93	jne 2f
71451de2	94
17557282 RH	95	1: LIBC_PROBE (lll_lock_wait_private, 1, %ebx)
17557282 RH	96	movl $SYS_futex, %eax
097eca29	97	ENTER_KERNEL
76a50749	98
ebddb424	99	2: movl %edx, %eax
a8fd5a02	100	xchgl %eax, (%ebx) /* NB: lock is implied */
76a50749	101
a8fd5a02	102	testl %eax, %eax
ab9a9ff8	103	jnz 1b
76a50749	104
ebddb424	105	popl %esi
cd248c3f UD	106	cfi_adjust_cfa_offset(-4)
cd248c3f UD	107	cfi_restore(%esi)
c0df57e1	108	popl %ebx
cd248c3f UD	109	cfi_adjust_cfa_offset(-4)
cd248c3f UD	110	cfi_restore(%ebx)
c0df57e1	111	popl %edx
cd248c3f UD	112	cfi_adjust_cfa_offset(-4)
cd248c3f UD	113	cfi_restore(%edx)
76a50749	114	ret
cd248c3f	115	cfi_endproc
e51deae7	116	.size __lll_lock_wait_private,.-__lll_lock_wait_private
71451de2	117
4f41c682	118	#if !IS_IN (libc)
e51deae7 UD	119	.globl __lll_lock_wait
	120	.type __lll_lock_wait,@function
	121	.hidden __lll_lock_wait
71451de2	122	.align 16
e51deae7 UD	123	__lll_lock_wait:
	124	cfi_startproc
	125	pushl %edx
	126	cfi_adjust_cfa_offset(4)
	127	pushl %ebx
	128	cfi_adjust_cfa_offset(4)
	129	pushl %esi
	130	cfi_adjust_cfa_offset(4)
	131	cfi_offset(%edx, -8)
	132	cfi_offset(%ebx, -12)
	133	cfi_offset(%esi, -16)
	134
	135	movl %edx, %ebx
	136	movl $2, %edx
	137	xorl %esi, %esi /* No timeout. */
	138	LOAD_FUTEX_WAIT (%ecx)
	139
	140	cmpl %edx, %eax /* NB: %edx == 2 */
	141	jne 2f
	142
	143	1: movl $SYS_futex, %eax
	144	ENTER_KERNEL
	145
	146	2: movl %edx, %eax
	147	xchgl %eax, (%ebx) /* NB: lock is implied */
	148
	149	testl %eax, %eax
	150	jnz 1b
	151
	152	popl %esi
	153	cfi_adjust_cfa_offset(-4)
	154	cfi_restore(%esi)
	155	popl %ebx
	156	cfi_adjust_cfa_offset(-4)
	157	cfi_restore(%ebx)
	158	popl %edx
	159	cfi_adjust_cfa_offset(-4)
	160	cfi_restore(%edx)
	161	ret
	162	cfi_endproc
	163	.size __lll_lock_wait,.-__lll_lock_wait
	164
45f77606 UD	165	/* %ecx: futex
	166	%esi: flags
	167	%edx: timeout
	168	%eax: futex value
	169	*/
e51deae7 UD	170	.globl __lll_timedlock_wait
	171	.type __lll_timedlock_wait,@function
	172	.hidden __lll_timedlock_wait
	173	.align 16
	174	__lll_timedlock_wait:
cd248c3f	175	cfi_startproc
45f77606 UD	176	pushl %ebp
	177	cfi_adjust_cfa_offset(4)
	178	cfi_rel_offset(%ebp, 0)
	179	pushl %ebx
	180	cfi_adjust_cfa_offset(4)
	181	cfi_rel_offset(%ebx, 0)
	182
	183	# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
	184	# ifdef PIC
	185	LOAD_PIC_REG (bx)
	186	cmpl $0, __have_futex_clock_realtime@GOTOFF(%ebx)
	187	# else
	188	cmpl $0, __have_futex_clock_realtime
	189	# endif
	190	je .Lreltmo
	191	# endif
	192
2983d85e AS	193	cmpl $0, (%edx)
	194	js 8f
	195
45f77606 UD	196	movl %ecx, %ebx
	197	movl %esi, %ecx
	198	movl %edx, %esi
	199	movl $0xffffffff, %ebp
	200	LOAD_FUTEX_WAIT_ABS (%ecx)
	201
	202	movl $2, %edx
	203	cmpl %edx, %eax
	204	jne 2f
	205
	206	1: movl $SYS_futex, %eax
	207	movl $2, %edx
	208	ENTER_KERNEL
	209
	210	2: xchgl %edx, (%ebx) /* NB: lock is implied */
	211
	212	testl %edx, %edx
	213	jz 3f
	214
	215	cmpl $-ETIMEDOUT, %eax
	216	je 4f
	217	cmpl $-EINVAL, %eax
	218	jne 1b
	219	4: movl %eax, %edx
	220	negl %edx
	221
	222	3: movl %edx, %eax
	223	7: popl %ebx
	224	cfi_adjust_cfa_offset(-4)
	225	cfi_restore(%ebx)
	226	popl %ebp
	227	cfi_adjust_cfa_offset(-4)
	228	cfi_restore(%ebp)
	229	ret
	230
2983d85e AS	231	8: movl $ETIMEDOUT, %eax
	232	jmp 7b
	233
45f77606 UD	234	# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
45f77606 UD	235	.Lreltmo:
71451de2 UD	236	/* Check for a valid timeout value. */
	237	cmpl $1000000000, 4(%edx)
	238	jae 3f
	239
71451de2	240	pushl %esi
cd248c3f	241	cfi_adjust_cfa_offset(4)
45f77606 UD	242	cfi_rel_offset(%esi, 0)
45f77606 UD	243	pushl %edi
cd248c3f	244	cfi_adjust_cfa_offset(4)
45f77606	245	cfi_rel_offset(%edi, 0)
71451de2 UD	246
	247	/* Stack frame for the timespec and timeval structs. */
	248	subl $8, %esp
cd248c3f	249	cfi_adjust_cfa_offset(8)
71451de2 UD	250
	251	movl %ecx, %ebp
	252	movl %edx, %edi
	253
c012be6f UD	254	movl $2, %edx
	255	xchgl %edx, (%ebp)
	256
	257	test %edx, %edx
	258	je 6f
	259
71451de2 UD	260	1:
	261	/* Get current time. */
	262	movl %esp, %ebx
	263	xorl %ecx, %ecx
e51deae7	264	movl $__NR_gettimeofday, %eax
71451de2 UD	265	ENTER_KERNEL
	266
	267	/* Compute relative timeout. */
	268	movl 4(%esp), %eax
	269	movl $1000, %edx
	270	mul %edx /* Milli seconds to nano seconds. */
	271	movl (%edi), %ecx
	272	movl 4(%edi), %edx
	273	subl (%esp), %ecx
	274	subl %eax, %edx
	275	jns 4f
	276	addl $1000000000, %edx
	277	subl $1, %ecx
	278	4: testl %ecx, %ecx
c012be6f	279	js 2f /* Time is already up. */
71451de2 UD	280
	281	/* Store relative timeout. */
	282	movl %ecx, (%esp)
	283	movl %edx, 4(%esp)
	284
c012be6f	285	/* Futex call. */
71451de2	286	movl %ebp, %ebx
71451de2	287	movl $2, %edx
71451de2	288	movl %esp, %esi
e51deae7	289	movl 16(%esp), %ecx
5a8075b1	290	LOAD_FUTEX_WAIT (%ecx)
71451de2 UD	291	movl $SYS_futex, %eax
71451de2 UD	292	ENTER_KERNEL
71451de2	293
c012be6f UD	294	/* NB: %edx == 2 */
c012be6f UD	295	xchgl %edx, (%ebp)
71451de2	296
c012be6f UD	297	testl %edx, %edx
	298	je 6f
	299
	300	cmpl $-ETIMEDOUT, %eax
	301	jne 1b
	302	2: movl $ETIMEDOUT, %edx
71451de2 UD	303
71451de2 UD	304	6: addl $8, %esp
cd248c3f	305	cfi_adjust_cfa_offset(-8)
45f77606	306	popl %edi
cd248c3f	307	cfi_adjust_cfa_offset(-4)
45f77606	308	cfi_restore(%edi)
71451de2	309	popl %esi
cd248c3f UD	310	cfi_adjust_cfa_offset(-4)
cd248c3f UD	311	cfi_restore(%esi)
45f77606	312	7: popl %ebx
cd248c3f	313	cfi_adjust_cfa_offset(-4)
45f77606 UD	314	cfi_restore(%ebx)
	315	popl %ebp
	316	cfi_adjust_cfa_offset(-4)
	317	cfi_restore(%ebp)
c012be6f	318	movl %edx, %eax
71451de2 UD	319	ret
71451de2 UD	320
45f77606 UD	321	3: movl $EINVAL, %edx
	322	jmp 7b
	323	# endif
cd248c3f	324	cfi_endproc
e51deae7	325	.size __lll_timedlock_wait,.-__lll_timedlock_wait
71451de2	326	#endif
76a50749	327
e51deae7 UD	328	.globl __lll_unlock_wake_private
	329	.type __lll_unlock_wake_private,@function
	330	.hidden __lll_unlock_wake_private
	331	.align 16
	332	__lll_unlock_wake_private:
	333	cfi_startproc
	334	pushl %ebx
	335	cfi_adjust_cfa_offset(4)
	336	pushl %ecx
	337	cfi_adjust_cfa_offset(4)
	338	pushl %edx
	339	cfi_adjust_cfa_offset(4)
	340	cfi_offset(%ebx, -8)
	341	cfi_offset(%ecx, -12)
	342	cfi_offset(%edx, -16)
	343
	344	movl %eax, %ebx
	345	movl $0, (%eax)
	346	LOAD_PRIVATE_FUTEX_WAKE (%ecx)
	347	movl $1, %edx /* Wake one thread. */
	348	movl $SYS_futex, %eax
	349	ENTER_KERNEL
	350
	351	popl %edx
	352	cfi_adjust_cfa_offset(-4)
	353	cfi_restore(%edx)
	354	popl %ecx
	355	cfi_adjust_cfa_offset(-4)
	356	cfi_restore(%ecx)
	357	popl %ebx
	358	cfi_adjust_cfa_offset(-4)
	359	cfi_restore(%ebx)
	360	ret
	361	cfi_endproc
	362	.size __lll_unlock_wake_private,.-__lll_unlock_wake_private
76a50749	363
4f41c682	364	#if !IS_IN (libc)
e51deae7 UD	365	.globl __lll_unlock_wake
	366	.type __lll_unlock_wake,@function
	367	.hidden __lll_unlock_wake
bd8bb78b	368	.align 16
e51deae7	369	__lll_unlock_wake:
cd248c3f	370	cfi_startproc
76a50749	371	pushl %ebx
cd248c3f	372	cfi_adjust_cfa_offset(4)
76a50749	373	pushl %ecx
cd248c3f	374	cfi_adjust_cfa_offset(4)
76a50749	375	pushl %edx
cd248c3f UD	376	cfi_adjust_cfa_offset(4)
	377	cfi_offset(%ebx, -8)
	378	cfi_offset(%ecx, -12)
	379	cfi_offset(%edx, -16)
76a50749 UD	380
76a50749 UD	381	movl %eax, %ebx
71451de2	382	movl $0, (%eax)
5a8075b1	383	LOAD_FUTEX_WAKE (%ecx)
76a50749	384	movl $1, %edx /* Wake one thread. */
76a50749	385	movl $SYS_futex, %eax
097eca29	386	ENTER_KERNEL
76a50749 UD	387
76a50749 UD	388	popl %edx
cd248c3f UD	389	cfi_adjust_cfa_offset(-4)
cd248c3f UD	390	cfi_restore(%edx)
76a50749	391	popl %ecx
cd248c3f UD	392	cfi_adjust_cfa_offset(-4)
cd248c3f UD	393	cfi_restore(%ecx)
76a50749	394	popl %ebx
cd248c3f UD	395	cfi_adjust_cfa_offset(-4)
cd248c3f UD	396	cfi_restore(%ebx)
76a50749	397	ret
cd248c3f	398	cfi_endproc
e51deae7	399	.size __lll_unlock_wake,.-__lll_unlock_wake
76a50749	400
76a50749 UD	401	.globl __lll_timedwait_tid
	402	.type __lll_timedwait_tid,@function
	403	.hidden __lll_timedwait_tid
bd8bb78b	404	.align 16
76a50749 UD	405	__lll_timedwait_tid:
	406	pushl %edi
	407	pushl %esi
	408	pushl %ebx
	409	pushl %ebp
	410
	411	movl %eax, %ebp
	412	movl %edx, %edi
	413	subl $8, %esp
	414
	415	/* Get current time. */
	416	2: movl %esp, %ebx
	417	xorl %ecx, %ecx
e51deae7	418	movl $__NR_gettimeofday, %eax
097eca29	419	ENTER_KERNEL
76a50749 UD	420
	421	/* Compute relative timeout. */
	422	movl 4(%esp), %eax
	423	movl $1000, %edx
	424	mul %edx /* Milli seconds to nano seconds. */
	425	movl (%edi), %ecx
	426	movl 4(%edi), %edx
	427	subl (%esp), %ecx
	428	subl %eax, %edx
	429	jns 5f
	430	addl $1000000000, %edx
ccf1d573	431	subl $1, %ecx
76a50749 UD	432	5: testl %ecx, %ecx
	433	js 6f /* Time is already up. */
	434
	435	movl %ecx, (%esp) /* Store relative timeout. */
	436	movl %edx, 4(%esp)
	437
	438	movl (%ebp), %edx
	439	testl %edx, %edx
	440	jz 4f
	441
	442	movl %esp, %esi
5a8075b1 UD	443	/* XXX The kernel so far uses global futex for the wakeup at
5a8075b1 UD	444	all times. */
76a50749 UD	445	xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */
	446	movl %ebp, %ebx
	447	movl $SYS_futex, %eax
097eca29	448	ENTER_KERNEL
76a50749	449
76a50749 UD	450	cmpl $0, (%ebx)
	451	jne 1f
	452	4: xorl %eax, %eax
	453
	454	3: addl $8, %esp
	455	popl %ebp
	456	popl %ebx
	457	popl %esi
	458	popl %edi
	459	ret
	460
3273832c	461	1: cmpl $-ETIMEDOUT, %eax
76a50749 UD	462	jne 2b
	463	6: movl $ETIMEDOUT, %eax
	464	jmp 3b
	465	.size __lll_timedwait_tid,.-__lll_timedwait_tid
9356d063	466	#endif