[thirdparty/glibc.git] / sysdeps / unix / sysv / linux / i386 / lowlevellock.S

/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <sysdep.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
#include <lowlevellock.h>

#include <stap-probe.h>

	.text

#define LOAD_PRIVATE_FUTEX_WAIT(reg) \
	movl	$(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
#define LOAD_PRIVATE_FUTEX_WAKE(reg) \
	movl	$(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
#define LOAD_FUTEX_WAIT(reg) \
	xorl	$(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
#define LOAD_FUTEX_WAIT_ABS(reg) \
	xorl	$(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
#define LOAD_FUTEX_WAKE(reg) \
	xorl	$(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg

	.globl	__lll_lock_wait_private
	.type	__lll_lock_wait_private,@function
	.hidden	__lll_lock_wait_private
	.align	16
__lll_lock_wait_private:
	cfi_startproc
	pushl	%edx
	cfi_adjust_cfa_offset(4)
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	pushl	%esi
	cfi_adjust_cfa_offset(4)
	cfi_offset(%edx, -8)
	cfi_offset(%ebx, -12)
	cfi_offset(%esi, -16)

	movl	$2, %edx
	movl	%ecx, %ebx
	xorl	%esi, %esi	/* No timeout.  */
	LOAD_PRIVATE_FUTEX_WAIT (%ecx)

	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
	jne 2f

1:	LIBC_PROBE (lll_lock_wait_private, 1, %ebx)
	movl	$SYS_futex, %eax
	ENTER_KERNEL

2:	movl	%edx, %eax
	xchgl	%eax, (%ebx)	/* NB:	 lock is implied */

	testl	%eax, %eax
	jnz	1b

	popl	%esi
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%esi)
	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	popl	%edx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edx)
	ret
	cfi_endproc
	.size	__lll_lock_wait_private,.-__lll_lock_wait_private

#if !IS_IN (libc)
	.globl	__lll_lock_wait
	.type	__lll_lock_wait,@function
	.hidden	__lll_lock_wait
	.align	16
__lll_lock_wait:
	cfi_startproc
	pushl	%edx
	cfi_adjust_cfa_offset(4)
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	pushl	%esi
	cfi_adjust_cfa_offset(4)
	cfi_offset(%edx, -8)
	cfi_offset(%ebx, -12)
	cfi_offset(%esi, -16)

	movl	%edx, %ebx
	movl	$2, %edx
	xorl	%esi, %esi	/* No timeout.  */
	LOAD_FUTEX_WAIT (%ecx)

	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
	jne 2f

1:	movl	$SYS_futex, %eax
	ENTER_KERNEL

2:	movl	%edx, %eax
	xchgl	%eax, (%ebx)	/* NB:	 lock is implied */

	testl	%eax, %eax
	jnz	1b

	popl	%esi
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%esi)
	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	popl	%edx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edx)
	ret
	cfi_endproc
	.size	__lll_lock_wait,.-__lll_lock_wait

	/*      %ecx: futex
		%esi: flags
		%edx: timeout
		%eax: futex value
	*/
	.globl	__lll_timedlock_wait
	.type	__lll_timedlock_wait,@function
	.hidden	__lll_timedlock_wait
	.align	16
__lll_timedlock_wait:
	cfi_startproc
	pushl	%ebp
	cfi_adjust_cfa_offset(4)
	cfi_rel_offset(%ebp, 0)
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	cfi_rel_offset(%ebx, 0)

	cmpl	$0, (%edx)
	js	8f

	movl	%ecx, %ebx
	movl	%esi, %ecx
	movl	%edx, %esi
	movl	$0xffffffff, %ebp
	LOAD_FUTEX_WAIT_ABS (%ecx)

	movl	$2, %edx
	cmpl	%edx, %eax
	jne	2f

1:	movl	$SYS_futex, %eax
	movl	$2, %edx
	ENTER_KERNEL

2:	xchgl	%edx, (%ebx)	/* NB:   lock is implied */

	testl	%edx, %edx
	jz	3f

	cmpl	$-ETIMEDOUT, %eax
	je	4f
	cmpl	$-EINVAL, %eax
	jne	1b
4:	movl	%eax, %edx
	negl	%edx

3:	movl	%edx, %eax
7:	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	popl	%ebp
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebp)
	ret

8:	movl	$ETIMEDOUT, %eax
	jmp	7b

	cfi_endproc
	.size	__lll_timedlock_wait,.-__lll_timedlock_wait
#endif

	.globl	__lll_unlock_wake_private
	.type	__lll_unlock_wake_private,@function
	.hidden	__lll_unlock_wake_private
	.align	16
__lll_unlock_wake_private:
	cfi_startproc
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	pushl	%ecx
	cfi_adjust_cfa_offset(4)
	pushl	%edx
	cfi_adjust_cfa_offset(4)
	cfi_offset(%ebx, -8)
	cfi_offset(%ecx, -12)
	cfi_offset(%edx, -16)

	movl	%eax, %ebx
	movl	$0, (%eax)
	LOAD_PRIVATE_FUTEX_WAKE (%ecx)
	movl	$1, %edx	/* Wake one thread.  */
	movl	$SYS_futex, %eax
	ENTER_KERNEL

	popl	%edx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edx)
	popl	%ecx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ecx)
	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	ret
	cfi_endproc
	.size	__lll_unlock_wake_private,.-__lll_unlock_wake_private

#if !IS_IN (libc)
	.globl	__lll_unlock_wake
	.type	__lll_unlock_wake,@function
	.hidden	__lll_unlock_wake
	.align	16
__lll_unlock_wake:
	cfi_startproc
	pushl	%ebx
	cfi_adjust_cfa_offset(4)
	pushl	%ecx
	cfi_adjust_cfa_offset(4)
	pushl	%edx
	cfi_adjust_cfa_offset(4)
	cfi_offset(%ebx, -8)
	cfi_offset(%ecx, -12)
	cfi_offset(%edx, -16)

	movl	%eax, %ebx
	movl	$0, (%eax)
	LOAD_FUTEX_WAKE (%ecx)
	movl	$1, %edx	/* Wake one thread.  */
	movl	$SYS_futex, %eax
	ENTER_KERNEL

	popl	%edx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%edx)
	popl	%ecx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ecx)
	popl	%ebx
	cfi_adjust_cfa_offset(-4)
	cfi_restore(%ebx)
	ret
	cfi_endproc
	.size	__lll_unlock_wake,.-__lll_unlock_wake
#endif
Commit	Line	Data
04277e02	1	/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
76a50749 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
76a50749 UD	18
76a50749 UD	19	#include <sysdep.h>
326132db	20	#include <pthread-errnos.h>
e51deae7 UD	21	#include <kernel-features.h>
e51deae7 UD	22	#include <lowlevellock.h>
76a50749	23
17557282 RH	24	#include <stap-probe.h>
17557282 RH	25
76a50749 UD	26	.text
76a50749 UD	27
0068c085	28	#define LOAD_PRIVATE_FUTEX_WAIT(reg) \
e51deae7	29	movl $(FUTEX_WAIT \| FUTEX_PRIVATE_FLAG), reg
0068c085	30	#define LOAD_PRIVATE_FUTEX_WAKE(reg) \
e51deae7	31	movl $(FUTEX_WAKE \| FUTEX_PRIVATE_FLAG), reg
0068c085	32	#define LOAD_FUTEX_WAIT(reg) \
e51deae7	33	xorl $(FUTEX_WAIT \| FUTEX_PRIVATE_FLAG), reg
0068c085	34	#define LOAD_FUTEX_WAIT_ABS(reg) \
45f77606	35	xorl $(FUTEX_WAIT_BITSET \| FUTEX_PRIVATE_FLAG \| FUTEX_CLOCK_REALTIME), reg
0068c085	36	#define LOAD_FUTEX_WAKE(reg) \
e51deae7	37	xorl $(FUTEX_WAKE \| FUTEX_PRIVATE_FLAG), reg
76a50749	38
e51deae7 UD	39	.globl __lll_lock_wait_private
	40	.type __lll_lock_wait_private,@function
	41	.hidden __lll_lock_wait_private
76a50749	42	.align 16
e51deae7	43	__lll_lock_wait_private:
cd248c3f	44	cfi_startproc
76a50749	45	pushl %edx
cd248c3f	46	cfi_adjust_cfa_offset(4)
c0df57e1	47	pushl %ebx
cd248c3f	48	cfi_adjust_cfa_offset(4)
c0df57e1	49	pushl %esi
cd248c3f UD	50	cfi_adjust_cfa_offset(4)
	51	cfi_offset(%edx, -8)
	52	cfi_offset(%ebx, -12)
	53	cfi_offset(%esi, -16)
76a50749	54
c0df57e1	55	movl $2, %edx
76a50749 UD	56	movl %ecx, %ebx
76a50749 UD	57	xorl %esi, %esi /* No timeout. */
e51deae7	58	LOAD_PRIVATE_FUTEX_WAIT (%ecx)
71451de2	59
a8fd5a02	60	cmpl %edx, %eax /* NB: %edx == 2 */
ebddb424	61	jne 2f
71451de2	62
17557282 RH	63	1: LIBC_PROBE (lll_lock_wait_private, 1, %ebx)
17557282 RH	64	movl $SYS_futex, %eax
097eca29	65	ENTER_KERNEL
76a50749	66
ebddb424	67	2: movl %edx, %eax
a8fd5a02	68	xchgl %eax, (%ebx) /* NB: lock is implied */
76a50749	69
a8fd5a02	70	testl %eax, %eax
ab9a9ff8	71	jnz 1b
76a50749	72
ebddb424	73	popl %esi
cd248c3f UD	74	cfi_adjust_cfa_offset(-4)
cd248c3f UD	75	cfi_restore(%esi)
c0df57e1	76	popl %ebx
cd248c3f UD	77	cfi_adjust_cfa_offset(-4)
cd248c3f UD	78	cfi_restore(%ebx)
c0df57e1	79	popl %edx
cd248c3f UD	80	cfi_adjust_cfa_offset(-4)
cd248c3f UD	81	cfi_restore(%edx)
76a50749	82	ret
cd248c3f	83	cfi_endproc
e51deae7	84	.size __lll_lock_wait_private,.-__lll_lock_wait_private
71451de2	85
4f41c682	86	#if !IS_IN (libc)
e51deae7 UD	87	.globl __lll_lock_wait
	88	.type __lll_lock_wait,@function
	89	.hidden __lll_lock_wait
71451de2	90	.align 16
e51deae7 UD	91	__lll_lock_wait:
	92	cfi_startproc
	93	pushl %edx
	94	cfi_adjust_cfa_offset(4)
	95	pushl %ebx
	96	cfi_adjust_cfa_offset(4)
	97	pushl %esi
	98	cfi_adjust_cfa_offset(4)
	99	cfi_offset(%edx, -8)
	100	cfi_offset(%ebx, -12)
	101	cfi_offset(%esi, -16)
	102
	103	movl %edx, %ebx
	104	movl $2, %edx
	105	xorl %esi, %esi /* No timeout. */
	106	LOAD_FUTEX_WAIT (%ecx)
	107
	108	cmpl %edx, %eax /* NB: %edx == 2 */
	109	jne 2f
	110
	111	1: movl $SYS_futex, %eax
	112	ENTER_KERNEL
	113
	114	2: movl %edx, %eax
	115	xchgl %eax, (%ebx) /* NB: lock is implied */
	116
	117	testl %eax, %eax
	118	jnz 1b
	119
	120	popl %esi
	121	cfi_adjust_cfa_offset(-4)
	122	cfi_restore(%esi)
	123	popl %ebx
	124	cfi_adjust_cfa_offset(-4)
	125	cfi_restore(%ebx)
	126	popl %edx
	127	cfi_adjust_cfa_offset(-4)
	128	cfi_restore(%edx)
	129	ret
	130	cfi_endproc
	131	.size __lll_lock_wait,.-__lll_lock_wait
	132
45f77606 UD	133	/* %ecx: futex
	134	%esi: flags
	135	%edx: timeout
	136	%eax: futex value
	137	*/
e51deae7 UD	138	.globl __lll_timedlock_wait
	139	.type __lll_timedlock_wait,@function
	140	.hidden __lll_timedlock_wait
	141	.align 16
	142	__lll_timedlock_wait:
cd248c3f	143	cfi_startproc
45f77606 UD	144	pushl %ebp
	145	cfi_adjust_cfa_offset(4)
	146	cfi_rel_offset(%ebp, 0)
	147	pushl %ebx
	148	cfi_adjust_cfa_offset(4)
	149	cfi_rel_offset(%ebx, 0)
	150
2983d85e AS	151	cmpl $0, (%edx)
	152	js 8f
	153
45f77606 UD	154	movl %ecx, %ebx
	155	movl %esi, %ecx
	156	movl %edx, %esi
	157	movl $0xffffffff, %ebp
	158	LOAD_FUTEX_WAIT_ABS (%ecx)
	159
	160	movl $2, %edx
	161	cmpl %edx, %eax
	162	jne 2f
	163
	164	1: movl $SYS_futex, %eax
	165	movl $2, %edx
	166	ENTER_KERNEL
	167
	168	2: xchgl %edx, (%ebx) /* NB: lock is implied */
	169
	170	testl %edx, %edx
	171	jz 3f
	172
	173	cmpl $-ETIMEDOUT, %eax
	174	je 4f
	175	cmpl $-EINVAL, %eax
	176	jne 1b
	177	4: movl %eax, %edx
	178	negl %edx
	179
	180	3: movl %edx, %eax
	181	7: popl %ebx
	182	cfi_adjust_cfa_offset(-4)
	183	cfi_restore(%ebx)
	184	popl %ebp
	185	cfi_adjust_cfa_offset(-4)
	186	cfi_restore(%ebp)
	187	ret
	188
2983d85e AS	189	8: movl $ETIMEDOUT, %eax
	190	jmp 7b
	191
cd248c3f	192	cfi_endproc
e51deae7	193	.size __lll_timedlock_wait,.-__lll_timedlock_wait
71451de2	194	#endif
76a50749	195
e51deae7 UD	196	.globl __lll_unlock_wake_private
	197	.type __lll_unlock_wake_private,@function
	198	.hidden __lll_unlock_wake_private
	199	.align 16
	200	__lll_unlock_wake_private:
	201	cfi_startproc
	202	pushl %ebx
	203	cfi_adjust_cfa_offset(4)
	204	pushl %ecx
	205	cfi_adjust_cfa_offset(4)
	206	pushl %edx
	207	cfi_adjust_cfa_offset(4)
	208	cfi_offset(%ebx, -8)
	209	cfi_offset(%ecx, -12)
	210	cfi_offset(%edx, -16)
	211
	212	movl %eax, %ebx
	213	movl $0, (%eax)
	214	LOAD_PRIVATE_FUTEX_WAKE (%ecx)
	215	movl $1, %edx /* Wake one thread. */
	216	movl $SYS_futex, %eax
	217	ENTER_KERNEL
	218
	219	popl %edx
	220	cfi_adjust_cfa_offset(-4)
	221	cfi_restore(%edx)
	222	popl %ecx
	223	cfi_adjust_cfa_offset(-4)
	224	cfi_restore(%ecx)
	225	popl %ebx
	226	cfi_adjust_cfa_offset(-4)
	227	cfi_restore(%ebx)
	228	ret
	229	cfi_endproc
	230	.size __lll_unlock_wake_private,.-__lll_unlock_wake_private
76a50749	231
4f41c682	232	#if !IS_IN (libc)
e51deae7 UD	233	.globl __lll_unlock_wake
	234	.type __lll_unlock_wake,@function
	235	.hidden __lll_unlock_wake
bd8bb78b	236	.align 16
e51deae7	237	__lll_unlock_wake:
cd248c3f	238	cfi_startproc
76a50749	239	pushl %ebx
cd248c3f	240	cfi_adjust_cfa_offset(4)
76a50749	241	pushl %ecx
cd248c3f	242	cfi_adjust_cfa_offset(4)
76a50749	243	pushl %edx
cd248c3f UD	244	cfi_adjust_cfa_offset(4)
	245	cfi_offset(%ebx, -8)
	246	cfi_offset(%ecx, -12)
	247	cfi_offset(%edx, -16)
76a50749 UD	248
76a50749 UD	249	movl %eax, %ebx
71451de2	250	movl $0, (%eax)
5a8075b1	251	LOAD_FUTEX_WAKE (%ecx)
76a50749	252	movl $1, %edx /* Wake one thread. */
76a50749	253	movl $SYS_futex, %eax
097eca29	254	ENTER_KERNEL
76a50749 UD	255
76a50749 UD	256	popl %edx
cd248c3f UD	257	cfi_adjust_cfa_offset(-4)
cd248c3f UD	258	cfi_restore(%edx)
76a50749	259	popl %ecx
cd248c3f UD	260	cfi_adjust_cfa_offset(-4)
cd248c3f UD	261	cfi_restore(%ecx)
76a50749	262	popl %ebx
cd248c3f UD	263	cfi_adjust_cfa_offset(-4)
cd248c3f UD	264	cfi_restore(%ebx)
76a50749	265	ret
cd248c3f	266	cfi_endproc
e51deae7	267	.size __lll_unlock_wake,.-__lll_unlock_wake
9356d063	268	#endif