1 /* Copyright (C) 2002-2015 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
20 #include <shlib-compat.h>
21 #include <lowlevellock.h>
22 #include <lowlevelcond.h>
23 #include <pthread-pi-defines.h>
24 #include <pthread-errnos.h>
25 #include <stap-probe.h>
27 #include <kernel-features.h>
33 /* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
34 const struct timespec *abstime) */
35 .globl __pthread_cond_timedwait
36 .type __pthread_cond_timedwait, @function
38 __pthread_cond_timedwait:
42 cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
43 DW.ref.__gcc_personality_v0)
44 cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
46 cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
47 cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
51 cfi_adjust_cfa_offset(8)
52 cfi_rel_offset(%r12, 0)
54 cfi_adjust_cfa_offset(8)
55 cfi_rel_offset(%r13, 0)
57 cfi_adjust_cfa_offset(8)
58 cfi_rel_offset(%r14, 0)
60 cfi_adjust_cfa_offset(8)
61 cfi_rel_offset(%r15, 0)
62 #define FRAME_SIZE (32+8)
63 subq $FRAME_SIZE, %rsp
64 cfi_adjust_cfa_offset(FRAME_SIZE)
67 LIBC_PROBE (cond_timedwait, 3, %rdi, %rsi, %rdx)
69 cmpq $1000000000, 8(%rdx)
76 +--------------------------+
77 rsp + 32 | timeout value |
78 +--------------------------+
79 rsp + 24 | old wake_seq value |
80 +--------------------------+
81 rsp + 16 | mutex pointer |
82 +--------------------------+
83 rsp + 8 | condvar pointer |
84 +--------------------------+
85 rsp + 4 | old broadcast_seq value |
86 +--------------------------+
87 rsp + 0 | old cancellation mode |
88 +--------------------------+
91 LP_OP(cmp) $-1, dep_mutex(%rdi)
93 /* Prepare structure passed to cancellation handler. */
99 mov %RSI_LP, dep_mutex(%rdi)
104 /* Get internal lock. */
109 cmpxchgl %esi, (%rdi)
111 cmpxchgl %esi, cond_lock(%rdi)
115 /* Unlock the mutex. */
116 32: movq 16(%rsp), %rdi
118 callq __pthread_mutex_unlock_usercnt
125 incl cond_futex(%rdi)
126 addl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
128 /* Get and store current wakeup_seq value. */
130 movq wakeup_seq(%rdi), %r9
131 movl broadcast_seq(%rdi), %edx
136 movq $-ETIMEDOUT, %r14
139 38: movl cond_futex(%rdi), %r12d
151 34: callq __pthread_enable_asynccancel
155 movl $FUTEX_WAIT_BITSET, %esi
156 LP_OP(cmp) $-1, dep_mutex(%rdi)
159 mov dep_mutex(%rdi), %R8_LP
160 /* Requeue to a non-robust PI mutex if the PI bit is set and
161 the robust bit is not set. */
162 movl MUTEX_KIND(%r8), %eax
163 andl $(ROBUST_BIT|PI_BIT), %eax
167 movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
169 /* The following only works like this because we only support
170 two clocks, represented using a single bit. */
171 testl $1, cond_nwaiters(%rdi)
172 movl $FUTEX_CLOCK_REALTIME, %edx
176 addq $cond_futex, %rdi
177 movl $SYS_futex, %eax
183 #ifdef __ASSUME_REQUEUE_PI
188 /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns
189 successfully, it has already locked the mutex for us and the
190 pi_flag (%r15b) is set to denote that fact. However, if another
191 thread changed the futex value before we entered the wait, the
192 syscall may return an EAGAIN and the mutex is not locked. We go
193 ahead with a success anyway since later we look at the pi_flag to
194 decide if we got the mutex or not. The sequence numbers then make
195 sure that only one of the threads actually wake up. We retry using
196 normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal
197 and PI futexes don't mix.
199 Note that we don't check for EAGAIN specifically; we assume that the
200 only other error the futex function could return is EAGAIN (barring
201 the ETIMEOUT of course, for the timeout case in futex) since
202 anything else would mean an error in our function. It is too
203 expensive to do that check for every call (which is quite common in
204 case of a large number of threads), so it has been skipped. */
208 subq $cond_futex, %rdi
211 61: movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
212 60: xorb %r15b, %r15b
214 /* The following only works like this because we only support
215 two clocks, represented using a single bit. */
216 testl $1, cond_nwaiters(%rdi)
217 movl $FUTEX_CLOCK_REALTIME, %edx
218 movl $0xffffffff, %r9d
222 addq $cond_futex, %rdi
223 movl $SYS_futex, %eax
228 callq __pthread_disable_asynccancel
237 cmpxchgl %esi, (%rdi)
239 cmpxchgl %esi, cond_lock(%rdi)
243 36: movl broadcast_seq(%rdi), %edx
245 movq woken_seq(%rdi), %rax
247 movq wakeup_seq(%rdi), %r9
258 45: cmpq $-ETIMEDOUT, %r14
261 /* We need to go back to futex_wait. If we're using requeue_pi, then
262 release the mutex we had acquired and go back. */
266 /* Adjust the mutex values first and then unlock it. The unlock
267 should always succeed or else the kernel did not lock the
270 callq __pthread_mutex_cond_lock_adjust
272 callq __pthread_mutex_unlock_usercnt
273 /* Reload cond_var. */
277 99: incq wakeup_seq(%rdi)
278 incl cond_futex(%rdi)
279 movl $ETIMEDOUT, %r14d
286 44: incq woken_seq(%rdi)
288 54: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
290 /* Wake up a thread which wants to destroy the condvar object. */
291 cmpq $0xffffffffffffffff, total_seq(%rdi)
293 movl cond_nwaiters(%rdi), %eax
294 andl $~((1 << nwaiters_shift) - 1), %eax
297 addq $cond_nwaiters, %rdi
298 LP_OP(cmp) $-1, dep_mutex-cond_nwaiters(%rdi)
300 #ifdef __ASSUME_PRIVATE_FUTEX
301 movl $FUTEX_WAKE, %eax
302 movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
306 movl %fs:PRIVATE_FUTEX, %esi
308 orl $FUTEX_WAKE, %esi
310 movl $SYS_futex, %eax
312 subq $cond_nwaiters, %rdi
322 /* If requeue_pi is used the kernel performs the locking of the
324 41: movq 16(%rsp), %rdi
328 callq __pthread_mutex_cond_lock
333 48: addq $FRAME_SIZE, %rsp
334 cfi_adjust_cfa_offset(-FRAME_SIZE)
336 cfi_adjust_cfa_offset(-8)
339 cfi_adjust_cfa_offset(-8)
342 cfi_adjust_cfa_offset(-8)
345 cfi_adjust_cfa_offset(-8)
352 64: callq __pthread_mutex_cond_lock_adjust
356 /* Initial locking failed. */
359 addq $cond_lock, %rdi
361 LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
362 movl $LLL_PRIVATE, %eax
363 movl $LLL_SHARED, %esi
365 callq __lll_lock_wait
368 /* Unlock in loop requires wakeup. */
371 addq $cond_lock, %rdi
373 LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
374 movl $LLL_PRIVATE, %eax
375 movl $LLL_SHARED, %esi
377 callq __lll_unlock_wake
380 /* Locking in loop failed. */
383 addq $cond_lock, %rdi
385 LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
386 movl $LLL_PRIVATE, %eax
387 movl $LLL_SHARED, %esi
389 callq __lll_lock_wait
391 subq $cond_lock, %rdi
395 /* Unlock after loop requires wakeup. */
398 addq $cond_lock, %rdi
400 LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
401 movl $LLL_PRIVATE, %eax
402 movl $LLL_SHARED, %esi
404 callq __lll_unlock_wake
407 /* The initial unlocking of the mutex failed. */
408 46: movq 8(%rsp), %rdi
419 addq $cond_lock, %rdi
421 LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
422 movl $LLL_PRIVATE, %eax
423 movl $LLL_SHARED, %esi
425 callq __lll_unlock_wake
427 47: movq (%rsp), %rax
430 .size __pthread_cond_timedwait, .-__pthread_cond_timedwait
431 versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
436 .type __condvar_cleanup2, @function
441 +--------------------------+
443 +--------------------------+
445 +--------------------------+
447 +--------------------------+
449 +--------------------------+
450 rsp + 16 | mutex pointer |
451 +--------------------------+
452 rsp + 8 | condvar pointer |
453 +--------------------------+
454 rsp + 4 | old broadcast_seq value |
455 +--------------------------+
456 rsp + 0 | old cancellation mode |
457 +--------------------------+
462 /* Get internal lock. */
468 cmpxchgl %esi, (%rdi)
470 cmpxchgl %esi, cond_lock(%rdi)
475 addq $cond_lock, %rdi
477 LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
478 movl $LLL_PRIVATE, %eax
479 movl $LLL_SHARED, %esi
481 callq __lll_lock_wait
483 subq $cond_lock, %rdi
486 1: movl broadcast_seq(%rdi), %edx
490 /* We increment the wakeup_seq counter only if it is lower than
491 total_seq. If this is not the case the thread was woken and
492 then canceled. In this case we ignore the signal. */
493 movq total_seq(%rdi), %rax
494 cmpq wakeup_seq(%rdi), %rax
496 incq wakeup_seq(%rdi)
497 incl cond_futex(%rdi)
498 6: incq woken_seq(%rdi)
500 3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
502 /* Wake up a thread which wants to destroy the condvar object. */
504 cmpq $0xffffffffffffffff, total_seq(%rdi)
506 movl cond_nwaiters(%rdi), %eax
507 andl $~((1 << nwaiters_shift) - 1), %eax
510 LP_OP(cmp) $-1, dep_mutex(%rdi)
511 leaq cond_nwaiters(%rdi), %rdi
513 #ifdef __ASSUME_PRIVATE_FUTEX
514 movl $FUTEX_WAKE, %eax
515 movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
519 movl %fs:PRIVATE_FUTEX, %esi
521 orl $FUTEX_WAKE, %esi
523 movl $SYS_futex, %eax
525 subq $cond_nwaiters, %rdi
536 addq $cond_lock, %rdi
538 LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
539 movl $LLL_PRIVATE, %eax
540 movl $LLL_SHARED, %esi
542 callq __lll_unlock_wake
544 /* Wake up all waiters to make sure no signal gets lost. */
547 addq $cond_futex, %rdi
548 LP_OP(cmp) $-1, dep_mutex-cond_futex(%rdi)
549 movl $0x7fffffff, %edx
550 #ifdef __ASSUME_PRIVATE_FUTEX
551 movl $FUTEX_WAKE, %eax
552 movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
556 movl %fs:PRIVATE_FUTEX, %esi
558 orl $FUTEX_WAKE, %esi
560 movl $SYS_futex, %eax
563 /* Lock the mutex only if we don't own it already. This only happens
564 in case of PI mutexes, if we got cancelled after a successful
565 return of the futex syscall and before disabling async
567 5: movq 16(%rsp), %rdi
568 movl MUTEX_KIND(%rdi), %eax
569 andl $(ROBUST_BIT|PI_BIT), %eax
577 /* We managed to get the lock. Fix it up before returning. */
578 callq __pthread_mutex_cond_lock_adjust
581 7: callq __pthread_mutex_cond_lock
583 8: movq 24(%rsp), %rdi
584 movq FRAME_SIZE(%rsp), %r15
585 movq FRAME_SIZE+8(%rsp), %r14
586 movq FRAME_SIZE+16(%rsp), %r13
587 movq FRAME_SIZE+24(%rsp), %r12
589 call _Unwind_Resume@PLT
593 .size __condvar_cleanup2, .-__condvar_cleanup2
596 .section .gcc_except_table,"a",@progbits
598 .byte DW_EH_PE_omit # @LPStart format
599 .byte DW_EH_PE_omit # @TType format
600 .byte DW_EH_PE_uleb128 # call-site format
601 .uleb128 .Lcstend-.Lcstbegin
603 .uleb128 .LcleanupSTART1-.LSTARTCODE
604 .uleb128 .LcleanupEND1-.LcleanupSTART1
605 .uleb128 __condvar_cleanup2-.LSTARTCODE
607 .uleb128 .LcallUR-.LSTARTCODE
608 .uleb128 .LENDCODE-.LcallUR
615 .hidden DW.ref.__gcc_personality_v0
616 .weak DW.ref.__gcc_personality_v0
617 .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
619 .type DW.ref.__gcc_personality_v0, @object
620 .size DW.ref.__gcc_personality_v0, LP_SIZE
621 DW.ref.__gcc_personality_v0:
622 ASM_ADDR __gcc_personality_v0