nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S

   1 /* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <sysdep.h>
  21 #include <shlib-compat.h>
  22 #include <lowlevelcond.h>
  23 #include <tcb-offsets.h>
  24
  25 #ifdef UP
  26 # define LOCK
  27 #else
  28 # define LOCK lock
  29 #endif
  30
  31 #define SYS_futex               202
  32 #define FUTEX_WAIT              0
  33 #define FUTEX_WAKE              1
  34
  35
  36         .text
  37
  38         .align  16
  39         .type   __condvar_cleanup, @function
  40         .globl  __condvar_cleanup
  41         .hidden __condvar_cleanup
  42 __condvar_cleanup:
  43         pushq   %r12
  44
  45         /* Get internal lock.  */
  46         movq    %rdi, %r8
  47         movq    8(%rdi), %rdi
  48         movl    $1, %esi
  49         xorl    %eax, %eax
  50         LOCK
  51 #if cond_lock == 0
  52         cmpxchgl %esi, (%rdi)
  53 #else
  54         cmpxchgl %esi, cond_lock(%rdi)
  55 #endif
  56         jz      1f
  57
  58 #if cond_lock != 0
  59         addq    $cond_lock, %rdi
  60 #endif
  61         callq   __lll_mutex_lock_wait
  62 #if cond_lock != 0
  63         subq    $cond_lock, %rdi
  64 #endif
  65
  66 1:      movl    broadcast_seq(%rdi), %edx
  67         cmpl    4(%r8), %edx
  68         jne     3f
  69
  70         incq    wakeup_seq(%rdi)
  71         incq    woken_seq(%rdi)
  72         incl    cond_futex(%rdi)
  73
  74 3:      subl    $(1 << clock_bits), cond_nwaiters(%rdi)
  75
  76         /* Wake up a thread which wants to destroy the condvar object.  */
  77         xorq    %r12, %r12
  78         cmpq    $0xffffffffffffffff, total_seq(%rdi)
  79         jne     4f
  80         movl    cond_nwaiters(%rdi), %eax
  81         andl    $~((1 << clock_bits) - 1), %eax
  82         jne     4f
  83
  84         addq    $cond_nwaiters, %rdi
  85         movq    $SYS_futex, %rax
  86         movq    $FUTEX_WAKE, %rsi
  87         movl    $1, %edx
  88         syscall
  89         subq    $cond_nwaiters, %rdi
  90         movq    $1, %r12
  91
  92 4:      LOCK
  93 #if cond_lock == 0
  94         decl    (%rdi)
  95 #else
  96         decl    cond_lock(%rdi)
  97 #endif
  98         je      2f
  99 #if cond_lock != 0
 100         addq    $cond_lock, %rdi
 101 #endif
 102         callq   __lll_mutex_unlock_wake
 103
 104         /* Wake up all waiters to make sure no signal gets lost.  */
 105 2:      testq   %r12, %r12
 106         jnz     5f
 107         addq    $cond_futex, %rdi
 108         movq    $FUTEX_WAKE, %rsi
 109         movl    $0x7fffffff, %edx
 110         movq    $SYS_futex, %rax
 111         syscall
 112
 113 5:      movq    16(%r8), %rdi
 114         callq   __pthread_mutex_cond_lock
 115
 116         popq    %r12
 117
 118         retq
 119         .size   __condvar_cleanup, .-__condvar_cleanup
 120
 121
 122 /* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)  */
 123         .globl  __pthread_cond_wait
 124         .type   __pthread_cond_wait, @function
 125         .align  16
 126 __pthread_cond_wait:
 127 .LSTARTCODE:
 128         pushq   %r12
 129 .Lpush_r12:
 130 #define FRAME_SIZE 64
 131         subq    $FRAME_SIZE, %rsp
 132 .Lsubq:
 133         /* Stack frame:
 134
 135            rsp + 64
 136                     +--------------------------+
 137            rsp + 32 | cleanup buffer           |
 138                     +--------------------------+
 139            rsp + 24 | old wake_seq value       |
 140                     +--------------------------+
 141            rsp + 16 | mutex pointer            |
 142                     +--------------------------+
 143            rsp +  8 | condvar pointer          |
 144                     +--------------------------+
 145            rsp +  4 | old broadcast_seq value  |
 146                     +--------------------------+
 147            rsp +  0 | old cancellation mode    |
 148                     +--------------------------+
 149         */
 150
 151         cmpq    $-1, dep_mutex(%rdi)
 152
 153                 /* Prepare structure passed to cancellation handler.  */
 154         movq    %rdi, 8(%rsp)
 155         movq    %rsi, 16(%rsp)
 156
 157         je      15f
 158         movq    %rsi, dep_mutex(%rdi)
 159
 160         /* Get internal lock.  */
 161 15:     movl    $1, %esi
 162         xorl    %eax, %eax
 163         LOCK
 164 #if cond_lock == 0
 165         cmpxchgl %esi, (%rdi)
 166 #else
 167         cmpxchgl %esi, cond_lock(%rdi)
 168 #endif
 169         jne     1f
 170
 171         /* Unlock the mutex.  */
 172 2:      movq    16(%rsp), %rdi
 173         xorq    %rsi, %rsi
 174         callq   __pthread_mutex_unlock_usercnt
 175
 176         testl   %eax, %eax
 177         jne     12f
 178
 179         movq    8(%rsp), %rdi
 180         incq    total_seq(%rdi)
 181         incl    cond_futex(%rdi)
 182         addl    $(1 << clock_bits), cond_nwaiters(%rdi)
 183
 184         /* Install cancellation handler.  */
 185 #ifdef PIC
 186         leaq    __condvar_cleanup(%rip), %rsi
 187 #else
 188         leaq    __condvar_cleanup, %rsi
 189 #endif
 190         leaq    32(%rsp), %rdi
 191         movq    %rsp, %rdx
 192         callq   __pthread_cleanup_push
 193
 194         /* Get and store current wakeup_seq value.  */
 195         movq    8(%rsp), %rdi
 196         movq    wakeup_seq(%rdi), %r9
 197         movl    broadcast_seq(%rdi), %edx
 198         movq    %r9, 24(%rsp)
 199         movl    %edx, 4(%rsp)
 200
 201         /* Unlock.  */
 202 8:      movl    cond_futex(%rdi), %r12d
 203         LOCK
 204 #if cond_lock == 0
 205         decl    (%rdi)
 206 #else
 207         decl    cond_lock(%rdi)
 208 #endif
 209         jne     3f
 210
 211 4:      callq   __pthread_enable_asynccancel
 212         movl    %eax, (%rsp)
 213
 214         movq    8(%rsp), %rdi
 215         xorq    %r10, %r10
 216         movq    %r12, %rdx
 217         addq    $cond_futex-cond_lock, %rdi
 218         movq    $SYS_futex, %rax
 219         movq    %r10, %rsi      /* movq $FUTEX_WAIT, %rsi */
 220         syscall
 221
 222         movl    (%rsp), %edi
 223         callq   __pthread_disable_asynccancel
 224
 225         /* Lock.  */
 226         movq    8(%rsp), %rdi
 227         movl    $1, %esi
 228         xorl    %eax, %eax
 229         LOCK
 230 #if cond_lock == 0
 231         cmpxchgl %esi, (%rdi)
 232 #else
 233         cmpxchgl %esi, cond_lock(%rdi)
 234 #endif
 235         jnz     5f
 236
 237 6:      movl    broadcast_seq(%rdi), %edx
 238
 239         movq    woken_seq(%rdi), %rax
 240
 241         movq    wakeup_seq(%rdi), %r9
 242
 243         cmpl    4(%rsp), %edx
 244         jne     16f
 245
 246         cmpq    24(%rsp), %r9
 247         jbe     8b
 248
 249         cmpq    %rax, %r9
 250         jna     8b
 251
 252         incq    woken_seq(%rdi)
 253
 254         /* Unlock */
 255 16:     subl    $(1 << clock_bits), cond_nwaiters(%rdi)
 256
 257         /* Wake up a thread which wants to destroy the condvar object.  */
 258         cmpq    $0xffffffffffffffff, total_seq(%rdi)
 259         jne     17f
 260         movl    cond_nwaiters(%rdi), %eax
 261         andl    $~((1 << clock_bits) - 1), %eax
 262         jne     17f
 263
 264         addq    $cond_nwaiters, %rdi
 265         movq    $SYS_futex, %rax
 266         movq    $FUTEX_WAKE, %rsi
 267         movl    $1, %edx
 268         syscall
 269         subq    $cond_nwaiters, %rdi
 270
 271 17:     LOCK
 272 #if cond_lock == 0
 273         decl    (%rdi)
 274 #else
 275         decl    cond_lock(%rdi)
 276 #endif
 277         jne     10f
 278
 279         /* Remove cancellation handler.  */
 280 11:     movq    32+CLEANUP_PREV(%rsp), %rdx
 281         movq    %rdx, %fs:CLEANUP
 282
 283         movq    16(%rsp), %rdi
 284         callq   __pthread_mutex_cond_lock
 285 14:     addq    $FRAME_SIZE, %rsp
 286 .Laddq:
 287
 288         popq    %r12
 289 .Lpop_r12:
 290
 291         /* We return the result of the mutex_lock operation.  */
 292         retq
 293
 294         /* Initial locking failed.  */
 295 1:
 296 .LSbl1:
 297 #if cond_lock != 0
 298         addq    $cond_lock, %rdi
 299 #endif
 300         callq   __lll_mutex_lock_wait
 301         jmp     2b
 302
 303         /* Unlock in loop requires wakeup.  */
 304 3:
 305 #if cond_lock != 0
 306         addq    $cond_lock, %rdi
 307 #endif
 308         callq   __lll_mutex_unlock_wake
 309         jmp     4b
 310
 311         /* Locking in loop failed.  */
 312 5:
 313 #if cond_lock != 0
 314         addq    $cond_lock, %rdi
 315 #endif
 316         callq   __lll_mutex_lock_wait
 317 #if cond_lock != 0
 318         subq    $cond_lock, %rdi
 319 #endif
 320         jmp     6b
 321
 322         /* Unlock after loop requires wakeup.  */
 323 10:
 324 #if cond_lock != 0
 325         addq    $cond_lock, %rdi
 326 #endif
 327         callq   __lll_mutex_unlock_wake
 328         jmp     11b
 329
 330         /* The initial unlocking of the mutex failed.  */
 331 12:     movq    %rax, %r10
 332         movq    8(%rsp), %rdi
 333         LOCK
 334 #if cond_lock == 0
 335         decl    (%rdi)
 336 #else
 337         decl    cond_lock(%rdi)
 338 #endif
 339         jne     13f
 340
 341 #if cond_lock != 0
 342         addq    $cond_lock, %rdi
 343 #endif
 344         callq   __lll_mutex_unlock_wake
 345
 346 13:     movq    %r10, %rax
 347         jmp     14b
 348 .LENDCODE:
 349         .size   __pthread_cond_wait, .-__pthread_cond_wait
 350 versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
 351                   GLIBC_2_3_2)
 352
 353
 354         .section .eh_frame,"a",@progbits
 355 .LSTARTFRAME:
 356         .long   L(ENDCIE)-L(STARTCIE)           # Length of the CIE.
 357 .LSTARTCIE:
 358         .long   0                               # CIE ID.
 359         .byte   1                               # Version number.
 360 #ifdef SHARED
 361         .string "zR"                            # NUL-terminated augmentation
 362                                                 # string.
 363 #else
 364         .ascii  "\0"                            # NUL-terminated augmentation
 365                                                 # string.
 366 #endif
 367         .uleb128 1                              # Code alignment factor.
 368         .sleb128 -8                             # Data alignment factor.
 369         .byte   16                              # Return address register
 370                                                 # column.
 371 #ifdef SHARED
 372         .uleb128 1                              # Augmentation value length.
 373         .byte   0x1b                            # Encoding: DW_EH_PE_pcrel
 374                                                 # + DW_EH_PE_sdata4.
 375 #endif
 376         .byte 0x0c                              # DW_CFA_def_cfa
 377         .uleb128 7
 378         .uleb128 8
 379         .byte   0x90                            # DW_CFA_offset, column 0x8
 380         .uleb128 1
 381         .align 8
 382 .LENDCIE:
 383
 384         .long   .LENDFDE-.LSTARTFDE             # Length of the FDE.
 385 .LSTARTFDE:
 386         .long   .LSTARTFDE-.LSTARTFRAME         # CIE pointer.
 387 #ifdef SHARED
 388         .long   .LSTARTCODE-.                   # PC-relative start address
 389                                                 # of the code
 390 #else
 391         .long   .LSTARTCODE                     # Start address of the code.
 392 #endif
 393         .long   .LENDCODE-.LSTARTCODE           # Length of the code.
 394 #ifdef SHARED
 395         .uleb128 0                              # No augmentation data.
 396 #endif
 397         .byte   0x40+.Lpush_r12-.LSTARTCODE     # DW_CFA_advance_loc+N
 398         .byte   14                              # DW_CFA_def_cfa_offset
 399         .uleb128 16
 400         .byte   0x8c                            # DW_CFA_offset %r12
 401         .uleb128 2
 402         .byte   0x40+.Lsubq-.Lpush_r12          # DW_CFA_advance_loc+N
 403         .byte   14                              # DW_CFA_def_cfa_offset
 404         .uleb128 16+FRAME_SIZE
 405         .byte   3                               # DW_CFA_advance_loc2
 406         .2byte  .Laddq-.Lsubq
 407         .byte   14                              # DW_CFA_def_cfa_offset
 408         .uleb128 16
 409         .byte   0x40+.Lpop_r12-.Laddq           # DW_CFA_advance_loc+N
 410         .byte   14                              # DW_CFA_def_cfa_offset
 411         .uleb128 8
 412         .byte   0xcc                            # DW_CFA_restore %r12
 413         .byte   0x40+.LSbl1-.Lpop_r12           # DW_CFA_advance_loc+N
 414         .byte   14                              # DW_CFA_def_cfa_offset
 415         .uleb128 80
 416         .byte   0x8c                            # DW_CFA_offset %r12
 417         .uleb128 2
 418         .align  8
 419 .LENDFDE: