sysdeps/unix/sysv/linux/i386/lowlevellock.S

   1 /* Copyright (C) 2002-2018 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <sysdep.h>
  20 #include <pthread-errnos.h>
  21 #include <kernel-features.h>
  22 #include <lowlevellock.h>
  23
  24 #include <stap-probe.h>
  25
  26         .text
  27
  28 #ifdef __ASSUME_PRIVATE_FUTEX
  29 # define LOAD_PRIVATE_FUTEX_WAIT(reg) \
  30         movl    $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
  31 # define LOAD_PRIVATE_FUTEX_WAKE(reg) \
  32         movl    $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
  33 # define LOAD_FUTEX_WAIT(reg) \
  34         xorl    $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
  35 # define LOAD_FUTEX_WAIT_ABS(reg) \
  36         xorl    $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
  37 # define LOAD_FUTEX_WAKE(reg) \
  38         xorl    $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
  39 #else
  40 # if FUTEX_WAIT == 0
  41 #  define LOAD_PRIVATE_FUTEX_WAIT(reg) \
  42         movl    %gs:PRIVATE_FUTEX, reg
  43 # else
  44 #  define LOAD_PRIVATE_FUTEX_WAIT(reg) \
  45         movl    %gs:PRIVATE_FUTEX, reg ; \
  46         orl     $FUTEX_WAIT, reg
  47 # endif
  48 # define LOAD_PRIVATE_FUTEX_WAKE(reg) \
  49         movl    %gs:PRIVATE_FUTEX, reg ; \
  50         orl     $FUTEX_WAKE, reg
  51 # if FUTEX_WAIT == 0
  52 #  define LOAD_FUTEX_WAIT(reg) \
  53         xorl    $FUTEX_PRIVATE_FLAG, reg ; \
  54         andl    %gs:PRIVATE_FUTEX, reg
  55 # else
  56 #  define LOAD_FUTEX_WAIT(reg) \
  57         xorl    $FUTEX_PRIVATE_FLAG, reg ; \
  58         andl    %gs:PRIVATE_FUTEX, reg ; \
  59         orl     $FUTEX_WAIT, reg
  60 # endif
  61 # define LOAD_FUTEX_WAIT_ABS(reg) \
  62         xorl    $FUTEX_PRIVATE_FLAG, reg ; \
  63         andl    %gs:PRIVATE_FUTEX, reg ; \
  64         orl     $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg
  65 # define LOAD_FUTEX_WAKE(reg) \
  66         xorl    $FUTEX_PRIVATE_FLAG, reg ; \
  67         andl    %gs:PRIVATE_FUTEX, reg ; \
  68         orl     $FUTEX_WAKE, reg
  69 #endif
  70
  71         .globl  __lll_lock_wait_private
  72         .type   __lll_lock_wait_private,@function
  73         .hidden __lll_lock_wait_private
  74         .align  16
  75 __lll_lock_wait_private:
  76         cfi_startproc
  77         pushl   %edx
  78         cfi_adjust_cfa_offset(4)
  79         pushl   %ebx
  80         cfi_adjust_cfa_offset(4)
  81         pushl   %esi
  82         cfi_adjust_cfa_offset(4)
  83         cfi_offset(%edx, -8)
  84         cfi_offset(%ebx, -12)
  85         cfi_offset(%esi, -16)
  86
  87         movl    $2, %edx
  88         movl    %ecx, %ebx
  89         xorl    %esi, %esi      /* No timeout.  */
  90         LOAD_PRIVATE_FUTEX_WAIT (%ecx)
  91
  92         cmpl    %edx, %eax      /* NB:   %edx == 2 */
  93         jne 2f
  94
  95 1:      LIBC_PROBE (lll_lock_wait_private, 1, %ebx)
  96         movl    $SYS_futex, %eax
  97         ENTER_KERNEL
  98
  99 2:      movl    %edx, %eax
 100         xchgl   %eax, (%ebx)    /* NB:   lock is implied */
 101
 102         testl   %eax, %eax
 103         jnz     1b
 104
 105         popl    %esi
 106         cfi_adjust_cfa_offset(-4)
 107         cfi_restore(%esi)
 108         popl    %ebx
 109         cfi_adjust_cfa_offset(-4)
 110         cfi_restore(%ebx)
 111         popl    %edx
 112         cfi_adjust_cfa_offset(-4)
 113         cfi_restore(%edx)
 114         ret
 115         cfi_endproc
 116         .size   __lll_lock_wait_private,.-__lll_lock_wait_private
 117
 118 #if !IS_IN (libc)
 119         .globl  __lll_lock_wait
 120         .type   __lll_lock_wait,@function
 121         .hidden __lll_lock_wait
 122         .align  16
 123 __lll_lock_wait:
 124         cfi_startproc
 125         pushl   %edx
 126         cfi_adjust_cfa_offset(4)
 127         pushl   %ebx
 128         cfi_adjust_cfa_offset(4)
 129         pushl   %esi
 130         cfi_adjust_cfa_offset(4)
 131         cfi_offset(%edx, -8)
 132         cfi_offset(%ebx, -12)
 133         cfi_offset(%esi, -16)
 134
 135         movl    %edx, %ebx
 136         movl    $2, %edx
 137         xorl    %esi, %esi      /* No timeout.  */
 138         LOAD_FUTEX_WAIT (%ecx)
 139
 140         cmpl    %edx, %eax      /* NB:   %edx == 2 */
 141         jne 2f
 142
 143 1:      movl    $SYS_futex, %eax
 144         ENTER_KERNEL
 145
 146 2:      movl    %edx, %eax
 147         xchgl   %eax, (%ebx)    /* NB:   lock is implied */
 148
 149         testl   %eax, %eax
 150         jnz     1b
 151
 152         popl    %esi
 153         cfi_adjust_cfa_offset(-4)
 154         cfi_restore(%esi)
 155         popl    %ebx
 156         cfi_adjust_cfa_offset(-4)
 157         cfi_restore(%ebx)
 158         popl    %edx
 159         cfi_adjust_cfa_offset(-4)
 160         cfi_restore(%edx)
 161         ret
 162         cfi_endproc
 163         .size   __lll_lock_wait,.-__lll_lock_wait
 164
 165         /*      %ecx: futex
 166                 %esi: flags
 167                 %edx: timeout
 168                 %eax: futex value
 169         */
 170         .globl  __lll_timedlock_wait
 171         .type   __lll_timedlock_wait,@function
 172         .hidden __lll_timedlock_wait
 173         .align  16
 174 __lll_timedlock_wait:
 175         cfi_startproc
 176         pushl   %ebp
 177         cfi_adjust_cfa_offset(4)
 178         cfi_rel_offset(%ebp, 0)
 179         pushl   %ebx
 180         cfi_adjust_cfa_offset(4)
 181         cfi_rel_offset(%ebx, 0)
 182
 183 # ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 184 #  ifdef PIC
 185         LOAD_PIC_REG (bx)
 186         cmpl    $0, __have_futex_clock_realtime@GOTOFF(%ebx)
 187 #  else
 188         cmpl    $0, __have_futex_clock_realtime
 189 #  endif
 190         je      .Lreltmo
 191 # endif
 192
 193         cmpl    $0, (%edx)
 194         js      8f
 195
 196         movl    %ecx, %ebx
 197         movl    %esi, %ecx
 198         movl    %edx, %esi
 199         movl    $0xffffffff, %ebp
 200         LOAD_FUTEX_WAIT_ABS (%ecx)
 201
 202         movl    $2, %edx
 203         cmpl    %edx, %eax
 204         jne     2f
 205
 206 1:      movl    $SYS_futex, %eax
 207         movl    $2, %edx
 208         ENTER_KERNEL
 209
 210 2:      xchgl   %edx, (%ebx)    /* NB:   lock is implied */
 211
 212         testl   %edx, %edx
 213         jz      3f
 214
 215         cmpl    $-ETIMEDOUT, %eax
 216         je      4f
 217         cmpl    $-EINVAL, %eax
 218         jne     1b
 219 4:      movl    %eax, %edx
 220         negl    %edx
 221
 222 3:      movl    %edx, %eax
 223 7:      popl    %ebx
 224         cfi_adjust_cfa_offset(-4)
 225         cfi_restore(%ebx)
 226         popl    %ebp
 227         cfi_adjust_cfa_offset(-4)
 228         cfi_restore(%ebp)
 229         ret
 230
 231 8:      movl    $ETIMEDOUT, %eax
 232         jmp     7b
 233
 234 # ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 235 .Lreltmo:
 236         /* Check for a valid timeout value.  */
 237         cmpl    $1000000000, 4(%edx)
 238         jae     3f
 239
 240         pushl   %esi
 241         cfi_adjust_cfa_offset(4)
 242         cfi_rel_offset(%esi, 0)
 243         pushl   %edi
 244         cfi_adjust_cfa_offset(4)
 245         cfi_rel_offset(%edi, 0)
 246
 247         /* Stack frame for the timespec and timeval structs.  */
 248         subl    $8, %esp
 249         cfi_adjust_cfa_offset(8)
 250
 251         movl    %ecx, %ebp
 252         movl    %edx, %edi
 253
 254         movl    $2, %edx
 255         xchgl   %edx, (%ebp)
 256
 257         test    %edx, %edx
 258         je      6f
 259
 260 1:
 261         /* Get current time.  */
 262         movl    %esp, %ebx
 263         xorl    %ecx, %ecx
 264         movl    $__NR_gettimeofday, %eax
 265         ENTER_KERNEL
 266
 267         /* Compute relative timeout.  */
 268         movl    4(%esp), %eax
 269         movl    $1000, %edx
 270         mul     %edx            /* Milli seconds to nano seconds.  */
 271         movl    (%edi), %ecx
 272         movl    4(%edi), %edx
 273         subl    (%esp), %ecx
 274         subl    %eax, %edx
 275         jns     4f
 276         addl    $1000000000, %edx
 277         subl    $1, %ecx
 278 4:      testl   %ecx, %ecx
 279         js      2f              /* Time is already up.  */
 280
 281         /* Store relative timeout.  */
 282         movl    %ecx, (%esp)
 283         movl    %edx, 4(%esp)
 284
 285         /* Futex call.  */
 286         movl    %ebp, %ebx
 287         movl    $2, %edx
 288         movl    %esp, %esi
 289         movl    16(%esp), %ecx
 290         LOAD_FUTEX_WAIT (%ecx)
 291         movl    $SYS_futex, %eax
 292         ENTER_KERNEL
 293
 294         /* NB: %edx == 2 */
 295         xchgl   %edx, (%ebp)
 296
 297         testl   %edx, %edx
 298         je      6f
 299
 300         cmpl    $-ETIMEDOUT, %eax
 301         jne     1b
 302 2:      movl    $ETIMEDOUT, %edx
 303
 304 6:      addl    $8, %esp
 305         cfi_adjust_cfa_offset(-8)
 306         popl    %edi
 307         cfi_adjust_cfa_offset(-4)
 308         cfi_restore(%edi)
 309         popl    %esi
 310         cfi_adjust_cfa_offset(-4)
 311         cfi_restore(%esi)
 312 7:      popl    %ebx
 313         cfi_adjust_cfa_offset(-4)
 314         cfi_restore(%ebx)
 315         popl    %ebp
 316         cfi_adjust_cfa_offset(-4)
 317         cfi_restore(%ebp)
 318         movl    %edx, %eax
 319         ret
 320
 321 3:      movl    $EINVAL, %edx
 322         jmp     7b
 323 # endif
 324         cfi_endproc
 325         .size   __lll_timedlock_wait,.-__lll_timedlock_wait
 326 #endif
 327
 328         .globl  __lll_unlock_wake_private
 329         .type   __lll_unlock_wake_private,@function
 330         .hidden __lll_unlock_wake_private
 331         .align  16
 332 __lll_unlock_wake_private:
 333         cfi_startproc
 334         pushl   %ebx
 335         cfi_adjust_cfa_offset(4)
 336         pushl   %ecx
 337         cfi_adjust_cfa_offset(4)
 338         pushl   %edx
 339         cfi_adjust_cfa_offset(4)
 340         cfi_offset(%ebx, -8)
 341         cfi_offset(%ecx, -12)
 342         cfi_offset(%edx, -16)
 343
 344         movl    %eax, %ebx
 345         movl    $0, (%eax)
 346         LOAD_PRIVATE_FUTEX_WAKE (%ecx)
 347         movl    $1, %edx        /* Wake one thread.  */
 348         movl    $SYS_futex, %eax
 349         ENTER_KERNEL
 350
 351         popl    %edx
 352         cfi_adjust_cfa_offset(-4)
 353         cfi_restore(%edx)
 354         popl    %ecx
 355         cfi_adjust_cfa_offset(-4)
 356         cfi_restore(%ecx)
 357         popl    %ebx
 358         cfi_adjust_cfa_offset(-4)
 359         cfi_restore(%ebx)
 360         ret
 361         cfi_endproc
 362         .size   __lll_unlock_wake_private,.-__lll_unlock_wake_private
 363
 364 #if !IS_IN (libc)
 365         .globl  __lll_unlock_wake
 366         .type   __lll_unlock_wake,@function
 367         .hidden __lll_unlock_wake
 368         .align  16
 369 __lll_unlock_wake:
 370         cfi_startproc
 371         pushl   %ebx
 372         cfi_adjust_cfa_offset(4)
 373         pushl   %ecx
 374         cfi_adjust_cfa_offset(4)
 375         pushl   %edx
 376         cfi_adjust_cfa_offset(4)
 377         cfi_offset(%ebx, -8)
 378         cfi_offset(%ecx, -12)
 379         cfi_offset(%edx, -16)
 380
 381         movl    %eax, %ebx
 382         movl    $0, (%eax)
 383         LOAD_FUTEX_WAKE (%ecx)
 384         movl    $1, %edx        /* Wake one thread.  */
 385         movl    $SYS_futex, %eax
 386         ENTER_KERNEL
 387
 388         popl    %edx
 389         cfi_adjust_cfa_offset(-4)
 390         cfi_restore(%edx)
 391         popl    %ecx
 392         cfi_adjust_cfa_offset(-4)
 393         cfi_restore(%ecx)
 394         popl    %ebx
 395         cfi_adjust_cfa_offset(-4)
 396         cfi_restore(%ebx)
 397         ret
 398         cfi_endproc
 399         .size   __lll_unlock_wake,.-__lll_unlock_wake
 400
 401         .globl  __lll_timedwait_tid
 402         .type   __lll_timedwait_tid,@function
 403         .hidden __lll_timedwait_tid
 404         .align  16
 405 __lll_timedwait_tid:
 406         pushl   %edi
 407         pushl   %esi
 408         pushl   %ebx
 409         pushl   %ebp
 410
 411         movl    %eax, %ebp
 412         movl    %edx, %edi
 413         subl    $8, %esp
 414
 415         /* Get current time.  */
 416 2:      movl    %esp, %ebx
 417         xorl    %ecx, %ecx
 418         movl    $__NR_gettimeofday, %eax
 419         ENTER_KERNEL
 420
 421         /* Compute relative timeout.  */
 422         movl    4(%esp), %eax
 423         movl    $1000, %edx
 424         mul     %edx            /* Milli seconds to nano seconds.  */
 425         movl    (%edi), %ecx
 426         movl    4(%edi), %edx
 427         subl    (%esp), %ecx
 428         subl    %eax, %edx
 429         jns     5f
 430         addl    $1000000000, %edx
 431         subl    $1, %ecx
 432 5:      testl   %ecx, %ecx
 433         js      6f              /* Time is already up.  */
 434
 435         movl    %ecx, (%esp)    /* Store relative timeout.  */
 436         movl    %edx, 4(%esp)
 437
 438         movl    (%ebp), %edx
 439         testl   %edx, %edx
 440         jz      4f
 441
 442         movl    %esp, %esi
 443         /* XXX The kernel so far uses global futex for the wakeup at
 444            all times.  */
 445         xorl    %ecx, %ecx      /* movl $FUTEX_WAIT, %ecx */
 446         movl    %ebp, %ebx
 447         movl    $SYS_futex, %eax
 448         ENTER_KERNEL
 449
 450         cmpl    $0, (%ebx)
 451         jne     1f
 452 4:      xorl    %eax, %eax
 453
 454 3:      addl    $8, %esp
 455         popl    %ebp
 456         popl    %ebx
 457         popl    %esi
 458         popl    %edi
 459         ret
 460
 461 1:      cmpl    $-ETIMEDOUT, %eax
 462         jne     2b
 463 6:      movl    $ETIMEDOUT, %eax
 464         jmp     3b
 465         .size   __lll_timedwait_tid,.-__lll_timedwait_tid
 466 #endif