sysdeps/unix/sysv/linux/x86_64/lowlevellock.S

   1 /* Copyright (C) 2002-2019 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <sysdep.h>
  20 #include <pthread-errnos.h>
  21 #include <kernel-features.h>
  22 #include <lowlevellock.h>
  23
  24 #include <stap-probe.h>
  25
  26         .text
  27
  28 #define LOAD_PRIVATE_FUTEX_WAIT(reg) \
  29         movl    $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
  30 #define LOAD_PRIVATE_FUTEX_WAKE(reg) \
  31         movl    $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
  32 #define LOAD_FUTEX_WAIT(reg) \
  33         xorl    $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
  34 #define LOAD_FUTEX_WAIT_ABS(reg) \
  35         xorl    $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
  36 #define LOAD_FUTEX_WAKE(reg) \
  37         xorl    $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
  38
  39
  40         .globl  __lll_lock_wait_private
  41         .type   __lll_lock_wait_private,@function
  42         .hidden __lll_lock_wait_private
  43         .align  16
  44 __lll_lock_wait_private:
  45         cfi_startproc
  46         pushq   %r10
  47         cfi_adjust_cfa_offset(8)
  48         pushq   %rdx
  49         cfi_adjust_cfa_offset(8)
  50         cfi_offset(%r10, -16)
  51         cfi_offset(%rdx, -24)
  52         xorq    %r10, %r10      /* No timeout.  */
  53         movl    $2, %edx
  54         LOAD_PRIVATE_FUTEX_WAIT (%esi)
  55
  56         cmpl    %edx, %eax      /* NB:   %edx == 2 */
  57         jne     2f
  58
  59 1:      LIBC_PROBE (lll_lock_wait_private, 1, %rdi)
  60         movl    $SYS_futex, %eax
  61         syscall
  62
  63 2:      movl    %edx, %eax
  64         xchgl   %eax, (%rdi)    /* NB:   lock is implied */
  65
  66         testl   %eax, %eax
  67         jnz     1b
  68
  69         popq    %rdx
  70         cfi_adjust_cfa_offset(-8)
  71         cfi_restore(%rdx)
  72         popq    %r10
  73         cfi_adjust_cfa_offset(-8)
  74         cfi_restore(%r10)
  75         retq
  76         cfi_endproc
  77         .size   __lll_lock_wait_private,.-__lll_lock_wait_private
  78
  79 #if !IS_IN (libc)
  80         .globl  __lll_lock_wait
  81         .type   __lll_lock_wait,@function
  82         .hidden __lll_lock_wait
  83         .align  16
  84 __lll_lock_wait:
  85         cfi_startproc
  86         pushq   %r10
  87         cfi_adjust_cfa_offset(8)
  88         pushq   %rdx
  89         cfi_adjust_cfa_offset(8)
  90         cfi_offset(%r10, -16)
  91         cfi_offset(%rdx, -24)
  92         xorq    %r10, %r10      /* No timeout.  */
  93         movl    $2, %edx
  94         LOAD_FUTEX_WAIT (%esi)
  95
  96         cmpl    %edx, %eax      /* NB:   %edx == 2 */
  97         jne     2f
  98
  99 1:      LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
 100         movl    $SYS_futex, %eax
 101         syscall
 102
 103 2:      movl    %edx, %eax
 104         xchgl   %eax, (%rdi)    /* NB:   lock is implied */
 105
 106         testl   %eax, %eax
 107         jnz     1b
 108
 109         popq    %rdx
 110         cfi_adjust_cfa_offset(-8)
 111         cfi_restore(%rdx)
 112         popq    %r10
 113         cfi_adjust_cfa_offset(-8)
 114         cfi_restore(%r10)
 115         retq
 116         cfi_endproc
 117         .size   __lll_lock_wait,.-__lll_lock_wait
 118
 119         /*      %rdi: futex
 120                 %rsi: flags
 121                 %rdx: timeout
 122                 %eax: futex value
 123         */
 124         .globl  __lll_timedlock_wait
 125         .type   __lll_timedlock_wait,@function
 126         .hidden __lll_timedlock_wait
 127         .align  16
 128 __lll_timedlock_wait:
 129         cfi_startproc
 130         cmpq    $0, (%rdx)
 131         js      5f
 132
 133         pushq   %r9
 134         cfi_adjust_cfa_offset(8)
 135         cfi_rel_offset(%r9, 0)
 136
 137         movq    %rdx, %r10
 138         movl    $0xffffffff, %r9d
 139         LOAD_FUTEX_WAIT_ABS (%esi)
 140
 141         movl    $2, %edx
 142         cmpl    %edx, %eax
 143         jne     2f
 144
 145 1:      movl    $SYS_futex, %eax
 146         movl    $2, %edx
 147         syscall
 148
 149 2:      xchgl   %edx, (%rdi)    /* NB:   lock is implied */
 150
 151         testl   %edx, %edx
 152         jz      3f
 153
 154         cmpl    $-ETIMEDOUT, %eax
 155         je      4f
 156         cmpl    $-EINVAL, %eax
 157         jne     1b
 158 4:      movl    %eax, %edx
 159         negl    %edx
 160
 161 3:      movl    %edx, %eax
 162         popq    %r9
 163         cfi_adjust_cfa_offset(-8)
 164         cfi_restore(%r9)
 165         retq
 166
 167 5:      movl    $ETIMEDOUT, %eax
 168         retq
 169
 170         cfi_endproc
 171         .size   __lll_timedlock_wait,.-__lll_timedlock_wait
 172 #endif
 173
 174
 175         .globl  __lll_unlock_wake_private
 176         .type   __lll_unlock_wake_private,@function
 177         .hidden __lll_unlock_wake_private
 178         .align  16
 179 __lll_unlock_wake_private:
 180         cfi_startproc
 181         pushq   %rsi
 182         cfi_adjust_cfa_offset(8)
 183         pushq   %rdx
 184         cfi_adjust_cfa_offset(8)
 185         cfi_offset(%rsi, -16)
 186         cfi_offset(%rdx, -24)
 187
 188         movl    $0, (%rdi)
 189         LOAD_PRIVATE_FUTEX_WAKE (%esi)
 190         movl    $1, %edx        /* Wake one thread.  */
 191         movl    $SYS_futex, %eax
 192         syscall
 193
 194         popq    %rdx
 195         cfi_adjust_cfa_offset(-8)
 196         cfi_restore(%rdx)
 197         popq    %rsi
 198         cfi_adjust_cfa_offset(-8)
 199         cfi_restore(%rsi)
 200         retq
 201         cfi_endproc
 202         .size   __lll_unlock_wake_private,.-__lll_unlock_wake_private
 203
 204 #if !IS_IN (libc)
 205         .globl  __lll_unlock_wake
 206         .type   __lll_unlock_wake,@function
 207         .hidden __lll_unlock_wake
 208         .align  16
 209 __lll_unlock_wake:
 210         cfi_startproc
 211         pushq   %rsi
 212         cfi_adjust_cfa_offset(8)
 213         pushq   %rdx
 214         cfi_adjust_cfa_offset(8)
 215         cfi_offset(%rsi, -16)
 216         cfi_offset(%rdx, -24)
 217
 218         movl    $0, (%rdi)
 219         LOAD_FUTEX_WAKE (%esi)
 220         movl    $1, %edx        /* Wake one thread.  */
 221         movl    $SYS_futex, %eax
 222         syscall
 223
 224         popq    %rdx
 225         cfi_adjust_cfa_offset(-8)
 226         cfi_restore(%rdx)
 227         popq    %rsi
 228         cfi_adjust_cfa_offset(-8)
 229         cfi_restore(%rsi)
 230         retq
 231         cfi_endproc
 232         .size   __lll_unlock_wake,.-__lll_unlock_wake
 233 #endif