nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.S

   1 /* Copyright (C) 2003-2014 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3
   4    The GNU C Library is free software; you can redistribute it and/or
   5    modify it under the terms of the GNU Lesser General Public
   6    License as published by the Free Software Foundation; either
   7    version 2.1 of the License, or (at your option) any later version.
   8
   9    The GNU C Library is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12    Lesser General Public License for more details.
  13
  14    You should have received a copy of the GNU Lesser General Public
  15    License along with the GNU C Library; if not, see
  16    <http://www.gnu.org/licenses/>.  */
  17
  18 #include <sysdep.h>
  19 #include <pthread-errnos.h>
  20 #include <kernel-features.h>
  21 #include <lowlevellock.h>
  22 #include "lowlevel-atomic.h"
  23
  24         .text
  25
  26 #ifdef __ASSUME_PRIVATE_FUTEX
  27 # define LOAD_PRIVATE_FUTEX_WAIT(reg,tmp,tmp2) \
  28         mov     #(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg; \
  29         extu.b  reg, reg
  30 # define LOAD_PRIVATE_FUTEX_WAKE(reg,tmp,tmp2) \
  31         mov     #(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg; \
  32         extu.b  reg, reg
  33 # define LOAD_FUTEX_WAIT(reg,tmp,tmp2) \
  34         mov     #(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), tmp; \
  35         extu.b  tmp, tmp; \
  36         xor     tmp, reg
  37 # define LOAD_FUTEX_WAIT_ABS(reg,tmp,tmp2) \
  38         mov     #(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG), tmp; \
  39         extu.b  tmp, tmp; \
  40         mov     #(FUTEX_CLOCK_REALTIME >> 8), tmp2; \
  41         swap.b  tmp2, tmp2; \
  42         or      tmp2, tmp; \
  43         xor     tmp, reg
  44 # define LOAD_FUTEX_WAKE(reg,tmp,tmp2) \
  45         mov     #(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), tmp; \
  46         extu.b  tmp, tmp; \
  47         xor     tmp, reg
  48 #else
  49 # if FUTEX_WAIT == 0
  50 #  define LOAD_PRIVATE_FUTEX_WAIT(reg,tmp,tmp2) \
  51         stc     gbr, tmp        ; \
  52         mov.w   99f, reg        ; \
  53         add     reg, tmp        ; \
  54         bra     98f             ; \
  55          mov.l  @tmp, reg       ; \
  56 99:     .word   PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \
  57 98:
  58 # else
  59 #  define LOAD_PRIVATE_FUTEX_WAIT(reg,tmp,tmp2) \
  60         stc     gbr, tmp        ; \
  61         mov.w   99f, reg        ; \
  62         add     reg, tmp        ; \
  63         mov.l   @tmp, reg       ; \
  64         bra     98f             ; \
  65          mov    #FUTEX_WAIT, tmp ; \
  66 99:     .word   PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \
  67 98:     or      tmp, reg
  68 # endif
  69 # define LOAD_PRIVATE_FUTEX_WAKE(reg,tmp,tmp2) \
  70         stc     gbr, tmp        ; \
  71         mov.w   99f, reg        ; \
  72         add     reg, tmp        ; \
  73         mov.l   @tmp, reg       ; \
  74         bra     98f             ; \
  75          mov    #FUTEX_WAKE, tmp ; \
  76 99:     .word   PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \
  77 98:     or      tmp, reg
  78 # if FUTEX_WAIT == 0
  79 #  define LOAD_FUTEX_WAIT(reg,tmp,tmp2) \
  80         stc     gbr, tmp        ; \
  81         mov.w   99f, tmp2       ; \
  82         add     tmp2, tmp       ; \
  83         mov.l   @tmp, tmp2      ; \
  84         bra     98f             ; \
  85          mov    #FUTEX_PRIVATE_FLAG, tmp ; \
  86 99:     .word   PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \
  87 98:     extu.b  tmp, tmp        ; \
  88         xor     tmp, reg        ; \
  89         and     tmp2, reg
  90 # else
  91 #  define LOAD_FUTEX_WAIT(reg,tmp,tmp2) \
  92         stc     gbr, tmp        ; \
  93         mov.w   99f, tmp2       ; \
  94         add     tmp2, tmp       ; \
  95         mov.l   @tmp, tmp2      ; \
  96         bra     98f             ; \
  97          mov    #FUTEX_PRIVATE_FLAG, tmp ; \
  98 99:     .word   PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \
  99 98:     extu.b  tmp, tmp        ; \
 100         xor     tmp, reg        ; \
 101         and     tmp2, reg       ; \
 102         mov     #FUTEX_WAIT, tmp ; \
 103         or      tmp, reg
 104 # endif
 105 # define LOAD_FUTEX_WAIT_ABS(reg,tmp,tmp2) \
 106         stc     gbr, tmp        ; \
 107         mov.w   99f, tmp2       ; \
 108         add     tmp2, tmp       ; \
 109         mov.l   @tmp, tmp2      ; \
 110         bra     98f             ; \
 111          mov    #FUTEX_PRIVATE_FLAG, tmp ; \
 112 99:     .word   PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \
 113 98:     extu.b  tmp, tmp        ; \
 114         xor     tmp, reg        ; \
 115         and     tmp2, reg       ; \
 116         mov     #FUTEX_WAIT_BITSET, tmp ; \
 117         mov     #(FUTEX_CLOCK_REALTIME >> 8), tmp2; \
 118         swap.b  tmp2, tmp2; \
 119         or      tmp2, tmp; \
 120         or      tmp, reg
 121 # define LOAD_FUTEX_WAKE(reg,tmp,tmp2) \
 122         stc     gbr, tmp        ; \
 123         mov.w   99f, tmp2       ; \
 124         add     tmp2, tmp       ; \
 125         mov.l   @tmp, tmp2      ; \
 126         bra     98f             ; \
 127          mov    #FUTEX_PRIVATE_FLAG, tmp ; \
 128 99:     .word   PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \
 129 98:     extu.b  tmp, tmp        ; \
 130         xor     tmp, reg        ; \
 131         and     tmp2, reg       ; \
 132         mov     #FUTEX_WAKE, tmp ; \
 133         or      tmp, reg
 134 #endif
 135
 136         .globl  __lll_lock_wait_private
 137         .type   __lll_lock_wait_private,@function
 138         .hidden __lll_lock_wait_private
 139         .align  5
 140         cfi_startproc
 141 __lll_lock_wait_private:
 142         mov.l   r8, @-r15
 143         cfi_adjust_cfa_offset(4)
 144         cfi_rel_offset (r8, 0)
 145         mov     r4, r6
 146         mov     r5, r8
 147         mov     #0, r7          /* No timeout.  */
 148         LOAD_PRIVATE_FUTEX_WAIT (r5, r0, r1)
 149
 150         mov     #2, r4
 151         cmp/eq  r4, r6
 152         bf      2f
 153
 154 1:
 155         mov     r8, r4
 156         mov     #SYS_futex, r3
 157         extu.b  r3, r3
 158         trapa   #0x14
 159         SYSCALL_INST_PAD
 160
 161 2:
 162         mov     #2, r6
 163         XCHG (r6, @r8, r2)
 164         tst     r2, r2
 165         bf      1b
 166
 167         mov.l   @r15+, r8
 168         cfi_adjust_cfa_offset (-4)
 169         cfi_restore (r8)
 170         rts
 171          mov    r2, r0
 172         cfi_endproc
 173         .size   __lll_lock_wait_private,.-__lll_lock_wait_private
 174
 175 #ifdef NOT_IN_libc
 176         .globl  __lll_lock_wait
 177         .type   __lll_lock_wait,@function
 178         .hidden __lll_lock_wait
 179         .align  5
 180         cfi_startproc
 181 __lll_lock_wait:
 182         mov.l   r9, @-r15
 183         cfi_adjust_cfa_offset(4)
 184         cfi_rel_offset (r9, 0)
 185         mov.l   r8, @-r15
 186         cfi_adjust_cfa_offset(4)
 187         cfi_rel_offset (r8, 0)
 188         mov     r6, r9
 189         mov     r4, r6
 190         mov     r5, r8
 191         mov     #0, r7          /* No timeout.  */
 192         mov     r9, r5
 193         LOAD_FUTEX_WAIT (r5, r0, r1)
 194
 195         mov     #2, r4
 196         cmp/eq  r4, r6
 197         bf      2f
 198
 199 1:
 200         mov     r8, r4
 201         mov     #SYS_futex, r3
 202         extu.b  r3, r3
 203         trapa   #0x14
 204         SYSCALL_INST_PAD
 205
 206 2:
 207         mov     #2, r6
 208         XCHG (r6, @r8, r2)
 209         tst     r2, r2
 210         bf      1b
 211
 212         mov.l   @r15+, r8
 213         cfi_adjust_cfa_offset (-4)
 214         cfi_restore (r8)
 215         mov.l   @r15+, r9
 216         cfi_adjust_cfa_offset (-4)
 217         cfi_restore (r9)
 218         ret
 219          mov    r2, r0
 220         cfi_endproc
 221         .size   __lll_lock_wait,.-__lll_lock_wait
 222
 223         /*      r5  (r8): futex
 224                 r7 (r11): flags
 225                 r6  (r9): timeout
 226                 r4 (r10): futex value
 227         */
 228         .globl  __lll_timedlock_wait
 229         .type   __lll_timedlock_wait,@function
 230         .hidden __lll_timedlock_wait
 231         .align  5
 232         cfi_startproc
 233 __lll_timedlock_wait:
 234         mov.l   r12, @-r15
 235         cfi_adjust_cfa_offset(4)
 236         cfi_rel_offset (r12, 0)
 237
 238 # ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 239         mov.l   .Lhave, r1
 240 #  ifdef PIC
 241         mova    .Lgot, r0
 242         mov.l   .Lgot, r12
 243         add     r0, r12
 244         add     r12, r1
 245 #  endif
 246         mov.l   @r1, r0
 247         tst     r0, r0
 248         bt      .Lreltmo
 249 # endif
 250
 251         /* if (timeout->tv_sec < 0) return ETIMEDOUT; */
 252         mov.l   @r6, r1
 253         cmp/pz  r1
 254         bf/s    5f
 255          mov    #ETIMEDOUT, r0
 256
 257         mov     r4, r2
 258         mov     r5, r4
 259         mov     r7, r5
 260         mov     r6, r7
 261         LOAD_FUTEX_WAIT_ABS (r5, r0, r1)
 262
 263         mov     #2, r6
 264         cmp/eq  r6, r2
 265         bf/s    2f
 266          mov    r6, r2
 267
 268 1:
 269         mov     #2, r6
 270         mov     #-1, r1
 271         mov     #SYS_futex, r3
 272         extu.b  r3, r3
 273         trapa   #0x16
 274         SYSCALL_INST_PAD
 275         mov     r0, r6
 276
 277 2:
 278         XCHG    (r2, @r4, r3)   /* NB:   lock is implied */
 279
 280         tst     r3, r3
 281         bt/s    3f
 282          mov    r6, r0
 283
 284         cmp/eq  #-ETIMEDOUT, r0
 285         bt      4f
 286         cmp/eq  #-EINVAL, r0
 287         bf      1b
 288 4:
 289         neg     r0, r3
 290 3:
 291         mov     r3, r0
 292 5:
 293         rts
 294          mov.l  @r15+, r12
 295         /* Omit CFI for restore in delay slot.  */
 296
 297 # ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 298         .align  2
 299 # ifdef PIC
 300 .Lgot:
 301         .long   _GLOBAL_OFFSET_TABLE_
 302 .Lhave:
 303         .long   __have_futex_clock_realtime@GOTOFF
 304 # else
 305 .Lhave:
 306         .long   __have_futex_clock_realtime
 307 # endif
 308
 309 .Lreltmo:
 310         /* Check for a valid timeout value.  */
 311         mov.l   @(4,r6), r1
 312         mov.l   .L1g, r0
 313         cmp/hs  r0, r1
 314         bt      3f
 315
 316         cfi_remember_state
 317
 318         mov.l   r11, @-r15
 319         cfi_adjust_cfa_offset(4)
 320         cfi_rel_offset (r11, 0)
 321         mov.l   r10, @-r15
 322         cfi_adjust_cfa_offset(4)
 323         cfi_rel_offset (r10, 0)
 324         mov.l   r9, @-r15
 325         cfi_adjust_cfa_offset(4)
 326         cfi_rel_offset (r9, 0)
 327         mov.l   r8, @-r15
 328         cfi_adjust_cfa_offset(4)
 329         cfi_rel_offset (r8, 0)
 330         mov     r7, r11
 331         mov     r4, r10
 332         mov     r6, r9
 333         mov     r5, r8
 334
 335         /* Stack frame for the timespec and timeval structs.  */
 336         add     #-8, r15
 337         cfi_adjust_cfa_offset(8)
 338
 339         mov     #2, r2
 340         XCHG (r2, @r8, r3)
 341
 342         tst     r3, r3
 343         bt      6f
 344
 345 1:
 346         /* Get current time.  */
 347         mov     r15, r4
 348         mov     #0, r5
 349         mov     #__NR_gettimeofday, r3
 350         trapa   #0x12
 351         SYSCALL_INST_PAD
 352
 353         /* Compute relative timeout.  */
 354         mov.l   @(4,r15), r0
 355         mov.w   .L1k, r1
 356         dmulu.l r0, r1          /* Micro seconds to nano seconds.  */
 357         mov.l   @r9, r2
 358         mov.l   @(4,r9), r3
 359         mov.l   @r15, r0
 360         sts     macl, r1
 361         sub     r0, r2
 362         clrt
 363         subc    r1, r3
 364         bf      4f
 365         mov.l   .L1g, r1
 366         add     r1, r3
 367         add     #-1, r2
 368 4:
 369         cmp/pz  r2
 370         bf      2f              /* Time is already up.  */
 371
 372         mov.l   r2, @r15        /* Store relative timeout.  */
 373         mov.l   r3, @(4,r15)
 374
 375         mov     r8, r4
 376         mov     r11, r5
 377         LOAD_FUTEX_WAIT (r5, r0, r1)
 378         mov     r10, r6
 379         mov     r15, r7
 380         mov     #SYS_futex, r3
 381         extu.b  r3, r3
 382         trapa   #0x14
 383         SYSCALL_INST_PAD
 384         mov     r0, r5
 385
 386         mov     #2, r2
 387         XCHG (r2, @r8, r3)
 388
 389         tst     r3, r3
 390         bt/s    6f
 391          mov    #-ETIMEDOUT, r1
 392         cmp/eq  r5, r1
 393         bf      1b
 394
 395 2:      mov     #ETIMEDOUT, r3
 396
 397 6:
 398         add     #8, r15
 399         cfi_adjust_cfa_offset (-8)
 400         mov.l   @r15+, r8
 401         cfi_adjust_cfa_offset (-4)
 402         cfi_restore (r8)
 403         mov.l   @r15+, r9
 404         cfi_adjust_cfa_offset (-4)
 405         cfi_restore (r9)
 406         mov.l   @r15+, r10
 407         cfi_adjust_cfa_offset (-4)
 408         cfi_restore (r10)
 409         mov.l   @r15+, r11
 410         cfi_adjust_cfa_offset (-4)
 411         cfi_restore (r11)
 412         mov.l   @r15+, r12
 413         cfi_adjust_cfa_offset (-4)
 414         cfi_restore (r12)
 415         rts
 416          mov    r3, r0
 417
 418         cfi_restore_state
 419
 420 3:
 421         mov.l   @r15+, r12
 422         cfi_adjust_cfa_offset (-4)
 423         cfi_restore (r12)
 424         rts
 425          mov    #EINVAL, r0
 426 # endif
 427         cfi_endproc
 428
 429 .L1k:
 430         .word   1000
 431         .align  2
 432 .L1g:
 433         .long   1000000000
 434
 435         .size   __lll_timedlock_wait,.-__lll_timedlock_wait
 436 #endif
 437
 438         .globl  __lll_unlock_wake_private
 439         .type   __lll_unlock_wake_private,@function
 440         .hidden __lll_unlock_wake_private
 441         .align  5
 442         cfi_startproc
 443 __lll_unlock_wake_private:
 444         LOAD_PRIVATE_FUTEX_WAKE (r5, r0, r1)
 445         mov     #1, r6          /* Wake one thread.  */
 446         mov     #0, r7
 447         mov.l   r7, @r4         /* Stores 0.  */
 448         mov     #SYS_futex, r3
 449         extu.b  r3, r3
 450         trapa   #0x14
 451         SYSCALL_INST_PAD
 452         rts
 453          nop
 454         cfi_endproc
 455         .size   __lll_unlock_wake_private,.-__lll_unlock_wake_private
 456
 457 #ifdef NOT_IN_libc
 458         .globl  __lll_unlock_wake
 459         .type   __lll_unlock_wake,@function
 460         .hidden __lll_unlock_wake
 461         .align  5
 462         cfi_startproc
 463 __lll_unlock_wake:
 464         LOAD_FUTEX_WAKE (r5, r0, r1)
 465         mov     #1, r6          /* Wake one thread.  */
 466         mov     #0, r7
 467         mov.l   r7, @r4         /* Stores 0.  */
 468         mov     #SYS_futex, r3
 469         extu.b  r3, r3
 470         trapa   #0x14
 471         SYSCALL_INST_PAD
 472         rts
 473          nop
 474         cfi_endproc
 475         .size   __lll_unlock_wake,.-__lll_unlock_wake
 476
 477         .globl  __lll_timedwait_tid
 478         .type   __lll_timedwait_tid,@function
 479         .hidden __lll_timedwait_tid
 480         .align  5
 481         cfi_startproc
 482 __lll_timedwait_tid:
 483         mov.l   r9, @-r15
 484         cfi_adjust_cfa_offset(4)
 485         cfi_rel_offset (r9, 0)
 486         mov.l   r8, @-r15
 487         cfi_adjust_cfa_offset(4)
 488         cfi_rel_offset (r8, 0)
 489         mov     r4, r8
 490         mov     r5, r9
 491
 492         /* Stack frame for the timespec and timeval structs.  */
 493         add     #-8, r15
 494         cfi_adjust_cfa_offset(8)
 495
 496 2:
 497         /* Get current time.  */
 498         mov     r15, r4
 499         mov     #0, r5
 500         mov     #__NR_gettimeofday, r3
 501         trapa   #0x12
 502         SYSCALL_INST_PAD
 503
 504         /* Compute relative timeout.  */
 505         mov.l   @(4,r15), r0
 506         mov.w   .L1k2, r1
 507         dmulu.l r0, r1          /* Micro seconds to nano seconds.  */
 508         mov.l   @r9, r2
 509         mov.l   @(4,r9), r3
 510         mov.l   @r15, r0
 511         sts     macl, r1
 512         sub     r0, r2
 513         clrt
 514         subc    r1, r3
 515         bf      5f
 516         mov.l   .L1g2, r1
 517         add     r1, r3
 518         add     #-1, r2
 519 5:
 520         cmp/pz  r2
 521         bf      6f              /* Time is already up.  */
 522
 523         mov.l   r2, @r15        /* Store relative timeout.  */
 524         mov.l   r3, @(4,r15)
 525
 526         mov.l   @r8, r2
 527         tst     r2, r2
 528         bt      4f
 529
 530         mov     r8, r4
 531         /* XXX The kernel so far uses global futex for the wakeup at
 532            all times.  */
 533         mov     #0, r5
 534         extu.b  r5, r5
 535         mov     r2, r6
 536         mov     r15, r7
 537         mov     #SYS_futex, r3
 538         extu.b  r3, r3
 539         trapa   #0x14
 540         SYSCALL_INST_PAD
 541
 542         mov.l   @r8, r2
 543         tst     r2, r2
 544         bf      1f
 545 4:
 546         mov     #0, r0
 547 3:
 548         cfi_remember_state
 549         add     #8, r15
 550         cfi_adjust_cfa_offset (-8)
 551         mov.l   @r15+, r8
 552         cfi_adjust_cfa_offset (-4)
 553         cfi_restore (r8)
 554         rts
 555          mov.l  @r15+, r9
 556         /* Omit CFI for restore in delay slot.  */
 557         cfi_restore_state
 558 1:
 559         /* Check whether the time expired.  */
 560         mov     #-ETIMEDOUT, r1
 561         cmp/eq  r0, r1
 562         bf      2b
 563 6:
 564         bra     3b
 565          mov    #ETIMEDOUT, r0
 566         cfi_endproc
 567
 568 .L1k2:
 569         .word   1000
 570         .align  2
 571 .L1g2:
 572         .long   1000000000
 573         .size   __lll_timedwait_tid,.-__lll_timedwait_tid
 574 #endif