sysdeps/aarch64/dl-tlsdesc.S

   1 /* Thread-local storage handling in the ELF dynamic linker.
   2    AArch64 version.
   3    Copyright (C) 2011-2017 Free Software Foundation, Inc.
   4
   5    This file is part of the GNU C Library.
   6
   7    The GNU C Library is free software; you can redistribute it and/or
   8    modify it under the terms of the GNU Lesser General Public
   9    License as published by the Free Software Foundation; either
  10    version 2.1 of the License, or (at your option) any later version.
  11
  12    The GNU C Library is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    Lesser General Public License for more details.
  16
  17    You should have received a copy of the GNU Lesser General Public
  18    License along with the GNU C Library; if not, see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 #include <sysdep.h>
  22 #include <tls.h>
  23 #include "tlsdesc.h"
  24
  25 #define NSAVEDQREGPAIRS 16
  26 #define SAVE_Q_REGISTERS                                \
  27         stp     q0, q1, [sp, #-32*NSAVEDQREGPAIRS]!;    \
  28         cfi_adjust_cfa_offset (32*NSAVEDQREGPAIRS);     \
  29         stp      q2,  q3, [sp, #32*1];                  \
  30         stp      q4,  q5, [sp, #32*2];                  \
  31         stp      q6,  q7, [sp, #32*3];                  \
  32         stp      q8,  q9, [sp, #32*4];                  \
  33         stp     q10, q11, [sp, #32*5];                  \
  34         stp     q12, q13, [sp, #32*6];                  \
  35         stp     q14, q15, [sp, #32*7];                  \
  36         stp     q16, q17, [sp, #32*8];                  \
  37         stp     q18, q19, [sp, #32*9];                  \
  38         stp     q20, q21, [sp, #32*10];                 \
  39         stp     q22, q23, [sp, #32*11];                 \
  40         stp     q24, q25, [sp, #32*12];                 \
  41         stp     q26, q27, [sp, #32*13];                 \
  42         stp     q28, q29, [sp, #32*14];                 \
  43         stp     q30, q31, [sp, #32*15];
  44
  45 #define RESTORE_Q_REGISTERS                             \
  46         ldp      q2,  q3, [sp, #32*1];                  \
  47         ldp      q4,  q5, [sp, #32*2];                  \
  48         ldp      q6,  q7, [sp, #32*3];                  \
  49         ldp      q8,  q9, [sp, #32*4];                  \
  50         ldp     q10, q11, [sp, #32*5];                  \
  51         ldp     q12, q13, [sp, #32*6];                  \
  52         ldp     q14, q15, [sp, #32*7];                  \
  53         ldp     q16, q17, [sp, #32*8];                  \
  54         ldp     q18, q19, [sp, #32*9];                  \
  55         ldp     q20, q21, [sp, #32*10];                 \
  56         ldp     q22, q23, [sp, #32*11];                 \
  57         ldp     q24, q25, [sp, #32*12];                 \
  58         ldp     q26, q27, [sp, #32*13];                 \
  59         ldp     q28, q29, [sp, #32*14];                 \
  60         ldp     q30, q31, [sp, #32*15];                 \
  61         ldp      q0,  q1, [sp], #32*NSAVEDQREGPAIRS;    \
  62         cfi_adjust_cfa_offset (-32*NSAVEDQREGPAIRS);
  63
  64         .text
  65
  66         /* Compute the thread pointer offset for symbols in the static
  67            TLS block. The offset is the same for all threads.
  68            Prototype:
  69            _dl_tlsdesc_return (tlsdesc *) ;
  70          */
  71         .hidden _dl_tlsdesc_return
  72         .global _dl_tlsdesc_return
  73         .type   _dl_tlsdesc_return,%function
  74         cfi_startproc
  75         .align 2
  76 _dl_tlsdesc_return:
  77         DELOUSE (0)
  78         ldr     PTR_REG (0), [x0, #PTR_SIZE]
  79         RET
  80         cfi_endproc
  81         .size   _dl_tlsdesc_return, .-_dl_tlsdesc_return
  82
  83         /* Same as _dl_tlsdesc_return but with synchronization for
  84            lazy relocation.
  85            Prototype:
  86            _dl_tlsdesc_return_lazy (tlsdesc *) ;
  87          */
  88         .hidden _dl_tlsdesc_return_lazy
  89         .global _dl_tlsdesc_return_lazy
  90         .type   _dl_tlsdesc_return_lazy,%function
  91         cfi_startproc
  92         .align 2
  93 _dl_tlsdesc_return_lazy:
  94         /* The ldar here happens after the load from [x0] at the call site
  95            (that is generated by the compiler as part of the TLS access ABI),
  96            so it reads the same value (this function is the final value of
  97            td->entry) and thus it synchronizes with the release store to
  98            td->entry in _dl_tlsdesc_resolve_rela_fixup ensuring that the load
  99            from [x0,#PTR_SIZE] here happens after the initialization of td->arg. */
 100         DELOUSE (0)
 101         ldar    PTR_REG (zr), [x0]
 102         ldr     PTR_REG (0), [x0, #PTR_SIZE]
 103         RET
 104         cfi_endproc
 105         .size   _dl_tlsdesc_return_lazy, .-_dl_tlsdesc_return_lazy
 106
 107         /* Handler for undefined weak TLS symbols.
 108            Prototype:
 109            _dl_tlsdesc_undefweak (tlsdesc *);
 110
 111            The second word of the descriptor contains the addend.
 112            Return the addend minus the thread pointer. This ensures
 113            that when the caller adds on the thread pointer it gets back
 114            the addend.  */
 115
 116         .hidden _dl_tlsdesc_undefweak
 117         .global _dl_tlsdesc_undefweak
 118         .type   _dl_tlsdesc_undefweak,%function
 119         cfi_startproc
 120         .align  2
 121 _dl_tlsdesc_undefweak:
 122         str     x1, [sp, #-16]!
 123         cfi_adjust_cfa_offset (16)
 124         /* The ldar here happens after the load from [x0] at the call site
 125            (that is generated by the compiler as part of the TLS access ABI),
 126            so it reads the same value (this function is the final value of
 127            td->entry) and thus it synchronizes with the release store to
 128            td->entry in _dl_tlsdesc_resolve_rela_fixup ensuring that the load
 129            from [x0,#8] here happens after the initialization of td->arg.  */
 130         DELOUSE (0)
 131         ldar    PTR_REG (zr), [x0]
 132         ldr     PTR_REG (0), [x0, #PTR_SIZE]
 133         mrs     x1, tpidr_el0
 134         sub     PTR_REG (0), PTR_REG (0), PTR_REG (1)
 135         ldr     x1, [sp], #16
 136         cfi_adjust_cfa_offset (-16)
 137         RET
 138         cfi_endproc
 139         .size   _dl_tlsdesc_undefweak, .-_dl_tlsdesc_undefweak
 140
 141 #ifdef SHARED
 142         /* Handler for dynamic TLS symbols.
 143            Prototype:
 144            _dl_tlsdesc_dynamic (tlsdesc *) ;
 145
 146            The second word of the descriptor points to a
 147            tlsdesc_dynamic_arg structure.
 148
 149            Returns the offset between the thread pointer and the
 150            object referenced by the argument.
 151
 152            ptrdiff_t
 153            __attribute__ ((__regparm__ (1)))
 154            _dl_tlsdesc_dynamic (struct tlsdesc *tdp)
 155            {
 156              struct tlsdesc_dynamic_arg *td = tdp->arg;
 157              dtv_t *dtv = *(dtv_t **)((char *)__thread_pointer + TCBHEAD_DTV);
 158              if (__builtin_expect (td->gen_count <= dtv[0].counter
 159                 && (dtv[td->tlsinfo.ti_module].pointer.val
 160                     != TLS_DTV_UNALLOCATED),
 161                 1))
 162                return dtv[td->tlsinfo.ti_module].pointer.val
 163                 + td->tlsinfo.ti_offset
 164                 - __thread_pointer;
 165
 166              return ___tls_get_addr (&td->tlsinfo) - __thread_pointer;
 167            }
 168          */
 169
 170         .hidden _dl_tlsdesc_dynamic
 171         .global _dl_tlsdesc_dynamic
 172         .type   _dl_tlsdesc_dynamic,%function
 173         cfi_startproc
 174         .align 2
 175 _dl_tlsdesc_dynamic:
 176 # define NSAVEXREGPAIRS 2
 177         stp     x29, x30, [sp,#-(32+16*NSAVEXREGPAIRS)]!
 178         cfi_adjust_cfa_offset (32+16*NSAVEXREGPAIRS)
 179         cfi_rel_offset (x29, 0)
 180         cfi_rel_offset (x30, 8)
 181         mov     x29, sp
 182         DELOUSE (0)
 183
 184         /* Save just enough registers to support fast path, if we fall
 185            into slow path we will save additional registers.  */
 186
 187         stp     x1,  x2, [sp, #32+16*0]
 188         stp     x3,  x4, [sp, #32+16*1]
 189         cfi_rel_offset (x1, 32)
 190         cfi_rel_offset (x2, 32+8)
 191         cfi_rel_offset (x3, 32+16)
 192         cfi_rel_offset (x4, 32+24)
 193
 194         mrs     x4, tpidr_el0
 195         /* The ldar here happens after the load from [x0] at the call site
 196            (that is generated by the compiler as part of the TLS access ABI),
 197            so it reads the same value (this function is the final value of
 198            td->entry) and thus it synchronizes with the release store to
 199            td->entry in _dl_tlsdesc_resolve_rela_fixup ensuring that the load
 200            from [x0,#PTR_SIZE] here happens after the initialization of td->arg.  */
 201         ldar    PTR_REG (zr), [x0]
 202         ldr     PTR_REG (1), [x0,#TLSDESC_ARG]
 203         ldr     PTR_REG (0), [x4,#TCBHEAD_DTV]
 204         ldr     PTR_REG (3), [x1,#TLSDESC_GEN_COUNT]
 205         ldr     PTR_REG (2), [x0,#DTV_COUNTER]
 206         cmp     PTR_REG (3), PTR_REG (2)
 207         b.hi    2f
 208         ldr     PTR_REG (2), [x1,#TLSDESC_MODID]
 209         add     PTR_REG (0), PTR_REG (0), PTR_REG (2), lsl #(PTR_LOG_SIZE + 1)
 210         ldr     PTR_REG (0), [x0] /* Load val member of DTV entry.  */
 211         cmp     x0, #TLS_DTV_UNALLOCATED
 212         b.eq    2f
 213         ldr     PTR_REG (1), [x1,#TLSDESC_MODOFF]
 214         add     PTR_REG (0), PTR_REG (0), PTR_REG (1)
 215         sub     PTR_REG (0), PTR_REG (0), PTR_REG (4)
 216 1:
 217         ldp      x1,  x2, [sp, #32+16*0]
 218         ldp      x3,  x4, [sp, #32+16*1]
 219
 220         ldp     x29, x30, [sp], #(32+16*NSAVEXREGPAIRS)
 221         cfi_adjust_cfa_offset (-32-16*NSAVEXREGPAIRS)
 222         cfi_restore (x29)
 223         cfi_restore (x30)
 224 # undef NSAVEXREGPAIRS
 225         RET
 226 2:
 227         /* This is the slow path. We need to call __tls_get_addr() which
 228            means we need to save and restore all the register that the
 229            callee will trash.  */
 230
 231         /* Save the remaining registers that we must treat as caller save.  */
 232 # define NSAVEXREGPAIRS 7
 233         stp      x5,  x6, [sp, #-16*NSAVEXREGPAIRS]!
 234         cfi_adjust_cfa_offset (16*NSAVEXREGPAIRS)
 235         stp      x7,  x8, [sp, #16*1]
 236         stp      x9, x10, [sp, #16*2]
 237         stp     x11, x12, [sp, #16*3]
 238         stp     x13, x14, [sp, #16*4]
 239         stp     x15, x16, [sp, #16*5]
 240         stp     x17, x18, [sp, #16*6]
 241         cfi_rel_offset (x5, 0)
 242         cfi_rel_offset (x6, 8)
 243         cfi_rel_offset (x7, 16)
 244         cfi_rel_offset (x8, 16+8)
 245         cfi_rel_offset (x9, 16*2)
 246         cfi_rel_offset (x10, 16*2+8)
 247         cfi_rel_offset (x11, 16*3)
 248         cfi_rel_offset (x12, 16*3+8)
 249         cfi_rel_offset (x13, 16*4)
 250         cfi_rel_offset (x14, 16*4+8)
 251         cfi_rel_offset (x15, 16*5)
 252         cfi_rel_offset (x16, 16*5+8)
 253         cfi_rel_offset (x17, 16*6)
 254         cfi_rel_offset (x18, 16*6+8)
 255
 256         SAVE_Q_REGISTERS
 257
 258         mov     x0, x1
 259         bl      __tls_get_addr
 260
 261         mrs     x1, tpidr_el0
 262         sub     PTR_REG (0), PTR_REG (0), PTR_REG (1)
 263
 264         RESTORE_Q_REGISTERS
 265
 266         ldp      x7,  x8, [sp, #16*1]
 267         ldp      x9, x10, [sp, #16*2]
 268         ldp     x11, x12, [sp, #16*3]
 269         ldp     x13, x14, [sp, #16*4]
 270         ldp     x15, x16, [sp, #16*5]
 271         ldp     x17, x18, [sp, #16*6]
 272         ldp      x5,  x6, [sp], #16*NSAVEXREGPAIRS
 273         cfi_adjust_cfa_offset (-16*NSAVEXREGPAIRS)
 274         b       1b
 275         cfi_endproc
 276         .size   _dl_tlsdesc_dynamic, .-_dl_tlsdesc_dynamic
 277 # undef NSAVEXREGPAIRS
 278 #endif
 279
 280         /* This function is a wrapper for a lazy resolver for TLS_DESC
 281            RELA relocations.
 282            When the actual resolver returns, it will have adjusted the
 283            TLS descriptor such that we can tail-call it for it to return
 284            the TP offset of the symbol.  */
 285
 286         .hidden _dl_tlsdesc_resolve_rela
 287         .global _dl_tlsdesc_resolve_rela
 288         .type   _dl_tlsdesc_resolve_rela,%function
 289         cfi_startproc
 290         .align 2
 291 _dl_tlsdesc_resolve_rela:
 292 #define NSAVEXREGPAIRS 9
 293         /* The tlsdesc PLT entry pushes x2 and x3 to the stack.  */
 294         cfi_adjust_cfa_offset (16)
 295         cfi_rel_offset (x2, 0)
 296         cfi_rel_offset (x3, 8)
 297         stp     x29, x30, [sp, #-(32+16*NSAVEXREGPAIRS)]!
 298         cfi_adjust_cfa_offset (32+16*NSAVEXREGPAIRS)
 299         cfi_rel_offset (x29, 0)
 300         cfi_rel_offset (x30, 8)
 301         mov     x29, sp
 302         stp      x1,  x4, [sp, #32+16*0]
 303         stp      x5,  x6, [sp, #32+16*1]
 304         stp      x7,  x8, [sp, #32+16*2]
 305         stp      x9, x10, [sp, #32+16*3]
 306         stp     x11, x12, [sp, #32+16*4]
 307         stp     x13, x14, [sp, #32+16*5]
 308         stp     x15, x16, [sp, #32+16*6]
 309         stp     x17, x18, [sp, #32+16*7]
 310         str     x0,       [sp, #32+16*8]
 311         cfi_rel_offset (x1, 32)
 312         cfi_rel_offset (x4, 32+8)
 313         cfi_rel_offset (x5, 32+16)
 314         cfi_rel_offset (x6, 32+16+8)
 315         cfi_rel_offset (x7, 32+16*2)
 316         cfi_rel_offset (x8, 32+16*2+8)
 317         cfi_rel_offset (x9, 32+16*3)
 318         cfi_rel_offset (x10, 32+16*3+8)
 319         cfi_rel_offset (x11, 32+16*4)
 320         cfi_rel_offset (x12, 32+16*4+8)
 321         cfi_rel_offset (x13, 32+16*5)
 322         cfi_rel_offset (x14, 32+16*5+8)
 323         cfi_rel_offset (x15, 32+16*6)
 324         cfi_rel_offset (x16, 32+16*6+8)
 325         cfi_rel_offset (x17, 32+16*7)
 326         cfi_rel_offset (x18, 32+16*7+8)
 327         cfi_rel_offset (x0, 32+16*8)
 328
 329         SAVE_Q_REGISTERS
 330
 331         DELOUSE (3)
 332         ldr     PTR_REG (1), [x3, #PTR_SIZE]
 333         bl      _dl_tlsdesc_resolve_rela_fixup
 334
 335         RESTORE_Q_REGISTERS
 336
 337         ldr     x0, [sp, #32+16*8]
 338         DELOUSE (0)
 339         ldr     PTR_REG (1), [x0]
 340         blr     x1
 341
 342         ldp      x1,  x4, [sp, #32+16*0]
 343         ldp      x5,  x6, [sp, #32+16*1]
 344         ldp      x7,  x8, [sp, #32+16*2]
 345         ldp      x9, x10, [sp, #32+16*3]
 346         ldp     x11, x12, [sp, #32+16*4]
 347         ldp     x13, x14, [sp, #32+16*5]
 348         ldp     x15, x16, [sp, #32+16*6]
 349         ldp     x17, x18, [sp, #32+16*7]
 350         ldp     x29, x30, [sp], #(32+16*NSAVEXREGPAIRS)
 351         cfi_adjust_cfa_offset (-32-16*NSAVEXREGPAIRS)
 352         cfi_restore (x29)
 353         cfi_restore (x30)
 354         ldp     x2, x3, [sp], #16
 355         cfi_adjust_cfa_offset (-16)
 356         RET
 357 #undef NSAVEXREGPAIRS
 358         cfi_endproc
 359         .size   _dl_tlsdesc_resolve_rela, .-_dl_tlsdesc_resolve_rela
 360
 361         /* This function is a placeholder for lazy resolving of TLS
 362         relocations.  Once some thread starts resolving a TLS
 363         relocation, it sets up the TLS descriptor to use this
 364         resolver, such that other threads that would attempt to
 365         resolve it concurrently may skip the call to the original lazy
 366         resolver and go straight to a condition wait.
 367
 368         When the actual resolver returns, it will have adjusted the
 369         TLS descriptor such that we can tail-call it for it to return
 370         the TP offset of the symbol.  */
 371
 372         .hidden _dl_tlsdesc_resolve_hold
 373         .global _dl_tlsdesc_resolve_hold
 374         .type   _dl_tlsdesc_resolve_hold,%function
 375         cfi_startproc
 376         .align 2
 377 _dl_tlsdesc_resolve_hold:
 378 #define NSAVEXREGPAIRS 10
 379 1:
 380         stp     x29, x30, [sp, #-(32+16*NSAVEXREGPAIRS)]!
 381         cfi_adjust_cfa_offset (32+16*NSAVEXREGPAIRS)
 382         cfi_rel_offset (x29, 0)
 383         cfi_rel_offset (x30, 8)
 384         mov     x29, sp
 385         stp      x1,  x2, [sp, #32+16*0]
 386         stp      x3,  x4, [sp, #32+16*1]
 387         stp      x5,  x6, [sp, #32+16*2]
 388         stp      x7,  x8, [sp, #32+16*3]
 389         stp      x9, x10, [sp, #32+16*4]
 390         stp     x11, x12, [sp, #32+16*5]
 391         stp     x13, x14, [sp, #32+16*6]
 392         stp     x15, x16, [sp, #32+16*7]
 393         stp     x17, x18, [sp, #32+16*8]
 394         str     x0,       [sp, #32+16*9]
 395         cfi_rel_offset (x1, 32)
 396         cfi_rel_offset (x2, 32+8)
 397         cfi_rel_offset (x3, 32+16)
 398         cfi_rel_offset (x4, 32+16+8)
 399         cfi_rel_offset (x5, 32+16*2)
 400         cfi_rel_offset (x6, 32+16*2+8)
 401         cfi_rel_offset (x7, 32+16*3)
 402         cfi_rel_offset (x8, 32+16*3+8)
 403         cfi_rel_offset (x9, 32+16*4)
 404         cfi_rel_offset (x10, 32+16*4+8)
 405         cfi_rel_offset (x11, 32+16*5)
 406         cfi_rel_offset (x12, 32+16*5+8)
 407         cfi_rel_offset (x13, 32+16*6)
 408         cfi_rel_offset (x14, 32+16*6+8)
 409         cfi_rel_offset (x15, 32+16*7)
 410         cfi_rel_offset (x16, 32+16*7+8)
 411         cfi_rel_offset (x17, 32+16*8)
 412         cfi_rel_offset (x18, 32+16*8+8)
 413         cfi_rel_offset (x0, 32+16*9)
 414
 415         SAVE_Q_REGISTERS
 416
 417         adr     x1, 1b
 418         bl      _dl_tlsdesc_resolve_hold_fixup
 419
 420         RESTORE_Q_REGISTERS
 421
 422         ldr     x0, [sp, #32+16*9]
 423         DELOUSE (0)
 424         ldr     PTR_REG (1), [x0]
 425         blr     x1
 426
 427         ldp      x1,  x2, [sp, #32+16*0]
 428         ldp      x3,  x4, [sp, #32+16*1]
 429         ldp      x5,  x6, [sp, #32+16*2]
 430         ldp      x7,  x8, [sp, #32+16*3]
 431         ldp      x9, x10, [sp, #32+16*4]
 432         ldp     x11, x12, [sp, #32+16*5]
 433         ldp     x13, x14, [sp, #32+16*6]
 434         ldp     x15, x16, [sp, #32+16*7]
 435         ldp     x17, x18, [sp, #32+16*8]
 436         ldp     x29, x30, [sp], #(32+16*NSAVEXREGPAIRS)
 437         cfi_adjust_cfa_offset (-32-16*NSAVEXREGPAIRS)
 438         cfi_restore (x29)
 439         cfi_restore (x30)
 440         RET
 441         cfi_endproc
 442         .size   _dl_tlsdesc_resolve_hold, .-_dl_tlsdesc_resolve_hold
 443 #undef NSAVEXREGPAIRS