libatomic/config/linux/aarch64/atomic_16.S

   1 /* Copyright (C) 2022-2023 Free Software Foundation, Inc.
   2
   3    This file is part of the GNU Atomic Library (libatomic).
   4
   5    Libatomic is free software; you can redistribute it and/or modify it
   6    under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 3 of the License, or
   8    (at your option) any later version.
   9
  10    Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
  11    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  12    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13    more details.
  14
  15    Under Section 7 of GPL version 3, you are granted additional
  16    permissions described in the GCC Runtime Library Exception, version
  17    3.1, as published by the Free Software Foundation.
  18
  19    You should have received a copy of the GNU General Public License and
  20    a copy of the GCC Runtime Library Exception along with this program;
  21    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  22    <http://www.gnu.org/licenses/>.  */
  23
  24
  25         .arch   armv8-a+lse
  26
  27 #define ENTRY(name)             \
  28         .global name;           \
  29         .hidden name;           \
  30         .type name,%function;   \
  31         .p2align 4;             \
  32 name:                           \
  33         .cfi_startproc;         \
  34         hint    34      // bti c
  35
  36 #define END(name)               \
  37         .cfi_endproc;           \
  38         .size name, .-name;
  39
  40 #define res0 x0
  41 #define res1 x1
  42 #define in0  x2
  43 #define in1  x3
  44 #define tmp0 x6
  45 #define tmp1 x7
  46 #define exp0 x8
  47 #define exp1 x9
  48
  49 #ifdef __AARCH64EB__
  50 # define reslo x1
  51 # define reshi x0
  52 # define inlo  x3
  53 # define inhi  x2
  54 # define tmplo x7
  55 # define tmphi x6
  56 #else
  57 # define reslo x0
  58 # define reshi x1
  59 # define inlo  x2
  60 # define inhi  x3
  61 # define tmplo x6
  62 # define tmphi x7
  63 #endif
  64
  65 #define RELAXED 0
  66 #define CONSUME 1
  67 #define ACQUIRE 2
  68 #define RELEASE 3
  69 #define ACQ_REL 4
  70 #define SEQ_CST 5
  71
  72
  73 ENTRY (libat_load_16_i1)
  74         cbnz    w1, 1f
  75         ldp     res0, res1, [x0]
  76         ret
  77 1:
  78         cmp     w1, ACQUIRE
  79         b.hi    2f
  80         ldp     res0, res1, [x0]
  81         dmb     ishld
  82         ret
  83 2:
  84         ldp     res0, res1, [x0]
  85         dmb     ish
  86         ret
  87 END (libat_load_16_i1)
  88
  89
  90 ENTRY (libat_store_16_i1)
  91         cbnz    w4, 1f
  92         stp     in0, in1, [x0]
  93         ret
  94 1:
  95         dmb     ish
  96         stp     in0, in1, [x0]
  97         cmp     w4, SEQ_CST
  98         beq     2f
  99         ret
 100 2:
 101         dmb     ish
 102         ret
 103 END (libat_store_16_i1)
 104
 105
 106 ENTRY (libat_exchange_16_i1)
 107         mov     x5, x0
 108         cbnz    w4, 2f
 109 1:
 110         ldxp    res0, res1, [x5]
 111         stxp    w4, in0, in1, [x5]
 112         cbnz    w4, 1b
 113         ret
 114 2:
 115         cmp     w4, ACQUIRE
 116         b.hi    4f
 117 3:
 118         ldaxp   res0, res1, [x5]
 119         stxp    w4, in0, in1, [x5]
 120         cbnz    w4, 3b
 121         ret
 122 4:
 123         cmp     w4, RELEASE
 124         b.ne    6f
 125 5:
 126         ldxp    res0, res1, [x5]
 127         stlxp   w4, in0, in1, [x5]
 128         cbnz    w4, 5b
 129         ret
 130 6:
 131         ldaxp   res0, res1, [x5]
 132         stlxp   w4, in0, in1, [x5]
 133         cbnz    w4, 6b
 134         ret
 135 END (libat_exchange_16_i1)
 136
 137
 138 ENTRY (libat_compare_exchange_16_i1)
 139         ldp     exp0, exp1, [x1]
 140         mov     tmp0, exp0
 141         mov     tmp1, exp1
 142         cbz     w4, 2f
 143         cmp     w4, RELEASE
 144         b.hs    3f
 145         caspa   exp0, exp1, in0, in1, [x0]
 146 0:
 147         cmp     exp0, tmp0
 148         ccmp    exp1, tmp1, 0, eq
 149         bne     1f
 150         mov     x0, 1
 151         ret
 152 1:
 153         stp     exp0, exp1, [x1]
 154         mov     x0, 0
 155         ret
 156 2:
 157         casp    exp0, exp1, in0, in1, [x0]
 158         b       0b
 159 3:
 160         b.hi    4f
 161         caspl   exp0, exp1, in0, in1, [x0]
 162         b       0b
 163 4:
 164         caspal  exp0, exp1, in0, in1, [x0]
 165         b       0b
 166 END (libat_compare_exchange_16_i1)
 167
 168
 169 ENTRY (libat_fetch_add_16_i1)
 170         mov     x5, x0
 171         cbnz    w4, 2f
 172 1:
 173         ldxp    res0, res1, [x5]
 174         adds    tmplo, reslo, inlo
 175         adc     tmphi, reshi, inhi
 176         stxp    w4, tmp0, tmp1, [x5]
 177         cbnz    w4, 1b
 178         ret
 179 2:
 180         ldaxp   res0, res1, [x5]
 181         adds    tmplo, reslo, inlo
 182         adc     tmphi, reshi, inhi
 183         stlxp   w4, tmp0, tmp1, [x5]
 184         cbnz    w4, 2b
 185         ret
 186 END (libat_fetch_add_16_i1)
 187
 188
 189 ENTRY (libat_add_fetch_16_i1)
 190         mov     x5, x0
 191         cbnz    w4, 2f
 192 1:
 193         ldxp    res0, res1, [x5]
 194         adds    reslo, reslo, inlo
 195         adc     reshi, reshi, inhi
 196         stxp    w4, res0, res1, [x5]
 197         cbnz    w4, 1b
 198         ret
 199 2:
 200         ldaxp   res0, res1, [x5]
 201         adds    reslo, reslo, inlo
 202         adc     reshi, reshi, inhi
 203         stlxp   w4, res0, res1, [x5]
 204         cbnz    w4, 2b
 205         ret
 206 END (libat_add_fetch_16_i1)
 207
 208
 209 ENTRY (libat_fetch_sub_16_i1)
 210         mov     x5, x0
 211         cbnz    w4, 2f
 212 1:
 213         ldxp    res0, res1, [x5]
 214         subs    tmplo, reslo, inlo
 215         sbc     tmphi, reshi, inhi
 216         stxp    w4, tmp0, tmp1, [x5]
 217         cbnz    w4, 1b
 218         ret
 219 2:
 220         ldaxp   res0, res1, [x5]
 221         subs    tmplo, reslo, inlo
 222         sbc     tmphi, reshi, inhi
 223         stlxp   w4, tmp0, tmp1, [x5]
 224         cbnz    w4, 2b
 225         ret
 226 END (libat_fetch_sub_16_i1)
 227
 228
 229 ENTRY (libat_sub_fetch_16_i1)
 230         mov     x5, x0
 231         cbnz    w4, 2f
 232 1:
 233         ldxp    res0, res1, [x5]
 234         subs    reslo, reslo, inlo
 235         sbc     reshi, reshi, inhi
 236         stxp    w4, res0, res1, [x5]
 237         cbnz    w4, 1b
 238         ret
 239 2:
 240         ldaxp   res0, res1, [x5]
 241         subs    reslo, reslo, inlo
 242         sbc     reshi, reshi, inhi
 243         stlxp   w4, res0, res1, [x5]
 244         cbnz    w4, 2b
 245         ret
 246 END (libat_sub_fetch_16_i1)
 247
 248
 249 ENTRY (libat_fetch_or_16_i1)
 250         mov     x5, x0
 251         cbnz    w4, 2f
 252 1:
 253         ldxp    res0, res1, [x5]
 254         orr     tmp0, res0, in0
 255         orr     tmp1, res1, in1
 256         stxp    w4, tmp0, tmp1, [x5]
 257         cbnz    w4, 1b
 258         ret
 259 2:
 260         ldaxp   res0, res1, [x5]
 261         orr     tmp0, res0, in0
 262         orr     tmp1, res1, in1
 263         stlxp   w4, tmp0, tmp1, [x5]
 264         cbnz    w4, 2b
 265         ret
 266 END (libat_fetch_or_16_i1)
 267
 268
 269 ENTRY (libat_or_fetch_16_i1)
 270         mov     x5, x0
 271         cbnz    w4, 2f
 272 1:
 273         ldxp    res0, res1, [x5]
 274         orr     res0, res0, in0
 275         orr     res1, res1, in1
 276         stxp    w4, res0, res1, [x5]
 277         cbnz    w4, 1b
 278         ret
 279 2:
 280         ldaxp   res0, res1, [x5]
 281         orr     res0, res0, in0
 282         orr     res1, res1, in1
 283         stlxp   w4, res0, res1, [x5]
 284         cbnz    w4, 2b
 285         ret
 286 END (libat_or_fetch_16_i1)
 287
 288
 289 ENTRY (libat_fetch_and_16_i1)
 290         mov     x5, x0
 291         cbnz    w4, 2f
 292 1:
 293         ldxp    res0, res1, [x5]
 294         and     tmp0, res0, in0
 295         and     tmp1, res1, in1
 296         stxp    w4, tmp0, tmp1, [x5]
 297         cbnz    w4, 1b
 298         ret
 299 2:
 300         ldaxp   res0, res1, [x5]
 301         and     tmp0, res0, in0
 302         and     tmp1, res1, in1
 303         stlxp   w4, tmp0, tmp1, [x5]
 304         cbnz    w4, 2b
 305         ret
 306 END (libat_fetch_and_16_i1)
 307
 308
 309 ENTRY (libat_and_fetch_16_i1)
 310         mov     x5, x0
 311         cbnz    w4, 2f
 312 1:
 313         ldxp    res0, res1, [x5]
 314         and     res0, res0, in0
 315         and     res1, res1, in1
 316         stxp    w4, res0, res1, [x5]
 317         cbnz    w4, 1b
 318         ret
 319 2:
 320         ldaxp   res0, res1, [x5]
 321         and     res0, res0, in0
 322         and     res1, res1, in1
 323         stlxp   w4, res0, res1, [x5]
 324         cbnz    w4, 2b
 325         ret
 326 END (libat_and_fetch_16_i1)
 327
 328
 329 ENTRY (libat_fetch_xor_16_i1)
 330         mov     x5, x0
 331         cbnz    w4, 2f
 332 1:
 333         ldxp    res0, res1, [x5]
 334         eor     tmp0, res0, in0
 335         eor     tmp1, res1, in1
 336         stxp    w4, tmp0, tmp1, [x5]
 337         cbnz    w4, 1b
 338         ret
 339 2:
 340         ldaxp   res0, res1, [x5]
 341         eor     tmp0, res0, in0
 342         eor     tmp1, res1, in1
 343         stlxp   w4, tmp0, tmp1, [x5]
 344         cbnz    w4, 2b
 345         ret
 346 END (libat_fetch_xor_16_i1)
 347
 348
 349 ENTRY (libat_xor_fetch_16_i1)
 350         mov     x5, x0
 351         cbnz    w4, 2f
 352 1:
 353         ldxp    res0, res1, [x5]
 354         eor     res0, res0, in0
 355         eor     res1, res1, in1
 356         stxp    w4, res0, res1, [x5]
 357         cbnz    w4, 1b
 358         ret
 359 2:
 360         ldaxp   res0, res1, [x5]
 361         eor     res0, res0, in0
 362         eor     res1, res1, in1
 363         stlxp   w4, res0, res1, [x5]
 364         cbnz    w4, 2b
 365         ret
 366 END (libat_xor_fetch_16_i1)
 367
 368
 369 ENTRY (libat_fetch_nand_16_i1)
 370         mov     x5, x0
 371         mvn     in0, in0
 372         mvn     in1, in1
 373         cbnz    w4, 2f
 374 1:
 375         ldxp    res0, res1, [x5]
 376         orn     tmp0, in0, res0
 377         orn     tmp1, in1, res1
 378         stxp    w4, tmp0, tmp1, [x5]
 379         cbnz    w4, 1b
 380         ret
 381 2:
 382         ldaxp   res0, res1, [x5]
 383         orn     tmp0, in0, res0
 384         orn     tmp1, in1, res1
 385         stlxp   w4, tmp0, tmp1, [x5]
 386         cbnz    w4, 2b
 387         ret
 388 END (libat_fetch_nand_16_i1)
 389
 390
 391 ENTRY (libat_nand_fetch_16_i1)
 392         mov     x5, x0
 393         mvn     in0, in0
 394         mvn     in1, in1
 395         cbnz    w4, 2f
 396 1:
 397         ldxp    res0, res1, [x5]
 398         orn     res0, in0, res0
 399         orn     res1, in1, res1
 400         stxp    w4, res0, res1, [x5]
 401         cbnz    w4, 1b
 402         ret
 403 2:
 404         ldaxp   res0, res1, [x5]
 405         orn     res0, in0, res0
 406         orn     res1, in1, res1
 407         stlxp   w4, res0, res1, [x5]
 408         cbnz    w4, 2b
 409         ret
 410 END (libat_nand_fetch_16_i1)
 411
 412
 413 ENTRY (libat_test_and_set_16_i1)
 414         mov     w2, 1
 415         cbnz    w1, 2f
 416         swpb    w0, w2, [x0]
 417         ret
 418
 419 2:      swpalb  w0, w2, [x0]
 420         ret
 421 END (libat_test_and_set_16_i1)
 422
 423
 424 /* GNU_PROPERTY_AARCH64_* macros from elf.h for use in asm code.  */
 425 #define FEATURE_1_AND 0xc0000000
 426 #define FEATURE_1_BTI 1
 427 #define FEATURE_1_PAC 2
 428
 429 /* Supported features based on the code generation options.  */
 430 #if defined(__ARM_FEATURE_BTI_DEFAULT)
 431 # define BTI_FLAG FEATURE_1_BTI
 432 #else
 433 # define BTI_FLAG 0
 434 #endif
 435
 436 #if __ARM_FEATURE_PAC_DEFAULT & 3
 437 # define PAC_FLAG FEATURE_1_PAC
 438 #else
 439 # define PAC_FLAG 0
 440 #endif
 441
 442 /* Add a NT_GNU_PROPERTY_TYPE_0 note.  */
 443 #define GNU_PROPERTY(type, value)       \
 444   .section .note.gnu.property, "a";     \
 445   .p2align 3;                           \
 446   .word 4;                              \
 447   .word 16;                             \
 448   .word 5;                              \
 449   .asciz "GNU";                         \
 450   .word type;                           \
 451   .word 4;                              \
 452   .word value;                          \
 453   .word 0;
 454
 455 #if defined(__linux__) || defined(__FreeBSD__)
 456 .section .note.GNU-stack, "", %progbits
 457
 458 /* Add GNU property note if built with branch protection.  */
 459 # if (BTI_FLAG|PAC_FLAG) != 0
 460 GNU_PROPERTY (FEATURE_1_AND, BTI_FLAG|PAC_FLAG)
 461 # endif
 462 #endif