arch/arm64/include/asm/assembler.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S
   4  *
   5  * Copyright (C) 1996-2000 Russell King
   6  * Copyright (C) 2012 ARM Ltd.
   7  */
   8 #ifndef __ASSEMBLY__
   9 #error "Only include this from assembly code"
  10 #endif
  11
  12 #ifndef __ASM_ASSEMBLER_H
  13 #define __ASM_ASSEMBLER_H
  14
  15 #include <asm-generic/export.h>
  16
  17 #include <asm/asm-offsets.h>
  18 #include <asm/cpufeature.h>
  19 #include <asm/cputype.h>
  20 #include <asm/debug-monitors.h>
  21 #include <asm/page.h>
  22 #include <asm/pgtable-hwdef.h>
  23 #include <asm/ptrace.h>
  24 #include <asm/thread_info.h>
  25
  26         .macro save_and_disable_daif, flags
  27         mrs     \flags, daif
  28         msr     daifset, #0xf
  29         .endm
  30
  31         .macro disable_daif
  32         msr     daifset, #0xf
  33         .endm
  34
  35         .macro enable_daif
  36         msr     daifclr, #0xf
  37         .endm
  38
  39         .macro  restore_daif, flags:req
  40         msr     daif, \flags
  41         .endm
  42
  43         /* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */
  44         .macro  inherit_daif, pstate:req, tmp:req
  45         and     \tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
  46         msr     daif, \tmp
  47         .endm
  48
  49         /* IRQ is the lowest priority flag, unconditionally unmask the rest. */
  50         .macro enable_da_f
  51         msr     daifclr, #(8 | 4 | 1)
  52         .endm
  53
  54 /*
  55  * Save/restore interrupts.
  56  */
  57         .macro  save_and_disable_irq, flags
  58         mrs     \flags, daif
  59         msr     daifset, #2
  60         .endm
  61
  62         .macro  restore_irq, flags
  63         msr     daif, \flags
  64         .endm
  65
  66         .macro  enable_dbg
  67         msr     daifclr, #8
  68         .endm
  69
  70         .macro  disable_step_tsk, flgs, tmp
  71         tbz     \flgs, #TIF_SINGLESTEP, 9990f
  72         mrs     \tmp, mdscr_el1
  73         bic     \tmp, \tmp, #DBG_MDSCR_SS
  74         msr     mdscr_el1, \tmp
  75         isb     // Synchronise with enable_dbg
  76 9990:
  77         .endm
  78
  79         /* call with daif masked */
  80         .macro  enable_step_tsk, flgs, tmp
  81         tbz     \flgs, #TIF_SINGLESTEP, 9990f
  82         mrs     \tmp, mdscr_el1
  83         orr     \tmp, \tmp, #DBG_MDSCR_SS
  84         msr     mdscr_el1, \tmp
  85 9990:
  86         .endm
  87
  88 /*
  89  * SMP data memory barrier
  90  */
  91         .macro  smp_dmb, opt
  92         dmb     \opt
  93         .endm
  94
  95 /*
  96  * RAS Error Synchronization barrier
  97  */
  98         .macro  esb
  99         hint    #16
 100         .endm
 101
 102 /*
 103  * Value prediction barrier
 104  */
 105         .macro  csdb
 106         hint    #20
 107         .endm
 108
 109 /*
 110  * Speculation barrier
 111  */
 112         .macro  sb
 113 alternative_if_not ARM64_HAS_SB
 114         dsb     nsh
 115         isb
 116 alternative_else
 117         SB_BARRIER_INSN
 118         nop
 119 alternative_endif
 120         .endm
 121
 122 /*
 123  * Sanitise a 64-bit bounded index wrt speculation, returning zero if out
 124  * of bounds.
 125  */
 126         .macro  mask_nospec64, idx, limit, tmp
 127         sub     \tmp, \idx, \limit
 128         bic     \tmp, \tmp, \idx
 129         and     \idx, \idx, \tmp, asr #63
 130         csdb
 131         .endm
 132
 133 /*
 134  * NOP sequence
 135  */
 136         .macro  nops, num
 137         .rept   \num
 138         nop
 139         .endr
 140         .endm
 141
 142 /*
 143  * Emit an entry into the exception table
 144  */
 145         .macro          _asm_extable, from, to
 146         .pushsection    __ex_table, "a"
 147         .align          3
 148         .long           (\from - .), (\to - .)
 149         .popsection
 150         .endm
 151
 152 #define USER(l, x...)                           \
 153 9999:   x;                                      \
 154         _asm_extable    9999b, l
 155
 156 /*
 157  * Register aliases.
 158  */
 159 lr      .req    x30             // link register
 160
 161 /*
 162  * Vector entry
 163  */
 164          .macro ventry  label
 165         .align  7
 166         b       \label
 167         .endm
 168
 169 /*
 170  * Select code when configured for BE.
 171  */
 172 #ifdef CONFIG_CPU_BIG_ENDIAN
 173 #define CPU_BE(code...) code
 174 #else
 175 #define CPU_BE(code...)
 176 #endif
 177
 178 /*
 179  * Select code when configured for LE.
 180  */
 181 #ifdef CONFIG_CPU_BIG_ENDIAN
 182 #define CPU_LE(code...)
 183 #else
 184 #define CPU_LE(code...) code
 185 #endif
 186
 187 /*
 188  * Define a macro that constructs a 64-bit value by concatenating two
 189  * 32-bit registers. Note that on big endian systems the order of the
 190  * registers is swapped.
 191  */
 192 #ifndef CONFIG_CPU_BIG_ENDIAN
 193         .macro  regs_to_64, rd, lbits, hbits
 194 #else
 195         .macro  regs_to_64, rd, hbits, lbits
 196 #endif
 197         orr     \rd, \lbits, \hbits, lsl #32
 198         .endm
 199
 200 /*
 201  * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where
 202  * <symbol> is within the range +/- 4 GB of the PC.
 203  */
 204         /*
 205          * @dst: destination register (64 bit wide)
 206          * @sym: name of the symbol
 207          */
 208         .macro  adr_l, dst, sym
 209         adrp    \dst, \sym
 210         add     \dst, \dst, :lo12:\sym
 211         .endm
 212
 213         /*
 214          * @dst: destination register (32 or 64 bit wide)
 215          * @sym: name of the symbol
 216          * @tmp: optional 64-bit scratch register to be used if <dst> is a
 217          *       32-bit wide register, in which case it cannot be used to hold
 218          *       the address
 219          */
 220         .macro  ldr_l, dst, sym, tmp=
 221         .ifb    \tmp
 222         adrp    \dst, \sym
 223         ldr     \dst, [\dst, :lo12:\sym]
 224         .else
 225         adrp    \tmp, \sym
 226         ldr     \dst, [\tmp, :lo12:\sym]
 227         .endif
 228         .endm
 229
 230         /*
 231          * @src: source register (32 or 64 bit wide)
 232          * @sym: name of the symbol
 233          * @tmp: mandatory 64-bit scratch register to calculate the address
 234          *       while <src> needs to be preserved.
 235          */
 236         .macro  str_l, src, sym, tmp
 237         adrp    \tmp, \sym
 238         str     \src, [\tmp, :lo12:\sym]
 239         .endm
 240
 241         /*
 242          * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP)
 243          * @sym: The name of the per-cpu variable
 244          * @tmp: scratch register
 245          */
 246         .macro adr_this_cpu, dst, sym, tmp
 247         adrp    \tmp, \sym
 248         add     \dst, \tmp, #:lo12:\sym
 249 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
 250         mrs     \tmp, tpidr_el1
 251 alternative_else
 252         mrs     \tmp, tpidr_el2
 253 alternative_endif
 254         add     \dst, \dst, \tmp
 255         .endm
 256
 257         /*
 258          * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id()))
 259          * @sym: The name of the per-cpu variable
 260          * @tmp: scratch register
 261          */
 262         .macro ldr_this_cpu dst, sym, tmp
 263         adr_l   \dst, \sym
 264 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
 265         mrs     \tmp, tpidr_el1
 266 alternative_else
 267         mrs     \tmp, tpidr_el2
 268 alternative_endif
 269         ldr     \dst, [\dst, \tmp]
 270         .endm
 271
 272 /*
 273  * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm)
 274  */
 275         .macro  vma_vm_mm, rd, rn
 276         ldr     \rd, [\rn, #VMA_VM_MM]
 277         .endm
 278
 279 /*
 280  * mmid - get context id from mm pointer (mm->context.id)
 281  */
 282         .macro  mmid, rd, rn
 283         ldr     \rd, [\rn, #MM_CONTEXT_ID]
 284         .endm
 285 /*
 286  * read_ctr - read CTR_EL0. If the system has mismatched register fields,
 287  * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val
 288  */
 289         .macro  read_ctr, reg
 290 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE
 291         mrs     \reg, ctr_el0                   // read CTR
 292         nop
 293 alternative_else
 294         ldr_l   \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL
 295 alternative_endif
 296         .endm
 297
 298
 299 /*
 300  * raw_dcache_line_size - get the minimum D-cache line size on this CPU
 301  * from the CTR register.
 302  */
 303         .macro  raw_dcache_line_size, reg, tmp
 304         mrs     \tmp, ctr_el0                   // read CTR
 305         ubfm    \tmp, \tmp, #16, #19            // cache line size encoding
 306         mov     \reg, #4                        // bytes per word
 307         lsl     \reg, \reg, \tmp                // actual cache line size
 308         .endm
 309
 310 /*
 311  * dcache_line_size - get the safe D-cache line size across all CPUs
 312  */
 313         .macro  dcache_line_size, reg, tmp
 314         read_ctr        \tmp
 315         ubfm            \tmp, \tmp, #16, #19    // cache line size encoding
 316         mov             \reg, #4                // bytes per word
 317         lsl             \reg, \reg, \tmp        // actual cache line size
 318         .endm
 319
 320 /*
 321  * raw_icache_line_size - get the minimum I-cache line size on this CPU
 322  * from the CTR register.
 323  */
 324         .macro  raw_icache_line_size, reg, tmp
 325         mrs     \tmp, ctr_el0                   // read CTR
 326         and     \tmp, \tmp, #0xf                // cache line size encoding
 327         mov     \reg, #4                        // bytes per word
 328         lsl     \reg, \reg, \tmp                // actual cache line size
 329         .endm
 330
 331 /*
 332  * icache_line_size - get the safe I-cache line size across all CPUs
 333  */
 334         .macro  icache_line_size, reg, tmp
 335         read_ctr        \tmp
 336         and             \tmp, \tmp, #0xf        // cache line size encoding
 337         mov             \reg, #4                // bytes per word
 338         lsl             \reg, \reg, \tmp        // actual cache line size
 339         .endm
 340
 341 /*
 342  * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map
 343  */
 344         .macro  tcr_set_t0sz, valreg, t0sz
 345         bfi     \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
 346         .endm
 347
 348 /*
 349  * tcr_compute_pa_size - set TCR.(I)PS to the highest supported
 350  * ID_AA64MMFR0_EL1.PARange value
 351  *
 352  *      tcr:            register with the TCR_ELx value to be updated
 353  *      pos:            IPS or PS bitfield position
 354  *      tmp{0,1}:       temporary registers
 355  */
 356         .macro  tcr_compute_pa_size, tcr, pos, tmp0, tmp1
 357         mrs     \tmp0, ID_AA64MMFR0_EL1
 358         // Narrow PARange to fit the PS field in TCR_ELx
 359         ubfx    \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3
 360         mov     \tmp1, #ID_AA64MMFR0_PARANGE_MAX
 361         cmp     \tmp0, \tmp1
 362         csel    \tmp0, \tmp1, \tmp0, hi
 363         bfi     \tcr, \tmp0, \pos, #3
 364         .endm
 365
 366 /*
 367  * Macro to perform a data cache maintenance for the interval
 368  * [kaddr, kaddr + size)
 369  *
 370  *      op:             operation passed to dc instruction
 371  *      domain:         domain used in dsb instruciton
 372  *      kaddr:          starting virtual address of the region
 373  *      size:           size of the region
 374  *      Corrupts:       kaddr, size, tmp1, tmp2
 375  */
 376         .macro __dcache_op_workaround_clean_cache, op, kaddr
 377 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
 378         dc      \op, \kaddr
 379 alternative_else
 380         dc      civac, \kaddr
 381 alternative_endif
 382         .endm
 383
 384         .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
 385         dcache_line_size \tmp1, \tmp2
 386         add     \size, \kaddr, \size
 387         sub     \tmp2, \tmp1, #1
 388         bic     \kaddr, \kaddr, \tmp2
 389 9998:
 390         .ifc    \op, cvau
 391         __dcache_op_workaround_clean_cache \op, \kaddr
 392         .else
 393         .ifc    \op, cvac
 394         __dcache_op_workaround_clean_cache \op, \kaddr
 395         .else
 396         .ifc    \op, cvap
 397         sys     3, c7, c12, 1, \kaddr   // dc cvap
 398         .else
 399         .ifc    \op, cvadp
 400         sys     3, c7, c13, 1, \kaddr   // dc cvadp
 401         .else
 402         dc      \op, \kaddr
 403         .endif
 404         .endif
 405         .endif
 406         .endif
 407         add     \kaddr, \kaddr, \tmp1
 408         cmp     \kaddr, \size
 409         b.lo    9998b
 410         dsb     \domain
 411         .endm
 412
 413 /*
 414  * Macro to perform an instruction cache maintenance for the interval
 415  * [start, end)
 416  *
 417  *      start, end:     virtual addresses describing the region
 418  *      label:          A label to branch to on user fault.
 419  *      Corrupts:       tmp1, tmp2
 420  */
 421         .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
 422         icache_line_size \tmp1, \tmp2
 423         sub     \tmp2, \tmp1, #1
 424         bic     \tmp2, \start, \tmp2
 425 9997:
 426 USER(\label, ic ivau, \tmp2)                    // invalidate I line PoU
 427         add     \tmp2, \tmp2, \tmp1
 428         cmp     \tmp2, \end
 429         b.lo    9997b
 430         dsb     ish
 431         isb
 432         .endm
 433
 434 /*
 435  * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
 436  */
 437         .macro  reset_pmuserenr_el0, tmpreg
 438         mrs     \tmpreg, id_aa64dfr0_el1
 439         sbfx    \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4
 440         cmp     \tmpreg, #1                     // Skip if no PMU present
 441         b.lt    9000f
 442         msr     pmuserenr_el0, xzr              // Disable PMU access from EL0
 443 9000:
 444         .endm
 445
 446 /*
 447  * copy_page - copy src to dest using temp registers t1-t8
 448  */
 449         .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
 450 9998:   ldp     \t1, \t2, [\src]
 451         ldp     \t3, \t4, [\src, #16]
 452         ldp     \t5, \t6, [\src, #32]
 453         ldp     \t7, \t8, [\src, #48]
 454         add     \src, \src, #64
 455         stnp    \t1, \t2, [\dest]
 456         stnp    \t3, \t4, [\dest, #16]
 457         stnp    \t5, \t6, [\dest, #32]
 458         stnp    \t7, \t8, [\dest, #48]
 459         add     \dest, \dest, #64
 460         tst     \src, #(PAGE_SIZE - 1)
 461         b.ne    9998b
 462         .endm
 463
 464 /*
 465  * Annotate a function as position independent, i.e., safe to be called before
 466  * the kernel virtual mapping is activated.
 467  */
 468 #define ENDPIPROC(x)                    \
 469         .globl  __pi_##x;               \
 470         .type   __pi_##x, %function;    \
 471         .set    __pi_##x, x;            \
 472         .size   __pi_##x, . - x;        \
 473         ENDPROC(x)
 474
 475 /*
 476  * Annotate a function as being unsuitable for kprobes.
 477  */
 478 #ifdef CONFIG_KPROBES
 479 #define NOKPROBE(x)                             \
 480         .pushsection "_kprobe_blacklist", "aw"; \
 481         .quad   x;                              \
 482         .popsection;
 483 #else
 484 #define NOKPROBE(x)
 485 #endif
 486
 487 #ifdef CONFIG_KASAN
 488 #define EXPORT_SYMBOL_NOKASAN(name)
 489 #else
 490 #define EXPORT_SYMBOL_NOKASAN(name)     EXPORT_SYMBOL(name)
 491 #endif
 492
 493         /*
 494          * Emit a 64-bit absolute little endian symbol reference in a way that
 495          * ensures that it will be resolved at build time, even when building a
 496          * PIE binary. This requires cooperation from the linker script, which
 497          * must emit the lo32/hi32 halves individually.
 498          */
 499         .macro  le64sym, sym
 500         .long   \sym\()_lo32
 501         .long   \sym\()_hi32
 502         .endm
 503
 504         /*
 505          * mov_q - move an immediate constant into a 64-bit register using
 506          *         between 2 and 4 movz/movk instructions (depending on the
 507          *         magnitude and sign of the operand)
 508          */
 509         .macro  mov_q, reg, val
 510         .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
 511         movz    \reg, :abs_g1_s:\val
 512         .else
 513         .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
 514         movz    \reg, :abs_g2_s:\val
 515         .else
 516         movz    \reg, :abs_g3:\val
 517         movk    \reg, :abs_g2_nc:\val
 518         .endif
 519         movk    \reg, :abs_g1_nc:\val
 520         .endif
 521         movk    \reg, :abs_g0_nc:\val
 522         .endm
 523
 524 /*
 525  * Return the current task_struct.
 526  */
 527         .macro  get_current_task, rd
 528         mrs     \rd, sp_el0
 529         .endm
 530
 531 /*
 532  * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD.
 533  * orr is used as it can cover the immediate value (and is idempotent).
 534  * In future this may be nop'ed out when dealing with 52-bit kernel VAs.
 535  *      ttbr: Value of ttbr to set, modified.
 536  */
 537         .macro  offset_ttbr1, ttbr
 538 #ifdef CONFIG_ARM64_USER_VA_BITS_52
 539         orr     \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
 540 #endif
 541         .endm
 542
 543 /*
 544  * Perform the reverse of offset_ttbr1.
 545  * bic is used as it can cover the immediate value and, in future, won't need
 546  * to be nop'ed out when dealing with 52-bit kernel VAs.
 547  */
 548         .macro  restore_ttbr1, ttbr
 549 #ifdef CONFIG_ARM64_USER_VA_BITS_52
 550         bic     \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
 551 #endif
 552         .endm
 553
 554 /*
 555  * Arrange a physical address in a TTBR register, taking care of 52-bit
 556  * addresses.
 557  *
 558  *      phys:   physical address, preserved
 559  *      ttbr:   returns the TTBR value
 560  */
 561         .macro  phys_to_ttbr, ttbr, phys
 562 #ifdef CONFIG_ARM64_PA_BITS_52
 563         orr     \ttbr, \phys, \phys, lsr #46
 564         and     \ttbr, \ttbr, #TTBR_BADDR_MASK_52
 565 #else
 566         mov     \ttbr, \phys
 567 #endif
 568         .endm
 569
 570         .macro  phys_to_pte, pte, phys
 571 #ifdef CONFIG_ARM64_PA_BITS_52
 572         /*
 573          * We assume \phys is 64K aligned and this is guaranteed by only
 574          * supporting this configuration with 64K pages.
 575          */
 576         orr     \pte, \phys, \phys, lsr #36
 577         and     \pte, \pte, #PTE_ADDR_MASK
 578 #else
 579         mov     \pte, \phys
 580 #endif
 581         .endm
 582
 583         .macro  pte_to_phys, phys, pte
 584 #ifdef CONFIG_ARM64_PA_BITS_52
 585         ubfiz   \phys, \pte, #(48 - 16 - 12), #16
 586         bfxil   \phys, \pte, #16, #32
 587         lsl     \phys, \phys, #16
 588 #else
 589         and     \phys, \pte, #PTE_ADDR_MASK
 590 #endif
 591         .endm
 592
 593 /*
 594  * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU.
 595  */
 596         .macro  tcr_clear_errata_bits, tcr, tmp1, tmp2
 597 #ifdef CONFIG_FUJITSU_ERRATUM_010001
 598         mrs     \tmp1, midr_el1
 599
 600         mov_q   \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK
 601         and     \tmp1, \tmp1, \tmp2
 602         mov_q   \tmp2, MIDR_FUJITSU_ERRATUM_010001
 603         cmp     \tmp1, \tmp2
 604         b.ne    10f
 605
 606         mov_q   \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001
 607         bic     \tcr, \tcr, \tmp2
 608 10:
 609 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */
 610         .endm
 611
 612 /**
 613  * Errata workaround prior to disable MMU. Insert an ISB immediately prior
 614  * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0.
 615  */
 616         .macro pre_disable_mmu_workaround
 617 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041
 618         isb
 619 #endif
 620         .endm
 621
 622         /*
 623          * frame_push - Push @regcount callee saved registers to the stack,
 624          *              starting at x19, as well as x29/x30, and set x29 to
 625          *              the new value of sp. Add @extra bytes of stack space
 626          *              for locals.
 627          */
 628         .macro          frame_push, regcount:req, extra
 629         __frame         st, \regcount, \extra
 630         .endm
 631
 632         /*
 633          * frame_pop  - Pop the callee saved registers from the stack that were
 634          *              pushed in the most recent call to frame_push, as well
 635          *              as x29/x30 and any extra stack space that may have been
 636          *              allocated.
 637          */
 638         .macro          frame_pop
 639         __frame         ld
 640         .endm
 641
 642         .macro          __frame_regs, reg1, reg2, op, num
 643         .if             .Lframe_regcount == \num
 644         \op\()r         \reg1, [sp, #(\num + 1) * 8]
 645         .elseif         .Lframe_regcount > \num
 646         \op\()p         \reg1, \reg2, [sp, #(\num + 1) * 8]
 647         .endif
 648         .endm
 649
 650         .macro          __frame, op, regcount, extra=0
 651         .ifc            \op, st
 652         .if             (\regcount) < 0 || (\regcount) > 10
 653         .error          "regcount should be in the range [0 ... 10]"
 654         .endif
 655         .if             ((\extra) % 16) != 0
 656         .error          "extra should be a multiple of 16 bytes"
 657         .endif
 658         .ifdef          .Lframe_regcount
 659         .if             .Lframe_regcount != -1
 660         .error          "frame_push/frame_pop may not be nested"
 661         .endif
 662         .endif
 663         .set            .Lframe_regcount, \regcount
 664         .set            .Lframe_extra, \extra
 665         .set            .Lframe_local_offset, ((\regcount + 3) / 2) * 16
 666         stp             x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]!
 667         mov             x29, sp
 668         .endif
 669
 670         __frame_regs    x19, x20, \op, 1
 671         __frame_regs    x21, x22, \op, 3
 672         __frame_regs    x23, x24, \op, 5
 673         __frame_regs    x25, x26, \op, 7
 674         __frame_regs    x27, x28, \op, 9
 675
 676         .ifc            \op, ld
 677         .if             .Lframe_regcount == -1
 678         .error          "frame_push/frame_pop may not be nested"
 679         .endif
 680         ldp             x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra
 681         .set            .Lframe_regcount, -1
 682         .endif
 683         .endm
 684
 685 /*
 686  * Check whether to yield to another runnable task from kernel mode NEON code
 687  * (which runs with preemption disabled).
 688  *
 689  * if_will_cond_yield_neon
 690  *        // pre-yield patchup code
 691  * do_cond_yield_neon
 692  *        // post-yield patchup code
 693  * endif_yield_neon    <label>
 694  *
 695  * where <label> is optional, and marks the point where execution will resume
 696  * after a yield has been performed. If omitted, execution resumes right after
 697  * the endif_yield_neon invocation. Note that the entire sequence, including
 698  * the provided patchup code, will be omitted from the image if CONFIG_PREEMPT
 699  * is not defined.
 700  *
 701  * As a convenience, in the case where no patchup code is required, the above
 702  * sequence may be abbreviated to
 703  *
 704  * cond_yield_neon <label>
 705  *
 706  * Note that the patchup code does not support assembler directives that change
 707  * the output section, any use of such directives is undefined.
 708  *
 709  * The yield itself consists of the following:
 710  * - Check whether the preempt count is exactly 1 and a reschedule is also
 711  *   needed. If so, calling of preempt_enable() in kernel_neon_end() will
 712  *   trigger a reschedule. If it is not the case, yielding is pointless.
 713  * - Disable and re-enable kernel mode NEON, and branch to the yield fixup
 714  *   code.
 715  *
 716  * This macro sequence may clobber all CPU state that is not guaranteed by the
 717  * AAPCS to be preserved across an ordinary function call.
 718  */
 719
 720         .macro          cond_yield_neon, lbl
 721         if_will_cond_yield_neon
 722         do_cond_yield_neon
 723         endif_yield_neon        \lbl
 724         .endm
 725
 726         .macro          if_will_cond_yield_neon
 727 #ifdef CONFIG_PREEMPT
 728         get_current_task        x0
 729         ldr             x0, [x0, #TSK_TI_PREEMPT]
 730         sub             x0, x0, #PREEMPT_DISABLE_OFFSET
 731         cbz             x0, .Lyield_\@
 732         /* fall through to endif_yield_neon */
 733         .subsection     1
 734 .Lyield_\@ :
 735 #else
 736         .section        ".discard.cond_yield_neon", "ax"
 737 #endif
 738         .endm
 739
 740         .macro          do_cond_yield_neon
 741         bl              kernel_neon_end
 742         bl              kernel_neon_begin
 743         .endm
 744
 745         .macro          endif_yield_neon, lbl
 746         .ifnb           \lbl
 747         b               \lbl
 748         .else
 749         b               .Lyield_out_\@
 750         .endif
 751         .previous
 752 .Lyield_out_\@ :
 753         .endm
 754
 755 #endif  /* __ASM_ASSEMBLER_H */