arch/arm/mm/proc-macros.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * We need constants.h for:
   4  *  VMA_VM_MM
   5  *  VMA_VM_FLAGS
   6  *  VM_EXEC
   7  */
   8 #include <asm/asm-offsets.h>
   9 #include <asm/thread_info.h>
  10
  11 #ifdef CONFIG_CPU_V7M
  12 #include <asm/v7m.h>
  13 #endif
  14
  15 /*
  16  * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm)
  17  */
  18         .macro  vma_vm_mm, rd, rn
  19         ldr     \rd, [\rn, #VMA_VM_MM]
  20         .endm
  21
  22 /*
  23  * vma_vm_flags - get vma->vm_flags
  24  */
  25         .macro  vma_vm_flags, rd, rn
  26         ldr     \rd, [\rn, #VMA_VM_FLAGS]
  27         .endm
  28
  29 /*
  30  * act_mm - get current->active_mm
  31  */
  32         .macro  act_mm, rd
  33         bic     \rd, sp, #8128
  34         bic     \rd, \rd, #63
  35         ldr     \rd, [\rd, #TI_TASK]
  36         .if (TSK_ACTIVE_MM > IMM12_MASK)
  37         add     \rd, \rd, #TSK_ACTIVE_MM & ~IMM12_MASK
  38         .endif
  39         ldr     \rd, [\rd, #TSK_ACTIVE_MM & IMM12_MASK]
  40         .endm
  41
  42 /*
  43  * mmid - get context id from mm pointer (mm->context.id)
  44  * note, this field is 64bit, so in big-endian the two words are swapped too.
  45  */
  46         .macro  mmid, rd, rn
  47 #ifdef __ARMEB__
  48         ldr     \rd, [\rn, #MM_CONTEXT_ID + 4 ]
  49 #else
  50         ldr     \rd, [\rn, #MM_CONTEXT_ID]
  51 #endif
  52         .endm
  53
  54 /*
  55  * mask_asid - mask the ASID from the context ID
  56  */
  57         .macro  asid, rd, rn
  58         and     \rd, \rn, #255
  59         .endm
  60
  61         .macro  crval, clear, mmuset, ucset
  62 #ifdef CONFIG_MMU
  63         .word   \clear
  64         .word   \mmuset
  65 #else
  66         .word   \clear
  67         .word   \ucset
  68 #endif
  69         .endm
  70
  71 /*
  72  * dcache_line_size - get the minimum D-cache line size from the CTR register
  73  * on ARMv7.
  74  */
  75         .macro  dcache_line_size, reg, tmp
  76 #ifdef CONFIG_CPU_V7M
  77         movw    \tmp, #:lower16:BASEADDR_V7M_SCB + V7M_SCB_CTR
  78         movt    \tmp, #:upper16:BASEADDR_V7M_SCB + V7M_SCB_CTR
  79         ldr     \tmp, [\tmp]
  80 #else
  81         mrc     p15, 0, \tmp, c0, c0, 1         @ read ctr
  82 #endif
  83         lsr     \tmp, \tmp, #16
  84         and     \tmp, \tmp, #0xf                @ cache line size encoding
  85         mov     \reg, #4                        @ bytes per word
  86         mov     \reg, \reg, lsl \tmp            @ actual cache line size
  87         .endm
  88
  89 /*
  90  * icache_line_size - get the minimum I-cache line size from the CTR register
  91  * on ARMv7.
  92  */
  93         .macro  icache_line_size, reg, tmp
  94 #ifdef CONFIG_CPU_V7M
  95         movw    \tmp, #:lower16:BASEADDR_V7M_SCB + V7M_SCB_CTR
  96         movt    \tmp, #:upper16:BASEADDR_V7M_SCB + V7M_SCB_CTR
  97         ldr     \tmp, [\tmp]
  98 #else
  99         mrc     p15, 0, \tmp, c0, c0, 1         @ read ctr
 100 #endif
 101         and     \tmp, \tmp, #0xf                @ cache line size encoding
 102         mov     \reg, #4                        @ bytes per word
 103         mov     \reg, \reg, lsl \tmp            @ actual cache line size
 104         .endm
 105
 106 /*
 107  * Sanity check the PTE configuration for the code below - which makes
 108  * certain assumptions about how these bits are laid out.
 109  */
 110 #ifdef CONFIG_MMU
 111 #if L_PTE_SHARED != PTE_EXT_SHARED
 112 #error PTE shared bit mismatch
 113 #endif
 114 #if !defined (CONFIG_ARM_LPAE) && \
 115         (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\
 116          L_PTE_PRESENT) > L_PTE_SHARED
 117 #error Invalid Linux PTE bit settings
 118 #endif
 119 #endif  /* CONFIG_MMU */
 120
 121 /*
 122  * The ARMv6 and ARMv7 set_pte_ext translation function.
 123  *
 124  * Permission translation:
 125  *  YUWD  APX AP1 AP0   SVC     User
 126  *  0xxx   0   0   0    no acc  no acc
 127  *  100x   1   0   1    r/o     no acc
 128  *  10x0   1   0   1    r/o     no acc
 129  *  1011   0   0   1    r/w     no acc
 130  *  110x   1   1   1    r/o     r/o
 131  *  11x0   1   1   1    r/o     r/o
 132  *  1111   0   1   1    r/w     r/w
 133  */
 134         .macro  armv6_mt_table pfx
 135 \pfx\()_mt_table:
 136         .long   0x00                                            @ L_PTE_MT_UNCACHED
 137         .long   PTE_EXT_TEX(1)                                  @ L_PTE_MT_BUFFERABLE
 138         .long   PTE_CACHEABLE                                   @ L_PTE_MT_WRITETHROUGH
 139         .long   PTE_CACHEABLE | PTE_BUFFERABLE                  @ L_PTE_MT_WRITEBACK
 140         .long   PTE_BUFFERABLE                                  @ L_PTE_MT_DEV_SHARED
 141         .long   0x00                                            @ unused
 142         .long   0x00                                            @ L_PTE_MT_MINICACHE (not present)
 143         .long   PTE_EXT_TEX(1) | PTE_CACHEABLE | PTE_BUFFERABLE @ L_PTE_MT_WRITEALLOC
 144         .long   0x00                                            @ unused
 145         .long   PTE_EXT_TEX(1)                                  @ L_PTE_MT_DEV_WC
 146         .long   0x00                                            @ unused
 147         .long   PTE_CACHEABLE | PTE_BUFFERABLE                  @ L_PTE_MT_DEV_CACHED
 148         .long   PTE_EXT_TEX(2)                                  @ L_PTE_MT_DEV_NONSHARED
 149         .long   0x00                                            @ unused
 150         .long   0x00                                            @ unused
 151         .long   PTE_CACHEABLE | PTE_BUFFERABLE | PTE_EXT_APX    @ L_PTE_MT_VECTORS
 152         .endm
 153
 154         .macro  armv6_set_pte_ext pfx
 155         str     r1, [r0], #2048                 @ linux version
 156
 157         bic     r3, r1, #0x000003fc
 158         bic     r3, r3, #PTE_TYPE_MASK
 159         orr     r3, r3, r2
 160         orr     r3, r3, #PTE_EXT_AP0 | 2
 161
 162         adr     ip, \pfx\()_mt_table
 163         and     r2, r1, #L_PTE_MT_MASK
 164         ldr     r2, [ip, r2]
 165
 166         eor     r1, r1, #L_PTE_DIRTY
 167         tst     r1, #L_PTE_DIRTY|L_PTE_RDONLY
 168         orrne   r3, r3, #PTE_EXT_APX
 169
 170         tst     r1, #L_PTE_USER
 171         orrne   r3, r3, #PTE_EXT_AP1
 172         tstne   r3, #PTE_EXT_APX
 173
 174         @ user read-only -> kernel read-only
 175         bicne   r3, r3, #PTE_EXT_AP0
 176
 177         tst     r1, #L_PTE_XN
 178         orrne   r3, r3, #PTE_EXT_XN
 179
 180         eor     r3, r3, r2
 181
 182         tst     r1, #L_PTE_YOUNG
 183         tstne   r1, #L_PTE_PRESENT
 184         moveq   r3, #0
 185         tstne   r1, #L_PTE_NONE
 186         movne   r3, #0
 187
 188         str     r3, [r0]
 189         mcr     p15, 0, r0, c7, c10, 1          @ flush_pte
 190         .endm
 191
 192
 193 /*
 194  * The ARMv3, ARMv4 and ARMv5 set_pte_ext translation function,
 195  * covering most CPUs except Xscale and Xscale 3.
 196  *
 197  * Permission translation:
 198  *  YUWD   AP   SVC     User
 199  *  0xxx  0x00  no acc  no acc
 200  *  100x  0x00  r/o     no acc
 201  *  10x0  0x00  r/o     no acc
 202  *  1011  0x55  r/w     no acc
 203  *  110x  0xaa  r/w     r/o
 204  *  11x0  0xaa  r/w     r/o
 205  *  1111  0xff  r/w     r/w
 206  */
 207         .macro  armv3_set_pte_ext wc_disable=1
 208         str     r1, [r0], #2048                 @ linux version
 209
 210         eor     r3, r1, #L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY
 211
 212         bic     r2, r1, #PTE_SMALL_AP_MASK      @ keep C, B bits
 213         bic     r2, r2, #PTE_TYPE_MASK
 214         orr     r2, r2, #PTE_TYPE_SMALL
 215
 216         tst     r3, #L_PTE_USER                 @ user?
 217         orrne   r2, r2, #PTE_SMALL_AP_URO_SRW
 218
 219         tst     r3, #L_PTE_RDONLY | L_PTE_DIRTY @ write and dirty?
 220         orreq   r2, r2, #PTE_SMALL_AP_UNO_SRW
 221
 222         tst     r3, #L_PTE_PRESENT | L_PTE_YOUNG        @ present and young?
 223         movne   r2, #0
 224
 225         .if     \wc_disable
 226 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
 227         tst     r2, #PTE_CACHEABLE
 228         bicne   r2, r2, #PTE_BUFFERABLE
 229 #endif
 230         .endif
 231         str     r2, [r0]                @ hardware version
 232         .endm
 233
 234
 235 /*
 236  * Xscale set_pte_ext translation, split into two halves to cope
 237  * with work-arounds.  r3 must be preserved by code between these
 238  * two macros.
 239  *
 240  * Permission translation:
 241  *  YUWD  AP    SVC     User
 242  *  0xxx  00    no acc  no acc
 243  *  100x  00    r/o     no acc
 244  *  10x0  00    r/o     no acc
 245  *  1011  01    r/w     no acc
 246  *  110x  10    r/w     r/o
 247  *  11x0  10    r/w     r/o
 248  *  1111  11    r/w     r/w
 249  */
 250         .macro  xscale_set_pte_ext_prologue
 251         str     r1, [r0]                        @ linux version
 252
 253         eor     r3, r1, #L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY
 254
 255         bic     r2, r1, #PTE_SMALL_AP_MASK      @ keep C, B bits
 256         orr     r2, r2, #PTE_TYPE_EXT           @ extended page
 257
 258         tst     r3, #L_PTE_USER                 @ user?
 259         orrne   r2, r2, #PTE_EXT_AP_URO_SRW     @ yes -> user r/o, system r/w
 260
 261         tst     r3, #L_PTE_RDONLY | L_PTE_DIRTY @ write and dirty?
 262         orreq   r2, r2, #PTE_EXT_AP_UNO_SRW     @ yes -> user n/a, system r/w
 263                                                 @ combined with user -> user r/w
 264         .endm
 265
 266         .macro  xscale_set_pte_ext_epilogue
 267         tst     r3, #L_PTE_PRESENT | L_PTE_YOUNG        @ present and young?
 268         movne   r2, #0                          @ no -> fault
 269
 270         str     r2, [r0, #2048]!                @ hardware version
 271         mov     ip, #0
 272         mcr     p15, 0, r0, c7, c10, 1          @ clean L1 D line
 273         mcr     p15, 0, ip, c7, c10, 4          @ data write barrier
 274         .endm
 275
 276 .macro define_processor_functions name:req, dabort:req, pabort:req, nommu=0, suspend=0, bugs=0
 277 /*
 278  * If we are building for big.Little with branch predictor hardening,
 279  * we need the processor function tables to remain available after boot.
 280  */
 281 #if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)
 282         .section ".rodata"
 283 #endif
 284         .type   \name\()_processor_functions, #object
 285         .align 2
 286 ENTRY(\name\()_processor_functions)
 287         .word   \dabort
 288         .word   \pabort
 289         .word   cpu_\name\()_proc_init
 290         .word   \bugs
 291         .word   cpu_\name\()_proc_fin
 292         .word   cpu_\name\()_reset
 293         .word   cpu_\name\()_do_idle
 294         .word   cpu_\name\()_dcache_clean_area
 295         .word   cpu_\name\()_switch_mm
 296
 297         .if \nommu
 298         .word   0
 299         .else
 300         .word   cpu_\name\()_set_pte_ext
 301         .endif
 302
 303         .if \suspend
 304         .word   cpu_\name\()_suspend_size
 305 #ifdef CONFIG_ARM_CPU_SUSPEND
 306         .word   cpu_\name\()_do_suspend
 307         .word   cpu_\name\()_do_resume
 308 #else
 309         .word   0
 310         .word   0
 311 #endif
 312         .else
 313         .word   0
 314         .word   0
 315         .word   0
 316         .endif
 317
 318         .size   \name\()_processor_functions, . - \name\()_processor_functions
 319 #if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)
 320         .previous
 321 #endif
 322 .endm
 323
 324 .macro define_cache_functions name:req
 325         .align 2
 326         .type   \name\()_cache_fns, #object
 327 ENTRY(\name\()_cache_fns)
 328         .long   \name\()_flush_icache_all
 329         .long   \name\()_flush_kern_cache_all
 330         .long   \name\()_flush_kern_cache_louis
 331         .long   \name\()_flush_user_cache_all
 332         .long   \name\()_flush_user_cache_range
 333         .long   \name\()_coherent_kern_range
 334         .long   \name\()_coherent_user_range
 335         .long   \name\()_flush_kern_dcache_area
 336         .long   \name\()_dma_map_area
 337         .long   \name\()_dma_unmap_area
 338         .long   \name\()_dma_flush_range
 339         .size   \name\()_cache_fns, . - \name\()_cache_fns
 340 .endm
 341
 342 .macro define_tlb_functions name:req, flags_up:req, flags_smp
 343         .type   \name\()_tlb_fns, #object
 344 ENTRY(\name\()_tlb_fns)
 345         .long   \name\()_flush_user_tlb_range
 346         .long   \name\()_flush_kern_tlb_range
 347         .ifnb \flags_smp
 348                 ALT_SMP(.long   \flags_smp )
 349                 ALT_UP(.long    \flags_up )
 350         .else
 351                 .long   \flags_up
 352         .endif
 353         .size   \name\()_tlb_fns, . - \name\()_tlb_fns
 354 .endm
 355
 356 .macro globl_equ x, y
 357         .globl  \x
 358         .equ    \x, \y
 359 .endm
 360
 361 .macro  initfn, func, base
 362         .long   \func - \base
 363 .endm
 364
 365         /*
 366          * Macro to calculate the log2 size for the protection region
 367          * registers. This calculates rd = log2(size) - 1.  tmp must
 368          * not be the same register as rd.
 369          */
 370 .macro  pr_sz, rd, size, tmp
 371         mov     \tmp, \size, lsr #12
 372         mov     \rd, #11
 373 1:      movs    \tmp, \tmp, lsr #1
 374         addne   \rd, \rd, #1
 375         bne     1b
 376 .endm
 377
 378         /*
 379          * Macro to generate a protection region register value
 380          * given a pre-masked address, size, and enable bit.
 381          * Corrupts size.
 382          */
 383 .macro  pr_val, dest, addr, size, enable
 384         pr_sz   \dest, \size, \size             @ calculate log2(size) - 1
 385         orr     \dest, \addr, \dest, lsl #1     @ mask in the region size
 386         orr     \dest, \dest, \enable
 387 .endm