]>
git.ipfire.org Git - people/ms/u-boot.git/blob - arch/arm/cpu/armv8/cache_v8.c
3 * David Feng <fenghua@phytium.com.cn>
6 * Alexander Graf <agraf@suse.de>
8 * SPDX-License-Identifier: GPL-2.0+
12 #include <asm/system.h>
13 #include <asm/armv8/mmu.h>
15 DECLARE_GLOBAL_DATA_PTR
;
17 #ifndef CONFIG_SYS_DCACHE_OFF
20 * With 4k page granule, a virtual address is split into 4 lookup parts
21 * spanning 9 bits each:
23 * _______________________________________________
25 * | 0 | Lv0 | Lv1 | Lv2 | Lv3 | off |
26 * |_______|_______|_______|_______|_______|_______|
27 * 63-48 47-39 38-30 29-21 20-12 11-00
31 * Lv0: FF8000000000 --
38 u64
get_tcr(int el
, u64
*pips
, u64
*pva_bits
)
45 /* Find the largest address we need to support */
46 for (i
= 0; mem_map
[i
].size
|| mem_map
[i
].attrs
; i
++)
47 max_addr
= max(max_addr
, mem_map
[i
].virt
+ mem_map
[i
].size
);
49 /* Calculate the maximum physical (and thus virtual) address */
50 if (max_addr
> (1ULL << 44)) {
53 } else if (max_addr
> (1ULL << 42)) {
56 } else if (max_addr
> (1ULL << 40)) {
59 } else if (max_addr
> (1ULL << 36)) {
62 } else if (max_addr
> (1ULL << 32)) {
71 tcr
= TCR_EL1_RSVD
| (ips
<< 32) | TCR_EPD1_DISABLE
;
73 tcr
= TCR_EL2_RSVD
| (ips
<< 16);
75 tcr
= TCR_EL3_RSVD
| (ips
<< 16);
78 /* PTWs cacheable, inner/outer WBWA and inner shareable */
79 tcr
|= TCR_TG0_4K
| TCR_SHARED_INNER
| TCR_ORGN_WBWA
| TCR_IRGN_WBWA
;
80 tcr
|= TCR_T0SZ(va_bits
);
90 #define MAX_PTE_ENTRIES 512
92 static int pte_type(u64
*pte
)
94 return *pte
& PTE_TYPE_MASK
;
97 /* Returns the LSB number for a PTE on level <level> */
98 static int level2shift(int level
)
100 /* Page is 12 bits wide, every level translates 9 bits */
101 return (12 + 9 * (3 - level
));
104 static u64
*find_pte(u64 addr
, int level
)
112 debug("addr=%llx level=%d\n", addr
, level
);
114 get_tcr(0, NULL
, &va_bits
);
118 if (level
< start_level
)
121 /* Walk through all page table levels to find our PTE */
122 pte
= (u64
*)gd
->arch
.tlb_addr
;
123 for (i
= start_level
; i
< 4; i
++) {
124 idx
= (addr
>> level2shift(i
)) & 0x1FF;
126 debug("idx=%llx PTE %p at level %d: %llx\n", idx
, pte
, i
, *pte
);
131 /* PTE is no table (either invalid or block), can't traverse */
132 if (pte_type(pte
) != PTE_TYPE_TABLE
)
134 /* Off to the next level */
135 pte
= (u64
*)(*pte
& 0x0000fffffffff000ULL
);
138 /* Should never reach here */
142 /* Returns and creates a new full table (512 entries) */
143 static u64
*create_table(void)
145 u64
*new_table
= (u64
*)gd
->arch
.tlb_fillptr
;
146 u64 pt_len
= MAX_PTE_ENTRIES
* sizeof(u64
);
148 /* Allocate MAX_PTE_ENTRIES pte entries */
149 gd
->arch
.tlb_fillptr
+= pt_len
;
151 if (gd
->arch
.tlb_fillptr
- gd
->arch
.tlb_addr
> gd
->arch
.tlb_size
)
152 panic("Insufficient RAM for page table: 0x%lx > 0x%lx. "
153 "Please increase the size in get_page_table_size()",
154 gd
->arch
.tlb_fillptr
- gd
->arch
.tlb_addr
,
157 /* Mark all entries as invalid */
158 memset(new_table
, 0, pt_len
);
163 static void set_pte_table(u64
*pte
, u64
*table
)
165 /* Point *pte to the new table */
166 debug("Setting %p to addr=%p\n", pte
, table
);
167 *pte
= PTE_TYPE_TABLE
| (ulong
)table
;
170 /* Splits a block PTE into table with subpages spanning the old block */
171 static void split_block(u64
*pte
, int level
)
176 /* level describes the parent level, we need the child ones */
177 int levelshift
= level2shift(level
+ 1);
179 if (pte_type(pte
) != PTE_TYPE_BLOCK
)
180 panic("PTE %p (%llx) is not a block. Some driver code wants to "
181 "modify dcache settings for an range not covered in "
182 "mem_map.", pte
, old_pte
);
184 new_table
= create_table();
185 debug("Splitting pte %p (%llx) into %p\n", pte
, old_pte
, new_table
);
187 for (i
= 0; i
< MAX_PTE_ENTRIES
; i
++) {
188 new_table
[i
] = old_pte
| (i
<< levelshift
);
190 /* Level 3 block PTEs have the table type */
191 if ((level
+ 1) == 3)
192 new_table
[i
] |= PTE_TYPE_TABLE
;
194 debug("Setting new_table[%lld] = %llx\n", i
, new_table
[i
]);
197 /* Set the new table into effect */
198 set_pte_table(pte
, new_table
);
201 /* Add one mm_region map entry to the page tables */
202 static void add_map(struct mm_region
*map
)
205 u64 virt
= map
->virt
;
206 u64 phys
= map
->phys
;
207 u64 size
= map
->size
;
208 u64 attrs
= map
->attrs
| PTE_TYPE_BLOCK
| PTE_BLOCK_AF
;
214 pte
= find_pte(virt
, 0);
215 if (pte
&& (pte_type(pte
) == PTE_TYPE_FAULT
)) {
216 debug("Creating table for virt 0x%llx\n", virt
);
217 new_table
= create_table();
218 set_pte_table(pte
, new_table
);
221 for (level
= 1; level
< 4; level
++) {
222 pte
= find_pte(virt
, level
);
224 panic("pte not found\n");
226 blocksize
= 1ULL << level2shift(level
);
227 debug("Checking if pte fits for virt=%llx size=%llx blocksize=%llx\n",
228 virt
, size
, blocksize
);
229 if (size
>= blocksize
&& !(virt
& (blocksize
- 1))) {
230 /* Page fits, create block PTE */
231 debug("Setting PTE %p to block virt=%llx\n",
234 *pte
= phys
| attrs
| PTE_TYPE_PAGE
;
241 } else if (pte_type(pte
) == PTE_TYPE_FAULT
) {
242 /* Page doesn't fit, create subpages */
243 debug("Creating subtable for virt 0x%llx blksize=%llx\n",
245 new_table
= create_table();
246 set_pte_table(pte
, new_table
);
247 } else if (pte_type(pte
) == PTE_TYPE_BLOCK
) {
248 debug("Split block into subtable for virt 0x%llx blksize=0x%llx\n",
250 split_block(pte
, level
);
263 * This is a recursively called function to count the number of
264 * page tables we need to cover a particular PTE range. If you
265 * call this with level = -1 you basically get the full 48 bit
268 static int count_required_pts(u64 addr
, int level
, u64 maxaddr
)
270 int levelshift
= level2shift(level
);
271 u64 levelsize
= 1ULL << levelshift
;
272 u64 levelmask
= levelsize
- 1;
273 u64 levelend
= addr
+ levelsize
;
276 enum pte_type pte_type
= PTE_INVAL
;
278 for (i
= 0; mem_map
[i
].size
|| mem_map
[i
].attrs
; i
++) {
279 struct mm_region
*map
= &mem_map
[i
];
280 u64 start
= map
->virt
;
281 u64 end
= start
+ map
->size
;
283 /* Check if the PTE would overlap with the map */
284 if (max(addr
, start
) <= min(levelend
, end
)) {
285 start
= max(addr
, start
);
286 end
= min(levelend
, end
);
288 /* We need a sub-pt for this level */
289 if ((start
& levelmask
) || (end
& levelmask
)) {
290 pte_type
= PTE_LEVEL
;
294 /* Lv0 can not do block PTEs, so do levels here too */
296 pte_type
= PTE_LEVEL
;
300 /* PTE is active, but fits into a block */
301 pte_type
= PTE_BLOCK
;
306 * Block PTEs at this level are already covered by the parent page
307 * table, so we only need to count sub page tables.
309 if (pte_type
== PTE_LEVEL
) {
310 int sublevel
= level
+ 1;
311 u64 sublevelsize
= 1ULL << level2shift(sublevel
);
313 /* Account for the new sub page table ... */
316 /* ... and for all child page tables that one might have */
317 for (i
= 0; i
< MAX_PTE_ENTRIES
; i
++) {
318 r
+= count_required_pts(addr
, sublevel
, maxaddr
);
319 addr
+= sublevelsize
;
321 if (addr
>= maxaddr
) {
323 * We reached the end of address space, no need
324 * to look any further.
334 /* Returns the estimated required size of all page tables */
335 __weak u64
get_page_table_size(void)
337 u64 one_pt
= MAX_PTE_ENTRIES
* sizeof(u64
);
342 get_tcr(0, NULL
, &va_bits
);
346 /* Account for all page tables we would need to cover our memory map */
347 size
= one_pt
* count_required_pts(0, start_level
- 1, 1ULL << va_bits
);
350 * We need to duplicate our page table once to have an emergency pt to
351 * resort to when splitting page tables later on
356 * We may need to split page tables later on if dcache settings change,
357 * so reserve up to 4 (random pick) page tables for that.
364 void setup_pgtables(void)
368 if (!gd
->arch
.tlb_fillptr
|| !gd
->arch
.tlb_addr
)
369 panic("Page table pointer not setup.");
372 * Allocate the first level we're on with invalidate entries.
373 * If the starting level is 0 (va_bits >= 39), then this is our
374 * Lv0 page table, otherwise it's the entry Lv1 page table.
378 /* Now add all MMU table entries one after another to the table */
379 for (i
= 0; mem_map
[i
].size
|| mem_map
[i
].attrs
; i
++)
380 add_map(&mem_map
[i
]);
383 static void setup_all_pgtables(void)
385 u64 tlb_addr
= gd
->arch
.tlb_addr
;
386 u64 tlb_size
= gd
->arch
.tlb_size
;
388 /* Reset the fill ptr */
389 gd
->arch
.tlb_fillptr
= tlb_addr
;
391 /* Create normal system page tables */
394 /* Create emergency page tables */
395 gd
->arch
.tlb_size
-= (uintptr_t)gd
->arch
.tlb_fillptr
-
396 (uintptr_t)gd
->arch
.tlb_addr
;
397 gd
->arch
.tlb_addr
= gd
->arch
.tlb_fillptr
;
399 gd
->arch
.tlb_emerg
= gd
->arch
.tlb_addr
;
400 gd
->arch
.tlb_addr
= tlb_addr
;
401 gd
->arch
.tlb_size
= tlb_size
;
404 /* to activate the MMU we need to set up virtual memory */
405 __weak
void mmu_setup(void)
409 /* Set up page tables only once */
410 if (!gd
->arch
.tlb_fillptr
)
411 setup_all_pgtables();
414 set_ttbr_tcr_mair(el
, gd
->arch
.tlb_addr
, get_tcr(el
, NULL
, NULL
),
418 set_sctlr(get_sctlr() | CR_M
);
422 * Performs a invalidation of the entire data cache at all levels
424 void invalidate_dcache_all(void)
426 __asm_invalidate_dcache_all();
427 __asm_invalidate_l3_dcache();
431 * Performs a clean & invalidation of the entire data cache at all levels.
432 * This function needs to be inline to avoid using stack.
433 * __asm_flush_l3_dcache return status of timeout
435 inline void flush_dcache_all(void)
439 __asm_flush_dcache_all();
440 ret
= __asm_flush_l3_dcache();
442 debug("flushing dcache returns 0x%x\n", ret
);
444 debug("flushing dcache successfully.\n");
448 * Invalidates range in all levels of D-cache/unified cache
450 void invalidate_dcache_range(unsigned long start
, unsigned long stop
)
452 __asm_invalidate_dcache_range(start
, stop
);
456 * Flush range(clean & invalidate) from all levels of D-cache/unified cache
458 void flush_dcache_range(unsigned long start
, unsigned long stop
)
460 __asm_flush_dcache_range(start
, stop
);
463 void dcache_enable(void)
465 /* The data cache is not active unless the mmu is enabled */
466 if (!(get_sctlr() & CR_M
)) {
467 invalidate_dcache_all();
468 __asm_invalidate_tlb_all();
472 set_sctlr(get_sctlr() | CR_C
);
475 void dcache_disable(void)
481 /* if cache isn't enabled no need to disable */
485 set_sctlr(sctlr
& ~(CR_C
|CR_M
));
488 __asm_invalidate_tlb_all();
491 int dcache_status(void)
493 return (get_sctlr() & CR_C
) != 0;
496 u64
*__weak
arch_get_page_table(void) {
497 puts("No page table offset defined\n");
502 static bool is_aligned(u64 addr
, u64 size
, u64 align
)
504 return !(addr
& (align
- 1)) && !(size
& (align
- 1));
507 /* Use flag to indicate if attrs has more than d-cache attributes */
508 static u64
set_one_region(u64 start
, u64 size
, u64 attrs
, bool flag
, int level
)
510 int levelshift
= level2shift(level
);
511 u64 levelsize
= 1ULL << levelshift
;
512 u64
*pte
= find_pte(start
, level
);
514 /* Can we can just modify the current level block PTE? */
515 if (is_aligned(start
, size
, levelsize
)) {
517 *pte
&= ~PMD_ATTRMASK
;
518 *pte
|= attrs
& PMD_ATTRMASK
;
520 *pte
&= ~PMD_ATTRINDX_MASK
;
521 *pte
|= attrs
& PMD_ATTRINDX_MASK
;
523 debug("Set attrs=%llx pte=%p level=%d\n", attrs
, pte
, level
);
528 /* Unaligned or doesn't fit, maybe split block into table */
529 debug("addr=%llx level=%d pte=%p (%llx)\n", start
, level
, pte
, *pte
);
531 /* Maybe we need to split the block into a table */
532 if (pte_type(pte
) == PTE_TYPE_BLOCK
)
533 split_block(pte
, level
);
535 /* And then double-check it became a table or already is one */
536 if (pte_type(pte
) != PTE_TYPE_TABLE
)
537 panic("PTE %p (%llx) for addr=%llx should be a table",
540 /* Roll on to the next page table level */
544 void mmu_set_region_dcache_behaviour(phys_addr_t start
, size_t size
,
545 enum dcache_option option
)
547 u64 attrs
= PMD_ATTRINDX(option
);
548 u64 real_start
= start
;
549 u64 real_size
= size
;
551 debug("start=%lx size=%lx\n", (ulong
)start
, (ulong
)size
);
553 if (!gd
->arch
.tlb_emerg
)
554 panic("Emergency page table not setup.");
557 * We can not modify page tables that we're currently running on,
558 * so we first need to switch to the "emergency" page tables where
559 * we can safely modify our primary page tables and then switch back
561 __asm_switch_ttbr(gd
->arch
.tlb_emerg
);
564 * Loop through the address range until we find a page granule that fits
565 * our alignment constraints, then set it to the new cache attributes
571 for (level
= 1; level
< 4; level
++) {
572 /* Set d-cache attributes only */
573 r
= set_one_region(start
, size
, attrs
, false, level
);
575 /* PTE successfully replaced */
584 /* We're done modifying page tables, switch back to our primary ones */
585 __asm_switch_ttbr(gd
->arch
.tlb_addr
);
588 * Make sure there's nothing stale in dcache for a region that might
589 * have caches off now
591 flush_dcache_range(real_start
, real_start
+ real_size
);
595 * Modify MMU table for a region with updated PXN/UXN/Memory type/valid bits.
596 * The procecess is break-before-make. The target region will be marked as
597 * invalid during the process of changing.
599 void mmu_change_region_attr(phys_addr_t addr
, size_t siz
, u64 attrs
)
607 * Loop through the address range until we find a page granule that fits
608 * our alignment constraints, then set it to "invalid".
611 for (level
= 1; level
< 4; level
++) {
612 /* Set PTE to fault */
613 r
= set_one_region(start
, size
, PTE_TYPE_FAULT
, true,
616 /* PTE successfully invalidated */
624 flush_dcache_range(gd
->arch
.tlb_addr
,
625 gd
->arch
.tlb_addr
+ gd
->arch
.tlb_size
);
626 __asm_invalidate_tlb_all();
629 * Loop through the address range until we find a page granule that fits
630 * our alignment constraints, then set it to the new cache attributes
635 for (level
= 1; level
< 4; level
++) {
636 /* Set PTE to new attributes */
637 r
= set_one_region(start
, size
, attrs
, true, level
);
639 /* PTE successfully updated */
646 flush_dcache_range(gd
->arch
.tlb_addr
,
647 gd
->arch
.tlb_addr
+ gd
->arch
.tlb_size
);
648 __asm_invalidate_tlb_all();
651 #else /* CONFIG_SYS_DCACHE_OFF */
654 * For SPL builds, we may want to not have dcache enabled. Any real U-Boot
655 * running however really wants to have dcache and the MMU active. Check that
656 * everything is sane and give the developer a hint if it isn't.
658 #ifndef CONFIG_SPL_BUILD
659 #error Please describe your MMU layout in CONFIG_SYS_MEM_MAP and enable dcache.
662 void invalidate_dcache_all(void)
666 void flush_dcache_all(void)
670 void dcache_enable(void)
674 void dcache_disable(void)
678 int dcache_status(void)
683 void mmu_set_region_dcache_behaviour(phys_addr_t start
, size_t size
,
684 enum dcache_option option
)
688 #endif /* CONFIG_SYS_DCACHE_OFF */
690 #ifndef CONFIG_SYS_ICACHE_OFF
692 void icache_enable(void)
694 invalidate_icache_all();
695 set_sctlr(get_sctlr() | CR_I
);
698 void icache_disable(void)
700 set_sctlr(get_sctlr() & ~CR_I
);
703 int icache_status(void)
705 return (get_sctlr() & CR_I
) != 0;
708 void invalidate_icache_all(void)
710 __asm_invalidate_icache_all();
711 __asm_invalidate_l3_icache();
714 #else /* CONFIG_SYS_ICACHE_OFF */
716 void icache_enable(void)
720 void icache_disable(void)
724 int icache_status(void)
729 void invalidate_icache_all(void)
733 #endif /* CONFIG_SYS_ICACHE_OFF */
736 * Enable dCache & iCache, whether cache is actually enabled
737 * depend on CONFIG_SYS_DCACHE_OFF and CONFIG_SYS_ICACHE_OFF
739 void __weak
enable_caches(void)