]>
git.ipfire.org Git - people/ms/u-boot.git/blob - arch/arm/cpu/armv8/cache_v8.c
3 * David Feng <fenghua@phytium.com.cn>
6 * Alexander Graf <agraf@suse.de>
8 * SPDX-License-Identifier: GPL-2.0+
12 #include <asm/system.h>
13 #include <asm/armv8/mmu.h>
15 DECLARE_GLOBAL_DATA_PTR
;
17 #ifndef CONFIG_SYS_DCACHE_OFF
20 * With 4k page granule, a virtual address is split into 4 lookup parts
21 * spanning 9 bits each:
23 * _______________________________________________
25 * | 0 | Lv0 | Lv1 | Lv2 | Lv3 | off |
26 * |_______|_______|_______|_______|_______|_______|
27 * 63-48 47-39 38-30 29-21 20-12 11-00
31 * Lv0: FF8000000000 --
38 u64
get_tcr(int el
, u64
*pips
, u64
*pva_bits
)
45 /* Find the largest address we need to support */
46 for (i
= 0; mem_map
[i
].size
|| mem_map
[i
].attrs
; i
++)
47 max_addr
= max(max_addr
, mem_map
[i
].virt
+ mem_map
[i
].size
);
49 /* Calculate the maximum physical (and thus virtual) address */
50 if (max_addr
> (1ULL << 44)) {
53 } else if (max_addr
> (1ULL << 42)) {
56 } else if (max_addr
> (1ULL << 40)) {
59 } else if (max_addr
> (1ULL << 36)) {
62 } else if (max_addr
> (1ULL << 32)) {
71 tcr
= TCR_EL1_RSVD
| (ips
<< 32) | TCR_EPD1_DISABLE
;
73 tcr
= TCR_EL2_RSVD
| (ips
<< 16);
75 tcr
= TCR_EL3_RSVD
| (ips
<< 16);
78 /* PTWs cacheable, inner/outer WBWA and inner shareable */
79 tcr
|= TCR_TG0_4K
| TCR_SHARED_INNER
| TCR_ORGN_WBWA
| TCR_IRGN_WBWA
;
80 tcr
|= TCR_T0SZ(va_bits
);
90 #define MAX_PTE_ENTRIES 512
92 static int pte_type(u64
*pte
)
94 return *pte
& PTE_TYPE_MASK
;
97 /* Returns the LSB number for a PTE on level <level> */
98 static int level2shift(int level
)
100 /* Page is 12 bits wide, every level translates 9 bits */
101 return (12 + 9 * (3 - level
));
104 static u64
*find_pte(u64 addr
, int level
)
112 debug("addr=%llx level=%d\n", addr
, level
);
114 get_tcr(0, NULL
, &va_bits
);
118 if (level
< start_level
)
121 /* Walk through all page table levels to find our PTE */
122 pte
= (u64
*)gd
->arch
.tlb_addr
;
123 for (i
= start_level
; i
< 4; i
++) {
124 idx
= (addr
>> level2shift(i
)) & 0x1FF;
126 debug("idx=%llx PTE %p at level %d: %llx\n", idx
, pte
, i
, *pte
);
131 /* PTE is no table (either invalid or block), can't traverse */
132 if (pte_type(pte
) != PTE_TYPE_TABLE
)
134 /* Off to the next level */
135 pte
= (u64
*)(*pte
& 0x0000fffffffff000ULL
);
138 /* Should never reach here */
142 /* Returns and creates a new full table (512 entries) */
143 static u64
*create_table(void)
145 u64
*new_table
= (u64
*)gd
->arch
.tlb_fillptr
;
146 u64 pt_len
= MAX_PTE_ENTRIES
* sizeof(u64
);
148 /* Allocate MAX_PTE_ENTRIES pte entries */
149 gd
->arch
.tlb_fillptr
+= pt_len
;
151 if (gd
->arch
.tlb_fillptr
- gd
->arch
.tlb_addr
> gd
->arch
.tlb_size
)
152 panic("Insufficient RAM for page table: 0x%lx > 0x%lx. "
153 "Please increase the size in get_page_table_size()",
154 gd
->arch
.tlb_fillptr
- gd
->arch
.tlb_addr
,
157 /* Mark all entries as invalid */
158 memset(new_table
, 0, pt_len
);
163 static void set_pte_table(u64
*pte
, u64
*table
)
165 /* Point *pte to the new table */
166 debug("Setting %p to addr=%p\n", pte
, table
);
167 *pte
= PTE_TYPE_TABLE
| (ulong
)table
;
170 /* Splits a block PTE into table with subpages spanning the old block */
171 static void split_block(u64
*pte
, int level
)
176 /* level describes the parent level, we need the child ones */
177 int levelshift
= level2shift(level
+ 1);
179 if (pte_type(pte
) != PTE_TYPE_BLOCK
)
180 panic("PTE %p (%llx) is not a block. Some driver code wants to "
181 "modify dcache settings for an range not covered in "
182 "mem_map.", pte
, old_pte
);
184 new_table
= create_table();
185 debug("Splitting pte %p (%llx) into %p\n", pte
, old_pte
, new_table
);
187 for (i
= 0; i
< MAX_PTE_ENTRIES
; i
++) {
188 new_table
[i
] = old_pte
| (i
<< levelshift
);
190 /* Level 3 block PTEs have the table type */
191 if ((level
+ 1) == 3)
192 new_table
[i
] |= PTE_TYPE_TABLE
;
194 debug("Setting new_table[%lld] = %llx\n", i
, new_table
[i
]);
197 /* Set the new table into effect */
198 set_pte_table(pte
, new_table
);
201 /* Add one mm_region map entry to the page tables */
202 static void add_map(struct mm_region
*map
)
205 u64 virt
= map
->virt
;
206 u64 phys
= map
->phys
;
207 u64 size
= map
->size
;
208 u64 attrs
= map
->attrs
| PTE_TYPE_BLOCK
| PTE_BLOCK_AF
;
214 pte
= find_pte(virt
, 0);
215 if (pte
&& (pte_type(pte
) == PTE_TYPE_FAULT
)) {
216 debug("Creating table for virt 0x%llx\n", virt
);
217 new_table
= create_table();
218 set_pte_table(pte
, new_table
);
221 for (level
= 1; level
< 4; level
++) {
222 pte
= find_pte(virt
, level
);
224 panic("pte not found\n");
226 blocksize
= 1ULL << level2shift(level
);
227 debug("Checking if pte fits for virt=%llx size=%llx blocksize=%llx\n",
228 virt
, size
, blocksize
);
229 if (size
>= blocksize
&& !(virt
& (blocksize
- 1))) {
230 /* Page fits, create block PTE */
231 debug("Setting PTE %p to block virt=%llx\n",
238 } else if (pte_type(pte
) == PTE_TYPE_FAULT
) {
239 /* Page doesn't fit, create subpages */
240 debug("Creating subtable for virt 0x%llx blksize=%llx\n",
242 new_table
= create_table();
243 set_pte_table(pte
, new_table
);
244 } else if (pte_type(pte
) == PTE_TYPE_BLOCK
) {
245 debug("Split block into subtable for virt 0x%llx blksize=0x%llx\n",
247 split_block(pte
, level
);
260 * This is a recursively called function to count the number of
261 * page tables we need to cover a particular PTE range. If you
262 * call this with level = -1 you basically get the full 48 bit
265 static int count_required_pts(u64 addr
, int level
, u64 maxaddr
)
267 int levelshift
= level2shift(level
);
268 u64 levelsize
= 1ULL << levelshift
;
269 u64 levelmask
= levelsize
- 1;
270 u64 levelend
= addr
+ levelsize
;
273 enum pte_type pte_type
= PTE_INVAL
;
275 for (i
= 0; mem_map
[i
].size
|| mem_map
[i
].attrs
; i
++) {
276 struct mm_region
*map
= &mem_map
[i
];
277 u64 start
= map
->virt
;
278 u64 end
= start
+ map
->size
;
280 /* Check if the PTE would overlap with the map */
281 if (max(addr
, start
) <= min(levelend
, end
)) {
282 start
= max(addr
, start
);
283 end
= min(levelend
, end
);
285 /* We need a sub-pt for this level */
286 if ((start
& levelmask
) || (end
& levelmask
)) {
287 pte_type
= PTE_LEVEL
;
291 /* Lv0 can not do block PTEs, so do levels here too */
293 pte_type
= PTE_LEVEL
;
297 /* PTE is active, but fits into a block */
298 pte_type
= PTE_BLOCK
;
303 * Block PTEs at this level are already covered by the parent page
304 * table, so we only need to count sub page tables.
306 if (pte_type
== PTE_LEVEL
) {
307 int sublevel
= level
+ 1;
308 u64 sublevelsize
= 1ULL << level2shift(sublevel
);
310 /* Account for the new sub page table ... */
313 /* ... and for all child page tables that one might have */
314 for (i
= 0; i
< MAX_PTE_ENTRIES
; i
++) {
315 r
+= count_required_pts(addr
, sublevel
, maxaddr
);
316 addr
+= sublevelsize
;
318 if (addr
>= maxaddr
) {
320 * We reached the end of address space, no need
321 * to look any further.
331 /* Returns the estimated required size of all page tables */
332 __weak u64
get_page_table_size(void)
334 u64 one_pt
= MAX_PTE_ENTRIES
* sizeof(u64
);
339 get_tcr(0, NULL
, &va_bits
);
343 /* Account for all page tables we would need to cover our memory map */
344 size
= one_pt
* count_required_pts(0, start_level
- 1, 1ULL << va_bits
);
347 * We need to duplicate our page table once to have an emergency pt to
348 * resort to when splitting page tables later on
353 * We may need to split page tables later on if dcache settings change,
354 * so reserve up to 4 (random pick) page tables for that.
361 void setup_pgtables(void)
365 if (!gd
->arch
.tlb_fillptr
|| !gd
->arch
.tlb_addr
)
366 panic("Page table pointer not setup.");
369 * Allocate the first level we're on with invalidate entries.
370 * If the starting level is 0 (va_bits >= 39), then this is our
371 * Lv0 page table, otherwise it's the entry Lv1 page table.
375 /* Now add all MMU table entries one after another to the table */
376 for (i
= 0; mem_map
[i
].size
|| mem_map
[i
].attrs
; i
++)
377 add_map(&mem_map
[i
]);
380 static void setup_all_pgtables(void)
382 u64 tlb_addr
= gd
->arch
.tlb_addr
;
383 u64 tlb_size
= gd
->arch
.tlb_size
;
385 /* Reset the fill ptr */
386 gd
->arch
.tlb_fillptr
= tlb_addr
;
388 /* Create normal system page tables */
391 /* Create emergency page tables */
392 gd
->arch
.tlb_size
-= (uintptr_t)gd
->arch
.tlb_fillptr
-
393 (uintptr_t)gd
->arch
.tlb_addr
;
394 gd
->arch
.tlb_addr
= gd
->arch
.tlb_fillptr
;
396 gd
->arch
.tlb_emerg
= gd
->arch
.tlb_addr
;
397 gd
->arch
.tlb_addr
= tlb_addr
;
398 gd
->arch
.tlb_size
= tlb_size
;
401 /* to activate the MMU we need to set up virtual memory */
402 __weak
void mmu_setup(void)
406 /* Set up page tables only once */
407 if (!gd
->arch
.tlb_fillptr
)
408 setup_all_pgtables();
411 set_ttbr_tcr_mair(el
, gd
->arch
.tlb_addr
, get_tcr(el
, NULL
, NULL
),
415 set_sctlr(get_sctlr() | CR_M
);
419 * Performs a invalidation of the entire data cache at all levels
421 void invalidate_dcache_all(void)
423 __asm_invalidate_dcache_all();
424 __asm_invalidate_l3_dcache();
428 * Performs a clean & invalidation of the entire data cache at all levels.
429 * This function needs to be inline to avoid using stack.
430 * __asm_flush_l3_dcache return status of timeout
432 inline void flush_dcache_all(void)
436 __asm_flush_dcache_all();
437 ret
= __asm_flush_l3_dcache();
439 debug("flushing dcache returns 0x%x\n", ret
);
441 debug("flushing dcache successfully.\n");
445 * Invalidates range in all levels of D-cache/unified cache
447 void invalidate_dcache_range(unsigned long start
, unsigned long stop
)
449 __asm_invalidate_dcache_range(start
, stop
);
453 * Flush range(clean & invalidate) from all levels of D-cache/unified cache
455 void flush_dcache_range(unsigned long start
, unsigned long stop
)
457 __asm_flush_dcache_range(start
, stop
);
460 void dcache_enable(void)
462 /* The data cache is not active unless the mmu is enabled */
463 if (!(get_sctlr() & CR_M
)) {
464 invalidate_dcache_all();
465 __asm_invalidate_tlb_all();
469 set_sctlr(get_sctlr() | CR_C
);
472 void dcache_disable(void)
478 /* if cache isn't enabled no need to disable */
482 set_sctlr(sctlr
& ~(CR_C
|CR_M
));
485 __asm_invalidate_tlb_all();
488 int dcache_status(void)
490 return (get_sctlr() & CR_C
) != 0;
493 u64
*__weak
arch_get_page_table(void) {
494 puts("No page table offset defined\n");
499 static bool is_aligned(u64 addr
, u64 size
, u64 align
)
501 return !(addr
& (align
- 1)) && !(size
& (align
- 1));
504 /* Use flag to indicate if attrs has more than d-cache attributes */
505 static u64
set_one_region(u64 start
, u64 size
, u64 attrs
, bool flag
, int level
)
507 int levelshift
= level2shift(level
);
508 u64 levelsize
= 1ULL << levelshift
;
509 u64
*pte
= find_pte(start
, level
);
511 /* Can we can just modify the current level block PTE? */
512 if (is_aligned(start
, size
, levelsize
)) {
514 *pte
&= ~PMD_ATTRMASK
;
515 *pte
|= attrs
& PMD_ATTRMASK
;
517 *pte
&= ~PMD_ATTRINDX_MASK
;
518 *pte
|= attrs
& PMD_ATTRINDX_MASK
;
520 debug("Set attrs=%llx pte=%p level=%d\n", attrs
, pte
, level
);
525 /* Unaligned or doesn't fit, maybe split block into table */
526 debug("addr=%llx level=%d pte=%p (%llx)\n", start
, level
, pte
, *pte
);
528 /* Maybe we need to split the block into a table */
529 if (pte_type(pte
) == PTE_TYPE_BLOCK
)
530 split_block(pte
, level
);
532 /* And then double-check it became a table or already is one */
533 if (pte_type(pte
) != PTE_TYPE_TABLE
)
534 panic("PTE %p (%llx) for addr=%llx should be a table",
537 /* Roll on to the next page table level */
541 void mmu_set_region_dcache_behaviour(phys_addr_t start
, size_t size
,
542 enum dcache_option option
)
544 u64 attrs
= PMD_ATTRINDX(option
);
545 u64 real_start
= start
;
546 u64 real_size
= size
;
548 debug("start=%lx size=%lx\n", (ulong
)start
, (ulong
)size
);
550 if (!gd
->arch
.tlb_emerg
)
551 panic("Emergency page table not setup.");
554 * We can not modify page tables that we're currently running on,
555 * so we first need to switch to the "emergency" page tables where
556 * we can safely modify our primary page tables and then switch back
558 __asm_switch_ttbr(gd
->arch
.tlb_emerg
);
561 * Loop through the address range until we find a page granule that fits
562 * our alignment constraints, then set it to the new cache attributes
568 for (level
= 1; level
< 4; level
++) {
569 /* Set d-cache attributes only */
570 r
= set_one_region(start
, size
, attrs
, false, level
);
572 /* PTE successfully replaced */
581 /* We're done modifying page tables, switch back to our primary ones */
582 __asm_switch_ttbr(gd
->arch
.tlb_addr
);
585 * Make sure there's nothing stale in dcache for a region that might
586 * have caches off now
588 flush_dcache_range(real_start
, real_start
+ real_size
);
592 * Modify MMU table for a region with updated PXN/UXN/Memory type/valid bits.
593 * The procecess is break-before-make. The target region will be marked as
594 * invalid during the process of changing.
596 void mmu_change_region_attr(phys_addr_t addr
, size_t siz
, u64 attrs
)
604 * Loop through the address range until we find a page granule that fits
605 * our alignment constraints, then set it to "invalid".
608 for (level
= 1; level
< 4; level
++) {
609 /* Set PTE to fault */
610 r
= set_one_region(start
, size
, PTE_TYPE_FAULT
, true,
613 /* PTE successfully invalidated */
621 flush_dcache_range(gd
->arch
.tlb_addr
,
622 gd
->arch
.tlb_addr
+ gd
->arch
.tlb_size
);
623 __asm_invalidate_tlb_all();
626 * Loop through the address range until we find a page granule that fits
627 * our alignment constraints, then set it to the new cache attributes
632 for (level
= 1; level
< 4; level
++) {
633 /* Set PTE to new attributes */
634 r
= set_one_region(start
, size
, attrs
, true, level
);
636 /* PTE successfully updated */
643 flush_dcache_range(gd
->arch
.tlb_addr
,
644 gd
->arch
.tlb_addr
+ gd
->arch
.tlb_size
);
645 __asm_invalidate_tlb_all();
648 #else /* CONFIG_SYS_DCACHE_OFF */
651 * For SPL builds, we may want to not have dcache enabled. Any real U-Boot
652 * running however really wants to have dcache and the MMU active. Check that
653 * everything is sane and give the developer a hint if it isn't.
655 #ifndef CONFIG_SPL_BUILD
656 #error Please describe your MMU layout in CONFIG_SYS_MEM_MAP and enable dcache.
659 void invalidate_dcache_all(void)
663 void flush_dcache_all(void)
667 void dcache_enable(void)
671 void dcache_disable(void)
675 int dcache_status(void)
680 void mmu_set_region_dcache_behaviour(phys_addr_t start
, size_t size
,
681 enum dcache_option option
)
685 #endif /* CONFIG_SYS_DCACHE_OFF */
687 #ifndef CONFIG_SYS_ICACHE_OFF
689 void icache_enable(void)
691 invalidate_icache_all();
692 set_sctlr(get_sctlr() | CR_I
);
695 void icache_disable(void)
697 set_sctlr(get_sctlr() & ~CR_I
);
700 int icache_status(void)
702 return (get_sctlr() & CR_I
) != 0;
705 void invalidate_icache_all(void)
707 __asm_invalidate_icache_all();
708 __asm_invalidate_l3_icache();
711 #else /* CONFIG_SYS_ICACHE_OFF */
713 void icache_enable(void)
717 void icache_disable(void)
721 int icache_status(void)
726 void invalidate_icache_all(void)
730 #endif /* CONFIG_SYS_ICACHE_OFF */
733 * Enable dCache & iCache, whether cache is actually enabled
734 * depend on CONFIG_SYS_DCACHE_OFF and CONFIG_SYS_ICACHE_OFF
736 void __weak
enable_caches(void)