1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2012 Regents of the University of California
4 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
7 #include <linux/init.h>
9 #include <linux/memblock.h>
10 #include <linux/initrd.h>
11 #include <linux/swap.h>
12 #include <linux/sizes.h>
13 #include <linux/of_fdt.h>
14 #include <linux/libfdt.h>
15 #include <linux/set_memory.h>
17 #include <asm/fixmap.h>
18 #include <asm/tlbflush.h>
19 #include <asm/sections.h>
20 #include <asm/pgtable.h>
23 #include "../kernel/head.h"
25 unsigned long empty_zero_page
[PAGE_SIZE
/ sizeof(unsigned long)]
27 EXPORT_SYMBOL(empty_zero_page
);
32 static void __init
zone_sizes_init(void)
34 unsigned long max_zone_pfns
[MAX_NR_ZONES
] = { 0, };
36 #ifdef CONFIG_ZONE_DMA32
37 max_zone_pfns
[ZONE_DMA32
] = PFN_DOWN(min(4UL * SZ_1G
,
38 (unsigned long) PFN_PHYS(max_low_pfn
)));
40 max_zone_pfns
[ZONE_NORMAL
] = max_low_pfn
;
42 free_area_init_nodes(max_zone_pfns
);
45 static void setup_zero_page(void)
47 memset((void *)empty_zero_page
, 0, PAGE_SIZE
);
50 #ifdef CONFIG_DEBUG_VM
51 static inline void print_mlk(char *name
, unsigned long b
, unsigned long t
)
53 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name
, b
, t
,
57 static inline void print_mlm(char *name
, unsigned long b
, unsigned long t
)
59 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name
, b
, t
,
63 static void print_vm_layout(void)
65 pr_notice("Virtual kernel memory layout:\n");
66 print_mlk("fixmap", (unsigned long)FIXADDR_START
,
67 (unsigned long)FIXADDR_TOP
);
68 print_mlm("pci io", (unsigned long)PCI_IO_START
,
69 (unsigned long)PCI_IO_END
);
70 print_mlm("vmemmap", (unsigned long)VMEMMAP_START
,
71 (unsigned long)VMEMMAP_END
);
72 print_mlm("vmalloc", (unsigned long)VMALLOC_START
,
73 (unsigned long)VMALLOC_END
);
74 print_mlm("lowmem", (unsigned long)PAGE_OFFSET
,
75 (unsigned long)high_memory
);
78 static void print_vm_layout(void) { }
79 #endif /* CONFIG_DEBUG_VM */
81 void __init
mem_init(void)
85 #endif /* CONFIG_FLATMEM */
87 high_memory
= (void *)(__va(PFN_PHYS(max_low_pfn
)));
90 mem_init_print_info(NULL
);
94 #ifdef CONFIG_BLK_DEV_INITRD
95 static void __init
setup_initrd(void)
99 if (initrd_start
>= initrd_end
) {
100 pr_info("initrd not found or empty");
103 if (__pa_symbol(initrd_end
) > PFN_PHYS(max_low_pfn
)) {
104 pr_err("initrd extends beyond end of memory");
108 size
= initrd_end
- initrd_start
;
109 memblock_reserve(__pa_symbol(initrd_start
), size
);
110 initrd_below_start_ok
= 1;
112 pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n",
113 (void *)(initrd_start
), size
);
116 pr_cont(" - disabling initrd\n");
120 #endif /* CONFIG_BLK_DEV_INITRD */
122 static phys_addr_t dtb_early_pa __initdata
;
124 void __init
setup_bootmem(void)
126 struct memblock_region
*reg
;
127 phys_addr_t mem_size
= 0;
128 phys_addr_t vmlinux_end
= __pa_symbol(&_end
);
129 phys_addr_t vmlinux_start
= __pa_symbol(&_start
);
131 /* Find the memory region containing the kernel */
132 for_each_memblock(memory
, reg
) {
133 phys_addr_t end
= reg
->base
+ reg
->size
;
135 if (reg
->base
<= vmlinux_start
&& vmlinux_end
<= end
) {
136 mem_size
= min(reg
->size
, (phys_addr_t
)-PAGE_OFFSET
);
139 * Remove memblock from the end of usable area to the
142 if (reg
->base
+ mem_size
< end
)
143 memblock_remove(reg
->base
+ mem_size
,
144 end
- reg
->base
- mem_size
);
147 BUG_ON(mem_size
== 0);
149 /* Reserve from the start of the kernel to the end of the kernel */
150 memblock_reserve(vmlinux_start
, vmlinux_end
- vmlinux_start
);
152 set_max_mapnr(PFN_DOWN(mem_size
));
153 max_low_pfn
= PFN_DOWN(memblock_end_of_DRAM());
155 #ifdef CONFIG_BLK_DEV_INITRD
157 #endif /* CONFIG_BLK_DEV_INITRD */
160 * Avoid using early_init_fdt_reserve_self() since __pa() does
161 * not work for DTB pointers that are fixmap addresses
163 memblock_reserve(dtb_early_pa
, fdt_totalsize(dtb_early_va
));
165 early_init_fdt_scan_reserved_mem();
166 memblock_allow_resize();
169 for_each_memblock(memory
, reg
) {
170 unsigned long start_pfn
= memblock_region_memory_base_pfn(reg
);
171 unsigned long end_pfn
= memblock_region_memory_end_pfn(reg
);
173 memblock_set_node(PFN_PHYS(start_pfn
),
174 PFN_PHYS(end_pfn
- start_pfn
),
175 &memblock
.memory
, 0);
180 unsigned long va_pa_offset
;
181 EXPORT_SYMBOL(va_pa_offset
);
182 unsigned long pfn_base
;
183 EXPORT_SYMBOL(pfn_base
);
185 pgd_t swapper_pg_dir
[PTRS_PER_PGD
] __page_aligned_bss
;
186 pgd_t trampoline_pg_dir
[PTRS_PER_PGD
] __page_aligned_bss
;
187 pte_t fixmap_pte
[PTRS_PER_PTE
] __page_aligned_bss
;
188 static bool mmu_enabled
;
190 #define MAX_EARLY_MAPPING_SIZE SZ_128M
192 pgd_t early_pg_dir
[PTRS_PER_PGD
] __initdata
__aligned(PAGE_SIZE
);
194 void __set_fixmap(enum fixed_addresses idx
, phys_addr_t phys
, pgprot_t prot
)
196 unsigned long addr
= __fix_to_virt(idx
);
199 BUG_ON(idx
<= FIX_HOLE
|| idx
>= __end_of_fixed_addresses
);
201 ptep
= &fixmap_pte
[pte_index(addr
)];
203 if (pgprot_val(prot
)) {
204 set_pte(ptep
, pfn_pte(phys
>> PAGE_SHIFT
, prot
));
206 pte_clear(&init_mm
, addr
, ptep
);
207 local_flush_tlb_page(addr
);
211 static pte_t
*__init
get_pte_virt(phys_addr_t pa
)
214 clear_fixmap(FIX_PTE
);
215 return (pte_t
*)set_fixmap_offset(FIX_PTE
, pa
);
217 return (pte_t
*)((uintptr_t)pa
);
221 static phys_addr_t __init
alloc_pte(uintptr_t va
)
224 * We only create PMD or PGD early mappings so we
225 * should never reach here with MMU disabled.
227 BUG_ON(!mmu_enabled
);
229 return memblock_phys_alloc(PAGE_SIZE
, PAGE_SIZE
);
232 static void __init
create_pte_mapping(pte_t
*ptep
,
233 uintptr_t va
, phys_addr_t pa
,
234 phys_addr_t sz
, pgprot_t prot
)
236 uintptr_t pte_index
= pte_index(va
);
238 BUG_ON(sz
!= PAGE_SIZE
);
240 if (pte_none(ptep
[pte_index
]))
241 ptep
[pte_index
] = pfn_pte(PFN_DOWN(pa
), prot
);
244 #ifndef __PAGETABLE_PMD_FOLDED
246 pmd_t trampoline_pmd
[PTRS_PER_PMD
] __page_aligned_bss
;
247 pmd_t fixmap_pmd
[PTRS_PER_PMD
] __page_aligned_bss
;
249 #if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE
250 #define NUM_EARLY_PMDS 1UL
252 #define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE)
254 pmd_t early_pmd
[PTRS_PER_PMD
* NUM_EARLY_PMDS
] __initdata
__aligned(PAGE_SIZE
);
256 static pmd_t
*__init
get_pmd_virt(phys_addr_t pa
)
259 clear_fixmap(FIX_PMD
);
260 return (pmd_t
*)set_fixmap_offset(FIX_PMD
, pa
);
262 return (pmd_t
*)((uintptr_t)pa
);
266 static phys_addr_t __init
alloc_pmd(uintptr_t va
)
271 return memblock_phys_alloc(PAGE_SIZE
, PAGE_SIZE
);
273 pmd_num
= (va
- PAGE_OFFSET
) >> PGDIR_SHIFT
;
274 BUG_ON(pmd_num
>= NUM_EARLY_PMDS
);
275 return (uintptr_t)&early_pmd
[pmd_num
* PTRS_PER_PMD
];
278 static void __init
create_pmd_mapping(pmd_t
*pmdp
,
279 uintptr_t va
, phys_addr_t pa
,
280 phys_addr_t sz
, pgprot_t prot
)
283 phys_addr_t pte_phys
;
284 uintptr_t pmd_index
= pmd_index(va
);
286 if (sz
== PMD_SIZE
) {
287 if (pmd_none(pmdp
[pmd_index
]))
288 pmdp
[pmd_index
] = pfn_pmd(PFN_DOWN(pa
), prot
);
292 if (pmd_none(pmdp
[pmd_index
])) {
293 pte_phys
= alloc_pte(va
);
294 pmdp
[pmd_index
] = pfn_pmd(PFN_DOWN(pte_phys
), PAGE_TABLE
);
295 ptep
= get_pte_virt(pte_phys
);
296 memset(ptep
, 0, PAGE_SIZE
);
298 pte_phys
= PFN_PHYS(_pmd_pfn(pmdp
[pmd_index
]));
299 ptep
= get_pte_virt(pte_phys
);
302 create_pte_mapping(ptep
, va
, pa
, sz
, prot
);
305 #define pgd_next_t pmd_t
306 #define alloc_pgd_next(__va) alloc_pmd(__va)
307 #define get_pgd_next_virt(__pa) get_pmd_virt(__pa)
308 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \
309 create_pmd_mapping(__nextp, __va, __pa, __sz, __prot)
310 #define fixmap_pgd_next fixmap_pmd
312 #define pgd_next_t pte_t
313 #define alloc_pgd_next(__va) alloc_pte(__va)
314 #define get_pgd_next_virt(__pa) get_pte_virt(__pa)
315 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \
316 create_pte_mapping(__nextp, __va, __pa, __sz, __prot)
317 #define fixmap_pgd_next fixmap_pte
320 static void __init
create_pgd_mapping(pgd_t
*pgdp
,
321 uintptr_t va
, phys_addr_t pa
,
322 phys_addr_t sz
, pgprot_t prot
)
325 phys_addr_t next_phys
;
326 uintptr_t pgd_index
= pgd_index(va
);
328 if (sz
== PGDIR_SIZE
) {
329 if (pgd_val(pgdp
[pgd_index
]) == 0)
330 pgdp
[pgd_index
] = pfn_pgd(PFN_DOWN(pa
), prot
);
334 if (pgd_val(pgdp
[pgd_index
]) == 0) {
335 next_phys
= alloc_pgd_next(va
);
336 pgdp
[pgd_index
] = pfn_pgd(PFN_DOWN(next_phys
), PAGE_TABLE
);
337 nextp
= get_pgd_next_virt(next_phys
);
338 memset(nextp
, 0, PAGE_SIZE
);
340 next_phys
= PFN_PHYS(_pgd_pfn(pgdp
[pgd_index
]));
341 nextp
= get_pgd_next_virt(next_phys
);
344 create_pgd_next_mapping(nextp
, va
, pa
, sz
, prot
);
347 static uintptr_t __init
best_map_size(phys_addr_t base
, phys_addr_t size
)
349 /* Upgrade to PMD_SIZE mappings whenever possible */
350 if ((base
& (PMD_SIZE
- 1)) || (size
& (PMD_SIZE
- 1)))
357 * setup_vm() is called from head.S with MMU-off.
359 * Following requirements should be honoured for setup_vm() to work
361 * 1) It should use PC-relative addressing for accessing kernel symbols.
362 * To achieve this we always use GCC cmodel=medany.
363 * 2) The compiler instrumentation for FTRACE will not work for setup_vm()
364 * so disable compiler instrumentation when FTRACE is enabled.
366 * Currently, the above requirements are honoured by using custom CFLAGS
367 * for init.o in mm/Makefile.
370 #ifndef __riscv_cmodel_medany
371 #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."
374 asmlinkage
void __init
setup_vm(uintptr_t dtb_pa
)
376 uintptr_t va
, end_va
;
377 uintptr_t load_pa
= (uintptr_t)(&_start
);
378 uintptr_t load_sz
= (uintptr_t)(&_end
) - load_pa
;
379 uintptr_t map_size
= best_map_size(load_pa
, MAX_EARLY_MAPPING_SIZE
);
381 va_pa_offset
= PAGE_OFFSET
- load_pa
;
382 pfn_base
= PFN_DOWN(load_pa
);
385 * Enforce boot alignment requirements of RV32 and
386 * RV64 by only allowing PMD or PGD mappings.
388 BUG_ON(map_size
== PAGE_SIZE
);
390 /* Sanity check alignment and size */
391 BUG_ON((PAGE_OFFSET
% PGDIR_SIZE
) != 0);
392 BUG_ON((load_pa
% map_size
) != 0);
393 BUG_ON(load_sz
> MAX_EARLY_MAPPING_SIZE
);
395 /* Setup early PGD for fixmap */
396 create_pgd_mapping(early_pg_dir
, FIXADDR_START
,
397 (uintptr_t)fixmap_pgd_next
, PGDIR_SIZE
, PAGE_TABLE
);
399 #ifndef __PAGETABLE_PMD_FOLDED
400 /* Setup fixmap PMD */
401 create_pmd_mapping(fixmap_pmd
, FIXADDR_START
,
402 (uintptr_t)fixmap_pte
, PMD_SIZE
, PAGE_TABLE
);
403 /* Setup trampoline PGD and PMD */
404 create_pgd_mapping(trampoline_pg_dir
, PAGE_OFFSET
,
405 (uintptr_t)trampoline_pmd
, PGDIR_SIZE
, PAGE_TABLE
);
406 create_pmd_mapping(trampoline_pmd
, PAGE_OFFSET
,
407 load_pa
, PMD_SIZE
, PAGE_KERNEL_EXEC
);
409 /* Setup trampoline PGD */
410 create_pgd_mapping(trampoline_pg_dir
, PAGE_OFFSET
,
411 load_pa
, PGDIR_SIZE
, PAGE_KERNEL_EXEC
);
415 * Setup early PGD covering entire kernel which will allows
416 * us to reach paging_init(). We map all memory banks later
417 * in setup_vm_final() below.
419 end_va
= PAGE_OFFSET
+ load_sz
;
420 for (va
= PAGE_OFFSET
; va
< end_va
; va
+= map_size
)
421 create_pgd_mapping(early_pg_dir
, va
,
422 load_pa
+ (va
- PAGE_OFFSET
),
423 map_size
, PAGE_KERNEL_EXEC
);
425 /* Create fixed mapping for early FDT parsing */
426 end_va
= __fix_to_virt(FIX_FDT
) + FIX_FDT_SIZE
;
427 for (va
= __fix_to_virt(FIX_FDT
); va
< end_va
; va
+= PAGE_SIZE
)
428 create_pte_mapping(fixmap_pte
, va
,
429 dtb_pa
+ (va
- __fix_to_virt(FIX_FDT
)),
430 PAGE_SIZE
, PAGE_KERNEL
);
432 /* Save pointer to DTB for early FDT parsing */
433 dtb_early_va
= (void *)fix_to_virt(FIX_FDT
) + (dtb_pa
& ~PAGE_MASK
);
434 /* Save physical address for memblock reservation */
435 dtb_early_pa
= dtb_pa
;
438 static void __init
setup_vm_final(void)
440 uintptr_t va
, map_size
;
441 phys_addr_t pa
, start
, end
;
442 struct memblock_region
*reg
;
444 /* Set mmu_enabled flag */
447 /* Setup swapper PGD for fixmap */
448 create_pgd_mapping(swapper_pg_dir
, FIXADDR_START
,
449 __pa_symbol(fixmap_pgd_next
),
450 PGDIR_SIZE
, PAGE_TABLE
);
452 /* Map all memory banks */
453 for_each_memblock(memory
, reg
) {
455 end
= start
+ reg
->size
;
459 if (memblock_is_nomap(reg
))
461 if (start
<= __pa(PAGE_OFFSET
) &&
462 __pa(PAGE_OFFSET
) < end
)
463 start
= __pa(PAGE_OFFSET
);
465 map_size
= best_map_size(start
, end
- start
);
466 for (pa
= start
; pa
< end
; pa
+= map_size
) {
467 va
= (uintptr_t)__va(pa
);
468 create_pgd_mapping(swapper_pg_dir
, va
, pa
,
469 map_size
, PAGE_KERNEL_EXEC
);
473 /* Clear fixmap PTE and PMD mappings */
474 clear_fixmap(FIX_PTE
);
475 clear_fixmap(FIX_PMD
);
477 /* Move to swapper page table */
478 csr_write(CSR_SATP
, PFN_DOWN(__pa_symbol(swapper_pg_dir
)) | SATP_MODE
);
479 local_flush_tlb_all();
482 void free_initmem(void)
484 unsigned long init_begin
= (unsigned long)__init_begin
;
485 unsigned long init_end
= (unsigned long)__init_end
;
487 /* Make the region as non-execuatble. */
488 set_memory_nx(init_begin
, (init_end
- init_begin
) >> PAGE_SHIFT
);
489 free_initmem_default(POISON_FREE_INITMEM
);
493 asmlinkage
void __init
setup_vm(uintptr_t dtb_pa
)
495 dtb_early_va
= (void *)dtb_pa
;
498 static inline void setup_vm_final(void)
501 #endif /* CONFIG_MMU */
503 #ifdef CONFIG_STRICT_KERNEL_RWX
504 void set_kernel_text_rw(void)
506 unsigned long text_start
= (unsigned long)_text
;
507 unsigned long text_end
= (unsigned long)_etext
;
509 set_memory_rw(text_start
, (text_end
- text_start
) >> PAGE_SHIFT
);
512 void set_kernel_text_ro(void)
514 unsigned long text_start
= (unsigned long)_text
;
515 unsigned long text_end
= (unsigned long)_etext
;
517 set_memory_ro(text_start
, (text_end
- text_start
) >> PAGE_SHIFT
);
520 void mark_rodata_ro(void)
522 unsigned long text_start
= (unsigned long)_text
;
523 unsigned long text_end
= (unsigned long)_etext
;
524 unsigned long rodata_start
= (unsigned long)__start_rodata
;
525 unsigned long data_start
= (unsigned long)_data
;
526 unsigned long max_low
= (unsigned long)(__va(PFN_PHYS(max_low_pfn
)));
528 set_memory_ro(text_start
, (text_end
- text_start
) >> PAGE_SHIFT
);
529 set_memory_ro(rodata_start
, (data_start
- rodata_start
) >> PAGE_SHIFT
);
530 set_memory_nx(rodata_start
, (data_start
- rodata_start
) >> PAGE_SHIFT
);
531 set_memory_nx(data_start
, (max_low
- data_start
) >> PAGE_SHIFT
);
535 void __init
paging_init(void)
544 #ifdef CONFIG_SPARSEMEM_VMEMMAP
545 int __meminit
vmemmap_populate(unsigned long start
, unsigned long end
, int node
,
546 struct vmem_altmap
*altmap
)
548 return vmemmap_populate_basepages(start
, end
, node
);