]>
Commit | Line | Data |
---|---|---|
50acfb2b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
76d2a049 PD |
2 | /* |
3 | * Copyright (C) 2012 Regents of the University of California | |
671f9a3e | 4 | * Copyright (C) 2019 Western Digital Corporation or its affiliates. |
76d2a049 PD |
5 | */ |
6 | ||
7 | #include <linux/init.h> | |
8 | #include <linux/mm.h> | |
76d2a049 | 9 | #include <linux/memblock.h> |
57c8a661 | 10 | #include <linux/initrd.h> |
76d2a049 | 11 | #include <linux/swap.h> |
5ec9c4ff | 12 | #include <linux/sizes.h> |
0651c263 | 13 | #include <linux/of_fdt.h> |
922b0375 | 14 | #include <linux/libfdt.h> |
d27c3c90 | 15 | #include <linux/set_memory.h> |
76d2a049 | 16 | |
f2c17aab | 17 | #include <asm/fixmap.h> |
76d2a049 PD |
18 | #include <asm/tlbflush.h> |
19 | #include <asm/sections.h> | |
20 | #include <asm/pgtable.h> | |
21 | #include <asm/io.h> | |
22 | ||
ffaee272 PW |
23 | #include "../kernel/head.h" |
24 | ||
387181dc AP |
25 | unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] |
26 | __page_aligned_bss; | |
27 | EXPORT_SYMBOL(empty_zero_page); | |
28 | ||
d90d45d7 | 29 | extern char _start[]; |
6bd33e1e | 30 | void *dtb_early_va; |
d90d45d7 | 31 | |
76d2a049 PD |
32 | static void __init zone_sizes_init(void) |
33 | { | |
5ec9c4ff | 34 | unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; |
76d2a049 | 35 | |
d5fad48c | 36 | #ifdef CONFIG_ZONE_DMA32 |
28198c46 GR |
37 | max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G, |
38 | (unsigned long) PFN_PHYS(max_low_pfn))); | |
d5fad48c | 39 | #endif |
5ec9c4ff CH |
40 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
41 | ||
42 | free_area_init_nodes(max_zone_pfns); | |
76d2a049 PD |
43 | } |
44 | ||
6bd33e1e | 45 | static void setup_zero_page(void) |
76d2a049 PD |
46 | { |
47 | memset((void *)empty_zero_page, 0, PAGE_SIZE); | |
48 | } | |
49 | ||
8fa3cdff | 50 | #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) |
2cc6c4a0 YS |
51 | static inline void print_mlk(char *name, unsigned long b, unsigned long t) |
52 | { | |
53 | pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, | |
54 | (((t) - (b)) >> 10)); | |
55 | } | |
56 | ||
57 | static inline void print_mlm(char *name, unsigned long b, unsigned long t) | |
58 | { | |
59 | pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, | |
60 | (((t) - (b)) >> 20)); | |
61 | } | |
62 | ||
63 | static void print_vm_layout(void) | |
64 | { | |
65 | pr_notice("Virtual kernel memory layout:\n"); | |
66 | print_mlk("fixmap", (unsigned long)FIXADDR_START, | |
67 | (unsigned long)FIXADDR_TOP); | |
68 | print_mlm("pci io", (unsigned long)PCI_IO_START, | |
69 | (unsigned long)PCI_IO_END); | |
70 | print_mlm("vmemmap", (unsigned long)VMEMMAP_START, | |
71 | (unsigned long)VMEMMAP_END); | |
72 | print_mlm("vmalloc", (unsigned long)VMALLOC_START, | |
73 | (unsigned long)VMALLOC_END); | |
74 | print_mlm("lowmem", (unsigned long)PAGE_OFFSET, | |
75 | (unsigned long)high_memory); | |
76 | } | |
77 | #else | |
78 | static void print_vm_layout(void) { } | |
79 | #endif /* CONFIG_DEBUG_VM */ | |
80 | ||
76d2a049 PD |
81 | void __init mem_init(void) |
82 | { | |
83 | #ifdef CONFIG_FLATMEM | |
84 | BUG_ON(!mem_map); | |
85 | #endif /* CONFIG_FLATMEM */ | |
86 | ||
87 | high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); | |
c6ffc5ca | 88 | memblock_free_all(); |
76d2a049 PD |
89 | |
90 | mem_init_print_info(NULL); | |
2cc6c4a0 | 91 | print_vm_layout(); |
76d2a049 PD |
92 | } |
93 | ||
76d2a049 | 94 | #ifdef CONFIG_BLK_DEV_INITRD |
0651c263 AP |
95 | static void __init setup_initrd(void) |
96 | { | |
97 | unsigned long size; | |
98 | ||
99 | if (initrd_start >= initrd_end) { | |
100 | pr_info("initrd not found or empty"); | |
101 | goto disable; | |
102 | } | |
ac51e005 | 103 | if (__pa_symbol(initrd_end) > PFN_PHYS(max_low_pfn)) { |
0651c263 AP |
104 | pr_err("initrd extends beyond end of memory"); |
105 | goto disable; | |
106 | } | |
107 | ||
108 | size = initrd_end - initrd_start; | |
ac51e005 | 109 | memblock_reserve(__pa_symbol(initrd_start), size); |
0651c263 AP |
110 | initrd_below_start_ok = 1; |
111 | ||
112 | pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", | |
113 | (void *)(initrd_start), size); | |
114 | return; | |
115 | disable: | |
116 | pr_cont(" - disabling initrd\n"); | |
117 | initrd_start = 0; | |
118 | initrd_end = 0; | |
119 | } | |
76d2a049 | 120 | #endif /* CONFIG_BLK_DEV_INITRD */ |
0651c263 | 121 | |
922b0375 AO |
122 | static phys_addr_t dtb_early_pa __initdata; |
123 | ||
0651c263 AP |
124 | void __init setup_bootmem(void) |
125 | { | |
126 | struct memblock_region *reg; | |
127 | phys_addr_t mem_size = 0; | |
ac51e005 ZL |
128 | phys_addr_t vmlinux_end = __pa_symbol(&_end); |
129 | phys_addr_t vmlinux_start = __pa_symbol(&_start); | |
0651c263 AP |
130 | |
131 | /* Find the memory region containing the kernel */ | |
132 | for_each_memblock(memory, reg) { | |
0651c263 AP |
133 | phys_addr_t end = reg->base + reg->size; |
134 | ||
a160eed4 | 135 | if (reg->base <= vmlinux_start && vmlinux_end <= end) { |
0651c263 | 136 | mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET); |
f05badde AP |
137 | |
138 | /* | |
139 | * Remove memblock from the end of usable area to the | |
140 | * end of region | |
141 | */ | |
142 | if (reg->base + mem_size < end) | |
143 | memblock_remove(reg->base + mem_size, | |
144 | end - reg->base - mem_size); | |
0651c263 AP |
145 | } |
146 | } | |
147 | BUG_ON(mem_size == 0); | |
148 | ||
d90d45d7 AP |
149 | /* Reserve from the start of the kernel to the end of the kernel */ |
150 | memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); | |
151 | ||
0651c263 | 152 | set_max_mapnr(PFN_DOWN(mem_size)); |
c749bb2d VC |
153 | max_pfn = PFN_DOWN(memblock_end_of_DRAM()); |
154 | max_low_pfn = max_pfn; | |
0651c263 AP |
155 | |
156 | #ifdef CONFIG_BLK_DEV_INITRD | |
157 | setup_initrd(); | |
158 | #endif /* CONFIG_BLK_DEV_INITRD */ | |
159 | ||
922b0375 AO |
160 | /* |
161 | * Avoid using early_init_fdt_reserve_self() since __pa() does | |
162 | * not work for DTB pointers that are fixmap addresses | |
163 | */ | |
164 | memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); | |
165 | ||
0651c263 AP |
166 | early_init_fdt_scan_reserved_mem(); |
167 | memblock_allow_resize(); | |
168 | memblock_dump_all(); | |
169 | ||
170 | for_each_memblock(memory, reg) { | |
171 | unsigned long start_pfn = memblock_region_memory_base_pfn(reg); | |
172 | unsigned long end_pfn = memblock_region_memory_end_pfn(reg); | |
173 | ||
174 | memblock_set_node(PFN_PHYS(start_pfn), | |
175 | PFN_PHYS(end_pfn - start_pfn), | |
176 | &memblock.memory, 0); | |
177 | } | |
178 | } | |
6f1e9e94 | 179 | |
6bd33e1e | 180 | #ifdef CONFIG_MMU |
387181dc AP |
181 | unsigned long va_pa_offset; |
182 | EXPORT_SYMBOL(va_pa_offset); | |
183 | unsigned long pfn_base; | |
184 | EXPORT_SYMBOL(pfn_base); | |
185 | ||
6f1e9e94 | 186 | pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
671f9a3e AP |
187 | pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
188 | pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; | |
189 | static bool mmu_enabled; | |
6f1e9e94 | 190 | |
671f9a3e | 191 | #define MAX_EARLY_MAPPING_SIZE SZ_128M |
6f1e9e94 | 192 | |
671f9a3e | 193 | pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); |
f2c17aab AP |
194 | |
195 | void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) | |
196 | { | |
197 | unsigned long addr = __fix_to_virt(idx); | |
198 | pte_t *ptep; | |
199 | ||
200 | BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); | |
201 | ||
202 | ptep = &fixmap_pte[pte_index(addr)]; | |
203 | ||
204 | if (pgprot_val(prot)) { | |
205 | set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); | |
206 | } else { | |
207 | pte_clear(&init_mm, addr, ptep); | |
208 | local_flush_tlb_page(addr); | |
209 | } | |
210 | } | |
211 | ||
671f9a3e AP |
212 | static pte_t *__init get_pte_virt(phys_addr_t pa) |
213 | { | |
214 | if (mmu_enabled) { | |
215 | clear_fixmap(FIX_PTE); | |
216 | return (pte_t *)set_fixmap_offset(FIX_PTE, pa); | |
217 | } else { | |
218 | return (pte_t *)((uintptr_t)pa); | |
219 | } | |
220 | } | |
221 | ||
222 | static phys_addr_t __init alloc_pte(uintptr_t va) | |
223 | { | |
224 | /* | |
225 | * We only create PMD or PGD early mappings so we | |
226 | * should never reach here with MMU disabled. | |
227 | */ | |
228 | BUG_ON(!mmu_enabled); | |
229 | ||
230 | return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); | |
231 | } | |
232 | ||
233 | static void __init create_pte_mapping(pte_t *ptep, | |
234 | uintptr_t va, phys_addr_t pa, | |
235 | phys_addr_t sz, pgprot_t prot) | |
236 | { | |
237 | uintptr_t pte_index = pte_index(va); | |
238 | ||
239 | BUG_ON(sz != PAGE_SIZE); | |
240 | ||
241 | if (pte_none(ptep[pte_index])) | |
242 | ptep[pte_index] = pfn_pte(PFN_DOWN(pa), prot); | |
243 | } | |
244 | ||
245 | #ifndef __PAGETABLE_PMD_FOLDED | |
246 | ||
247 | pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; | |
248 | pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; | |
249 | ||
250 | #if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE | |
251 | #define NUM_EARLY_PMDS 1UL | |
252 | #else | |
253 | #define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) | |
254 | #endif | |
255 | pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); | |
256 | ||
257 | static pmd_t *__init get_pmd_virt(phys_addr_t pa) | |
258 | { | |
259 | if (mmu_enabled) { | |
260 | clear_fixmap(FIX_PMD); | |
261 | return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); | |
262 | } else { | |
263 | return (pmd_t *)((uintptr_t)pa); | |
264 | } | |
265 | } | |
266 | ||
267 | static phys_addr_t __init alloc_pmd(uintptr_t va) | |
268 | { | |
269 | uintptr_t pmd_num; | |
270 | ||
271 | if (mmu_enabled) | |
272 | return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); | |
273 | ||
274 | pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; | |
275 | BUG_ON(pmd_num >= NUM_EARLY_PMDS); | |
276 | return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; | |
277 | } | |
278 | ||
279 | static void __init create_pmd_mapping(pmd_t *pmdp, | |
280 | uintptr_t va, phys_addr_t pa, | |
281 | phys_addr_t sz, pgprot_t prot) | |
282 | { | |
283 | pte_t *ptep; | |
284 | phys_addr_t pte_phys; | |
285 | uintptr_t pmd_index = pmd_index(va); | |
286 | ||
287 | if (sz == PMD_SIZE) { | |
288 | if (pmd_none(pmdp[pmd_index])) | |
289 | pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pa), prot); | |
290 | return; | |
291 | } | |
292 | ||
293 | if (pmd_none(pmdp[pmd_index])) { | |
294 | pte_phys = alloc_pte(va); | |
295 | pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); | |
296 | ptep = get_pte_virt(pte_phys); | |
297 | memset(ptep, 0, PAGE_SIZE); | |
298 | } else { | |
299 | pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_index])); | |
300 | ptep = get_pte_virt(pte_phys); | |
301 | } | |
302 | ||
303 | create_pte_mapping(ptep, va, pa, sz, prot); | |
304 | } | |
305 | ||
306 | #define pgd_next_t pmd_t | |
307 | #define alloc_pgd_next(__va) alloc_pmd(__va) | |
308 | #define get_pgd_next_virt(__pa) get_pmd_virt(__pa) | |
309 | #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ | |
310 | create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) | |
671f9a3e AP |
311 | #define fixmap_pgd_next fixmap_pmd |
312 | #else | |
313 | #define pgd_next_t pte_t | |
314 | #define alloc_pgd_next(__va) alloc_pte(__va) | |
315 | #define get_pgd_next_virt(__pa) get_pte_virt(__pa) | |
316 | #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ | |
317 | create_pte_mapping(__nextp, __va, __pa, __sz, __prot) | |
671f9a3e AP |
318 | #define fixmap_pgd_next fixmap_pte |
319 | #endif | |
320 | ||
321 | static void __init create_pgd_mapping(pgd_t *pgdp, | |
322 | uintptr_t va, phys_addr_t pa, | |
323 | phys_addr_t sz, pgprot_t prot) | |
324 | { | |
325 | pgd_next_t *nextp; | |
326 | phys_addr_t next_phys; | |
327 | uintptr_t pgd_index = pgd_index(va); | |
328 | ||
329 | if (sz == PGDIR_SIZE) { | |
330 | if (pgd_val(pgdp[pgd_index]) == 0) | |
331 | pgdp[pgd_index] = pfn_pgd(PFN_DOWN(pa), prot); | |
332 | return; | |
333 | } | |
334 | ||
335 | if (pgd_val(pgdp[pgd_index]) == 0) { | |
336 | next_phys = alloc_pgd_next(va); | |
337 | pgdp[pgd_index] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); | |
338 | nextp = get_pgd_next_virt(next_phys); | |
339 | memset(nextp, 0, PAGE_SIZE); | |
340 | } else { | |
341 | next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_index])); | |
342 | nextp = get_pgd_next_virt(next_phys); | |
343 | } | |
344 | ||
345 | create_pgd_next_mapping(nextp, va, pa, sz, prot); | |
346 | } | |
347 | ||
348 | static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) | |
349 | { | |
0fdc636c ZL |
350 | /* Upgrade to PMD_SIZE mappings whenever possible */ |
351 | if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) | |
352 | return PAGE_SIZE; | |
671f9a3e | 353 | |
0fdc636c | 354 | return PMD_SIZE; |
671f9a3e AP |
355 | } |
356 | ||
387181dc AP |
357 | /* |
358 | * setup_vm() is called from head.S with MMU-off. | |
359 | * | |
360 | * Following requirements should be honoured for setup_vm() to work | |
361 | * correctly: | |
362 | * 1) It should use PC-relative addressing for accessing kernel symbols. | |
363 | * To achieve this we always use GCC cmodel=medany. | |
364 | * 2) The compiler instrumentation for FTRACE will not work for setup_vm() | |
365 | * so disable compiler instrumentation when FTRACE is enabled. | |
366 | * | |
367 | * Currently, the above requirements are honoured by using custom CFLAGS | |
368 | * for init.o in mm/Makefile. | |
369 | */ | |
370 | ||
371 | #ifndef __riscv_cmodel_medany | |
6a527b67 | 372 | #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." |
387181dc AP |
373 | #endif |
374 | ||
671f9a3e | 375 | asmlinkage void __init setup_vm(uintptr_t dtb_pa) |
6f1e9e94 | 376 | { |
671f9a3e AP |
377 | uintptr_t va, end_va; |
378 | uintptr_t load_pa = (uintptr_t)(&_start); | |
379 | uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; | |
380 | uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); | |
381 | ||
382 | va_pa_offset = PAGE_OFFSET - load_pa; | |
383 | pfn_base = PFN_DOWN(load_pa); | |
6f1e9e94 | 384 | |
671f9a3e AP |
385 | /* |
386 | * Enforce boot alignment requirements of RV32 and | |
387 | * RV64 by only allowing PMD or PGD mappings. | |
388 | */ | |
389 | BUG_ON(map_size == PAGE_SIZE); | |
6f1e9e94 AP |
390 | |
391 | /* Sanity check alignment and size */ | |
392 | BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); | |
671f9a3e AP |
393 | BUG_ON((load_pa % map_size) != 0); |
394 | BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE); | |
395 | ||
396 | /* Setup early PGD for fixmap */ | |
397 | create_pgd_mapping(early_pg_dir, FIXADDR_START, | |
398 | (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); | |
6f1e9e94 AP |
399 | |
400 | #ifndef __PAGETABLE_PMD_FOLDED | |
671f9a3e AP |
401 | /* Setup fixmap PMD */ |
402 | create_pmd_mapping(fixmap_pmd, FIXADDR_START, | |
403 | (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); | |
404 | /* Setup trampoline PGD and PMD */ | |
405 | create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, | |
406 | (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); | |
407 | create_pmd_mapping(trampoline_pmd, PAGE_OFFSET, | |
408 | load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); | |
409 | #else | |
410 | /* Setup trampoline PGD */ | |
411 | create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, | |
412 | load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); | |
413 | #endif | |
6f1e9e94 | 414 | |
671f9a3e AP |
415 | /* |
416 | * Setup early PGD covering entire kernel which will allows | |
417 | * us to reach paging_init(). We map all memory banks later | |
418 | * in setup_vm_final() below. | |
419 | */ | |
420 | end_va = PAGE_OFFSET + load_sz; | |
421 | for (va = PAGE_OFFSET; va < end_va; va += map_size) | |
422 | create_pgd_mapping(early_pg_dir, va, | |
423 | load_pa + (va - PAGE_OFFSET), | |
424 | map_size, PAGE_KERNEL_EXEC); | |
425 | ||
426 | /* Create fixed mapping for early FDT parsing */ | |
427 | end_va = __fix_to_virt(FIX_FDT) + FIX_FDT_SIZE; | |
428 | for (va = __fix_to_virt(FIX_FDT); va < end_va; va += PAGE_SIZE) | |
429 | create_pte_mapping(fixmap_pte, va, | |
430 | dtb_pa + (va - __fix_to_virt(FIX_FDT)), | |
431 | PAGE_SIZE, PAGE_KERNEL); | |
432 | ||
433 | /* Save pointer to DTB for early FDT parsing */ | |
434 | dtb_early_va = (void *)fix_to_virt(FIX_FDT) + (dtb_pa & ~PAGE_MASK); | |
922b0375 AO |
435 | /* Save physical address for memblock reservation */ |
436 | dtb_early_pa = dtb_pa; | |
671f9a3e | 437 | } |
f2c17aab | 438 | |
671f9a3e AP |
439 | static void __init setup_vm_final(void) |
440 | { | |
441 | uintptr_t va, map_size; | |
442 | phys_addr_t pa, start, end; | |
443 | struct memblock_region *reg; | |
444 | ||
445 | /* Set mmu_enabled flag */ | |
446 | mmu_enabled = true; | |
6f1e9e94 | 447 | |
671f9a3e AP |
448 | /* Setup swapper PGD for fixmap */ |
449 | create_pgd_mapping(swapper_pg_dir, FIXADDR_START, | |
ac51e005 | 450 | __pa_symbol(fixmap_pgd_next), |
671f9a3e | 451 | PGDIR_SIZE, PAGE_TABLE); |
f2c17aab | 452 | |
671f9a3e AP |
453 | /* Map all memory banks */ |
454 | for_each_memblock(memory, reg) { | |
455 | start = reg->base; | |
456 | end = start + reg->size; | |
457 | ||
458 | if (start >= end) | |
459 | break; | |
460 | if (memblock_is_nomap(reg)) | |
461 | continue; | |
462 | if (start <= __pa(PAGE_OFFSET) && | |
463 | __pa(PAGE_OFFSET) < end) | |
464 | start = __pa(PAGE_OFFSET); | |
465 | ||
466 | map_size = best_map_size(start, end - start); | |
467 | for (pa = start; pa < end; pa += map_size) { | |
468 | va = (uintptr_t)__va(pa); | |
469 | create_pgd_mapping(swapper_pg_dir, va, pa, | |
470 | map_size, PAGE_KERNEL_EXEC); | |
471 | } | |
6f1e9e94 | 472 | } |
f2c17aab | 473 | |
671f9a3e AP |
474 | /* Clear fixmap PTE and PMD mappings */ |
475 | clear_fixmap(FIX_PTE); | |
476 | clear_fixmap(FIX_PMD); | |
477 | ||
478 | /* Move to swapper page table */ | |
ac51e005 | 479 | csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); |
671f9a3e AP |
480 | local_flush_tlb_all(); |
481 | } | |
d27c3c90 ZL |
482 | |
483 | void free_initmem(void) | |
484 | { | |
485 | unsigned long init_begin = (unsigned long)__init_begin; | |
486 | unsigned long init_end = (unsigned long)__init_end; | |
487 | ||
488 | /* Make the region as non-execuatble. */ | |
489 | set_memory_nx(init_begin, (init_end - init_begin) >> PAGE_SHIFT); | |
490 | free_initmem_default(POISON_FREE_INITMEM); | |
491 | } | |
492 | ||
6bd33e1e CH |
493 | #else |
494 | asmlinkage void __init setup_vm(uintptr_t dtb_pa) | |
495 | { | |
496 | dtb_early_va = (void *)dtb_pa; | |
497 | } | |
498 | ||
499 | static inline void setup_vm_final(void) | |
500 | { | |
501 | } | |
502 | #endif /* CONFIG_MMU */ | |
671f9a3e | 503 | |
d27c3c90 | 504 | #ifdef CONFIG_STRICT_KERNEL_RWX |
d27c3c90 ZL |
505 | void mark_rodata_ro(void) |
506 | { | |
507 | unsigned long text_start = (unsigned long)_text; | |
508 | unsigned long text_end = (unsigned long)_etext; | |
509 | unsigned long rodata_start = (unsigned long)__start_rodata; | |
510 | unsigned long data_start = (unsigned long)_data; | |
511 | unsigned long max_low = (unsigned long)(__va(PFN_PHYS(max_low_pfn))); | |
512 | ||
513 | set_memory_ro(text_start, (text_end - text_start) >> PAGE_SHIFT); | |
514 | set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); | |
515 | set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); | |
516 | set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT); | |
517 | } | |
518 | #endif | |
519 | ||
671f9a3e AP |
520 | void __init paging_init(void) |
521 | { | |
522 | setup_vm_final(); | |
d95f1a54 LG |
523 | memblocks_present(); |
524 | sparse_init(); | |
671f9a3e AP |
525 | setup_zero_page(); |
526 | zone_sizes_init(); | |
6f1e9e94 | 527 | } |
d95f1a54 | 528 | |
9fe57d8c | 529 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
d95f1a54 LG |
530 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, |
531 | struct vmem_altmap *altmap) | |
532 | { | |
533 | return vmemmap_populate_basepages(start, end, node); | |
534 | } | |
535 | #endif |