]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
be3606ff | 2 | #define DISABLE_BRANCH_PROFILING |
85155229 | 3 | #define pr_fmt(fmt) "kasan: " fmt |
39b95522 | 4 | |
ad3fe525 KS |
5 | /* cpu_feature_enabled() cannot be used this early */ |
6 | #define USE_EARLY_PGTABLE_L5 | |
39b95522 | 7 | |
57c8a661 | 8 | #include <linux/memblock.h> |
ef7f0d6a AR |
9 | #include <linux/kasan.h> |
10 | #include <linux/kdebug.h> | |
11 | #include <linux/mm.h> | |
12 | #include <linux/sched.h> | |
9164bb4a | 13 | #include <linux/sched/task.h> |
ef7f0d6a AR |
14 | #include <linux/vmalloc.h> |
15 | ||
5520b7e7 | 16 | #include <asm/e820/types.h> |
2aeb0736 | 17 | #include <asm/pgalloc.h> |
ef7f0d6a AR |
18 | #include <asm/tlbflush.h> |
19 | #include <asm/sections.h> | |
b9d05200 | 20 | #include <asm/pgtable.h> |
92a0f81d | 21 | #include <asm/cpu_entry_area.h> |
ef7f0d6a | 22 | |
08b46d5d | 23 | extern struct range pfn_mapped[E820_MAX_ENTRIES]; |
ef7f0d6a | 24 | |
c65e774f | 25 | static p4d_t tmp_p4d_table[MAX_PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE); |
12a8cc7f | 26 | |
26fb3dae | 27 | static __init void *early_alloc(size_t size, int nid, bool should_panic) |
2aeb0736 | 28 | { |
26fb3dae | 29 | void *ptr = memblock_alloc_try_nid(size, size, |
97ad1087 | 30 | __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); |
26fb3dae MR |
31 | |
32 | if (!ptr && should_panic) | |
33 | panic("%pS: Failed to allocate page, nid=%d from=%lx\n", | |
34 | (void *)_RET_IP_, nid, __pa(MAX_DMA_ADDRESS)); | |
35 | ||
36 | return ptr; | |
2aeb0736 AR |
37 | } |
38 | ||
39 | static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr, | |
40 | unsigned long end, int nid) | |
41 | { | |
42 | pte_t *pte; | |
43 | ||
44 | if (pmd_none(*pmd)) { | |
45 | void *p; | |
46 | ||
47 | if (boot_cpu_has(X86_FEATURE_PSE) && | |
48 | ((end - addr) == PMD_SIZE) && | |
49 | IS_ALIGNED(addr, PMD_SIZE)) { | |
0d39e266 | 50 | p = early_alloc(PMD_SIZE, nid, false); |
2aeb0736 AR |
51 | if (p && pmd_set_huge(pmd, __pa(p), PAGE_KERNEL)) |
52 | return; | |
53 | else if (p) | |
54 | memblock_free(__pa(p), PMD_SIZE); | |
55 | } | |
56 | ||
0d39e266 | 57 | p = early_alloc(PAGE_SIZE, nid, true); |
2aeb0736 AR |
58 | pmd_populate_kernel(&init_mm, pmd, p); |
59 | } | |
60 | ||
61 | pte = pte_offset_kernel(pmd, addr); | |
62 | do { | |
63 | pte_t entry; | |
64 | void *p; | |
65 | ||
66 | if (!pte_none(*pte)) | |
67 | continue; | |
68 | ||
0d39e266 | 69 | p = early_alloc(PAGE_SIZE, nid, true); |
2aeb0736 AR |
70 | entry = pfn_pte(PFN_DOWN(__pa(p)), PAGE_KERNEL); |
71 | set_pte_at(&init_mm, addr, pte, entry); | |
72 | } while (pte++, addr += PAGE_SIZE, addr != end); | |
73 | } | |
74 | ||
75 | static void __init kasan_populate_pud(pud_t *pud, unsigned long addr, | |
76 | unsigned long end, int nid) | |
77 | { | |
78 | pmd_t *pmd; | |
79 | unsigned long next; | |
80 | ||
81 | if (pud_none(*pud)) { | |
82 | void *p; | |
83 | ||
84 | if (boot_cpu_has(X86_FEATURE_GBPAGES) && | |
85 | ((end - addr) == PUD_SIZE) && | |
86 | IS_ALIGNED(addr, PUD_SIZE)) { | |
0d39e266 | 87 | p = early_alloc(PUD_SIZE, nid, false); |
2aeb0736 AR |
88 | if (p && pud_set_huge(pud, __pa(p), PAGE_KERNEL)) |
89 | return; | |
90 | else if (p) | |
91 | memblock_free(__pa(p), PUD_SIZE); | |
92 | } | |
93 | ||
0d39e266 | 94 | p = early_alloc(PAGE_SIZE, nid, true); |
2aeb0736 AR |
95 | pud_populate(&init_mm, pud, p); |
96 | } | |
97 | ||
98 | pmd = pmd_offset(pud, addr); | |
99 | do { | |
100 | next = pmd_addr_end(addr, end); | |
101 | if (!pmd_large(*pmd)) | |
102 | kasan_populate_pmd(pmd, addr, next, nid); | |
103 | } while (pmd++, addr = next, addr != end); | |
104 | } | |
105 | ||
106 | static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr, | |
107 | unsigned long end, int nid) | |
108 | { | |
109 | pud_t *pud; | |
110 | unsigned long next; | |
111 | ||
112 | if (p4d_none(*p4d)) { | |
0d39e266 | 113 | void *p = early_alloc(PAGE_SIZE, nid, true); |
2aeb0736 AR |
114 | |
115 | p4d_populate(&init_mm, p4d, p); | |
116 | } | |
117 | ||
118 | pud = pud_offset(p4d, addr); | |
119 | do { | |
120 | next = pud_addr_end(addr, end); | |
121 | if (!pud_large(*pud)) | |
122 | kasan_populate_pud(pud, addr, next, nid); | |
123 | } while (pud++, addr = next, addr != end); | |
124 | } | |
125 | ||
126 | static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr, | |
127 | unsigned long end, int nid) | |
128 | { | |
129 | void *p; | |
130 | p4d_t *p4d; | |
131 | unsigned long next; | |
132 | ||
133 | if (pgd_none(*pgd)) { | |
0d39e266 | 134 | p = early_alloc(PAGE_SIZE, nid, true); |
2aeb0736 AR |
135 | pgd_populate(&init_mm, pgd, p); |
136 | } | |
137 | ||
138 | p4d = p4d_offset(pgd, addr); | |
139 | do { | |
140 | next = p4d_addr_end(addr, end); | |
141 | kasan_populate_p4d(p4d, addr, next, nid); | |
142 | } while (p4d++, addr = next, addr != end); | |
143 | } | |
144 | ||
145 | static void __init kasan_populate_shadow(unsigned long addr, unsigned long end, | |
146 | int nid) | |
147 | { | |
148 | pgd_t *pgd; | |
149 | unsigned long next; | |
150 | ||
151 | addr = addr & PAGE_MASK; | |
152 | end = round_up(end, PAGE_SIZE); | |
153 | pgd = pgd_offset_k(addr); | |
154 | do { | |
155 | next = pgd_addr_end(addr, end); | |
156 | kasan_populate_pgd(pgd, addr, next, nid); | |
157 | } while (pgd++, addr = next, addr != end); | |
158 | } | |
159 | ||
160 | static void __init map_range(struct range *range) | |
ef7f0d6a AR |
161 | { |
162 | unsigned long start; | |
163 | unsigned long end; | |
164 | ||
165 | start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start)); | |
166 | end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end)); | |
167 | ||
2aeb0736 | 168 | kasan_populate_shadow(start, end, early_pfn_to_nid(range->start)); |
ef7f0d6a AR |
169 | } |
170 | ||
171 | static void __init clear_pgds(unsigned long start, | |
172 | unsigned long end) | |
173 | { | |
d691a3cf | 174 | pgd_t *pgd; |
12a8cc7f AR |
175 | /* See comment in kasan_init() */ |
176 | unsigned long pgd_end = end & PGDIR_MASK; | |
d691a3cf | 177 | |
12a8cc7f | 178 | for (; start < pgd_end; start += PGDIR_SIZE) { |
d691a3cf KS |
179 | pgd = pgd_offset_k(start); |
180 | /* | |
181 | * With folded p4d, pgd_clear() is nop, use p4d_clear() | |
182 | * instead. | |
183 | */ | |
ed7588d5 | 184 | if (pgtable_l5_enabled()) |
d691a3cf | 185 | pgd_clear(pgd); |
91f606a8 KS |
186 | else |
187 | p4d_clear(p4d_offset(pgd, start)); | |
d691a3cf | 188 | } |
12a8cc7f AR |
189 | |
190 | pgd = pgd_offset_k(start); | |
191 | for (; start < end; start += P4D_SIZE) | |
192 | p4d_clear(p4d_offset(pgd, start)); | |
193 | } | |
194 | ||
195 | static inline p4d_t *early_p4d_offset(pgd_t *pgd, unsigned long addr) | |
196 | { | |
197 | unsigned long p4d; | |
198 | ||
ed7588d5 | 199 | if (!pgtable_l5_enabled()) |
12a8cc7f AR |
200 | return (p4d_t *)pgd; |
201 | ||
f3176ec9 | 202 | p4d = pgd_val(*pgd) & PTE_PFN_MASK; |
12a8cc7f AR |
203 | p4d += __START_KERNEL_map - phys_base; |
204 | return (p4d_t *)p4d + p4d_index(addr); | |
205 | } | |
206 | ||
207 | static void __init kasan_early_p4d_populate(pgd_t *pgd, | |
208 | unsigned long addr, | |
209 | unsigned long end) | |
210 | { | |
211 | pgd_t pgd_entry; | |
212 | p4d_t *p4d, p4d_entry; | |
213 | unsigned long next; | |
214 | ||
215 | if (pgd_none(*pgd)) { | |
9577dd74 AK |
216 | pgd_entry = __pgd(_KERNPG_TABLE | |
217 | __pa_nodebug(kasan_early_shadow_p4d)); | |
12a8cc7f AR |
218 | set_pgd(pgd, pgd_entry); |
219 | } | |
220 | ||
221 | p4d = early_p4d_offset(pgd, addr); | |
222 | do { | |
223 | next = p4d_addr_end(addr, end); | |
224 | ||
225 | if (!p4d_none(*p4d)) | |
226 | continue; | |
227 | ||
9577dd74 AK |
228 | p4d_entry = __p4d(_KERNPG_TABLE | |
229 | __pa_nodebug(kasan_early_shadow_pud)); | |
12a8cc7f AR |
230 | set_p4d(p4d, p4d_entry); |
231 | } while (p4d++, addr = next, addr != end && p4d_none(*p4d)); | |
ef7f0d6a AR |
232 | } |
233 | ||
5d5aa3cf | 234 | static void __init kasan_map_early_shadow(pgd_t *pgd) |
ef7f0d6a | 235 | { |
12a8cc7f AR |
236 | /* See comment in kasan_init() */ |
237 | unsigned long addr = KASAN_SHADOW_START & PGDIR_MASK; | |
ef7f0d6a | 238 | unsigned long end = KASAN_SHADOW_END; |
12a8cc7f | 239 | unsigned long next; |
ef7f0d6a | 240 | |
12a8cc7f AR |
241 | pgd += pgd_index(addr); |
242 | do { | |
243 | next = pgd_addr_end(addr, end); | |
244 | kasan_early_p4d_populate(pgd, addr, next); | |
245 | } while (pgd++, addr = next, addr != end); | |
ef7f0d6a AR |
246 | } |
247 | ||
ef7f0d6a AR |
248 | #ifdef CONFIG_KASAN_INLINE |
249 | static int kasan_die_handler(struct notifier_block *self, | |
250 | unsigned long val, | |
251 | void *data) | |
252 | { | |
253 | if (val == DIE_GPF) { | |
2ba78056 DV |
254 | pr_emerg("CONFIG_KASAN_INLINE enabled\n"); |
255 | pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n"); | |
ef7f0d6a AR |
256 | } |
257 | return NOTIFY_OK; | |
258 | } | |
259 | ||
260 | static struct notifier_block kasan_die_notifier = { | |
261 | .notifier_call = kasan_die_handler, | |
262 | }; | |
263 | #endif | |
264 | ||
5d5aa3cf AP |
265 | void __init kasan_early_init(void) |
266 | { | |
267 | int i; | |
9577dd74 AK |
268 | pteval_t pte_val = __pa_nodebug(kasan_early_shadow_page) | |
269 | __PAGE_KERNEL | _PAGE_ENC; | |
270 | pmdval_t pmd_val = __pa_nodebug(kasan_early_shadow_pte) | _KERNPG_TABLE; | |
271 | pudval_t pud_val = __pa_nodebug(kasan_early_shadow_pmd) | _KERNPG_TABLE; | |
272 | p4dval_t p4d_val = __pa_nodebug(kasan_early_shadow_pud) | _KERNPG_TABLE; | |
5d5aa3cf | 273 | |
fb43d6cb DH |
274 | /* Mask out unsupported __PAGE_KERNEL bits: */ |
275 | pte_val &= __default_kernel_pte_mask; | |
276 | pmd_val &= __default_kernel_pte_mask; | |
277 | pud_val &= __default_kernel_pte_mask; | |
278 | p4d_val &= __default_kernel_pte_mask; | |
279 | ||
5d5aa3cf | 280 | for (i = 0; i < PTRS_PER_PTE; i++) |
9577dd74 | 281 | kasan_early_shadow_pte[i] = __pte(pte_val); |
5d5aa3cf AP |
282 | |
283 | for (i = 0; i < PTRS_PER_PMD; i++) | |
9577dd74 | 284 | kasan_early_shadow_pmd[i] = __pmd(pmd_val); |
5d5aa3cf AP |
285 | |
286 | for (i = 0; i < PTRS_PER_PUD; i++) | |
9577dd74 | 287 | kasan_early_shadow_pud[i] = __pud(pud_val); |
5d5aa3cf | 288 | |
ed7588d5 | 289 | for (i = 0; pgtable_l5_enabled() && i < PTRS_PER_P4D; i++) |
9577dd74 | 290 | kasan_early_shadow_p4d[i] = __p4d(p4d_val); |
5480bb61 | 291 | |
65ade2f8 KS |
292 | kasan_map_early_shadow(early_top_pgt); |
293 | kasan_map_early_shadow(init_top_pgt); | |
5d5aa3cf AP |
294 | } |
295 | ||
ef7f0d6a AR |
296 | void __init kasan_init(void) |
297 | { | |
298 | int i; | |
21506525 | 299 | void *shadow_cpu_entry_begin, *shadow_cpu_entry_end; |
ef7f0d6a AR |
300 | |
301 | #ifdef CONFIG_KASAN_INLINE | |
302 | register_die_notifier(&kasan_die_notifier); | |
303 | #endif | |
304 | ||
65ade2f8 | 305 | memcpy(early_top_pgt, init_top_pgt, sizeof(early_top_pgt)); |
12a8cc7f AR |
306 | |
307 | /* | |
308 | * We use the same shadow offset for 4- and 5-level paging to | |
309 | * facilitate boot-time switching between paging modes. | |
310 | * As result in 5-level paging mode KASAN_SHADOW_START and | |
311 | * KASAN_SHADOW_END are not aligned to PGD boundary. | |
312 | * | |
313 | * KASAN_SHADOW_START doesn't share PGD with anything else. | |
314 | * We claim whole PGD entry to make things easier. | |
315 | * | |
316 | * KASAN_SHADOW_END lands in the last PGD entry and it collides with | |
317 | * bunch of things like kernel code, modules, EFI mapping, etc. | |
318 | * We need to take extra steps to not overwrite them. | |
319 | */ | |
ed7588d5 | 320 | if (pgtable_l5_enabled()) { |
12a8cc7f AR |
321 | void *ptr; |
322 | ||
323 | ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END)); | |
324 | memcpy(tmp_p4d_table, (void *)ptr, sizeof(tmp_p4d_table)); | |
325 | set_pgd(&early_top_pgt[pgd_index(KASAN_SHADOW_END)], | |
326 | __pgd(__pa(tmp_p4d_table) | _KERNPG_TABLE)); | |
327 | } | |
328 | ||
65ade2f8 | 329 | load_cr3(early_top_pgt); |
241d2c54 | 330 | __flush_tlb_all(); |
ef7f0d6a | 331 | |
12a8cc7f | 332 | clear_pgds(KASAN_SHADOW_START & PGDIR_MASK, KASAN_SHADOW_END); |
ef7f0d6a | 333 | |
9577dd74 | 334 | kasan_populate_early_shadow((void *)(KASAN_SHADOW_START & PGDIR_MASK), |
ef7f0d6a AR |
335 | kasan_mem_to_shadow((void *)PAGE_OFFSET)); |
336 | ||
08b46d5d | 337 | for (i = 0; i < E820_MAX_ENTRIES; i++) { |
ef7f0d6a AR |
338 | if (pfn_mapped[i].end == 0) |
339 | break; | |
340 | ||
2aeb0736 | 341 | map_range(&pfn_mapped[i]); |
ef7f0d6a | 342 | } |
2aeb0736 | 343 | |
92a0f81d | 344 | shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE; |
21506525 | 345 | shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin); |
9577dd74 AK |
346 | shadow_cpu_entry_begin = (void *)round_down( |
347 | (unsigned long)shadow_cpu_entry_begin, PAGE_SIZE); | |
21506525 | 348 | |
92a0f81d TG |
349 | shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE + |
350 | CPU_ENTRY_AREA_MAP_SIZE); | |
21506525 | 351 | shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end); |
9577dd74 AK |
352 | shadow_cpu_entry_end = (void *)round_up( |
353 | (unsigned long)shadow_cpu_entry_end, PAGE_SIZE); | |
21506525 | 354 | |
9577dd74 | 355 | kasan_populate_early_shadow( |
92a0f81d TG |
356 | kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), |
357 | shadow_cpu_entry_begin); | |
21506525 AL |
358 | |
359 | kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin, | |
360 | (unsigned long)shadow_cpu_entry_end, 0); | |
361 | ||
9577dd74 AK |
362 | kasan_populate_early_shadow(shadow_cpu_entry_end, |
363 | kasan_mem_to_shadow((void *)__START_KERNEL_map)); | |
92a0f81d TG |
364 | |
365 | kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext), | |
366 | (unsigned long)kasan_mem_to_shadow(_end), | |
367 | early_pfn_to_nid(__pa(_stext))); | |
368 | ||
9577dd74 AK |
369 | kasan_populate_early_shadow(kasan_mem_to_shadow((void *)MODULES_END), |
370 | (void *)KASAN_SHADOW_END); | |
ef7f0d6a | 371 | |
65ade2f8 | 372 | load_cr3(init_top_pgt); |
241d2c54 | 373 | __flush_tlb_all(); |
85155229 | 374 | |
69e0210f | 375 | /* |
9577dd74 AK |
376 | * kasan_early_shadow_page has been used as early shadow memory, thus |
377 | * it may contain some garbage. Now we can clear and write protect it, | |
378 | * since after the TLB flush no one should write to it. | |
69e0210f | 379 | */ |
9577dd74 | 380 | memset(kasan_early_shadow_page, 0, PAGE_SIZE); |
063fb3e5 | 381 | for (i = 0; i < PTRS_PER_PTE; i++) { |
fb43d6cb DH |
382 | pte_t pte; |
383 | pgprot_t prot; | |
384 | ||
385 | prot = __pgprot(__PAGE_KERNEL_RO | _PAGE_ENC); | |
386 | pgprot_val(prot) &= __default_kernel_pte_mask; | |
387 | ||
9577dd74 AK |
388 | pte = __pte(__pa(kasan_early_shadow_page) | pgprot_val(prot)); |
389 | set_pte(&kasan_early_shadow_pte[i], pte); | |
063fb3e5 AR |
390 | } |
391 | /* Flush TLBs again to be sure that write protection applied. */ | |
392 | __flush_tlb_all(); | |
69e0210f AR |
393 | |
394 | init_task.kasan_depth = 0; | |
25add7ec | 395 | pr_info("KernelAddressSanitizer initialized\n"); |
ef7f0d6a | 396 | } |