1 // SPDX-License-Identifier: GPL-2.0-only
3 * This kernel test validates architecture page table helpers and
4 * accessors and helps in verifying their continued compliance with
5 * expected generic MM semantics.
7 * Copyright (C) 2019 ARM Ltd.
9 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
11 #define pr_fmt(fmt) "debug_vm_pgtable: [%-25s]: " fmt, __func__
13 #include <linux/gfp.h>
14 #include <linux/highmem.h>
15 #include <linux/hugetlb.h>
16 #include <linux/kernel.h>
17 #include <linux/kconfig.h>
18 #include <linux/memblock.h>
20 #include <linux/mman.h>
21 #include <linux/mm_types.h>
22 #include <linux/module.h>
23 #include <linux/pfn_t.h>
24 #include <linux/printk.h>
25 #include <linux/pgtable.h>
26 #include <linux/random.h>
27 #include <linux/spinlock.h>
28 #include <linux/swap.h>
29 #include <linux/swapops.h>
30 #include <linux/start_kernel.h>
31 #include <linux/sched/mm.h>
34 #include <asm/cacheflush.h>
35 #include <asm/pgalloc.h>
36 #include <asm/tlbflush.h>
39 * Please refer Documentation/mm/arch_pgtable_helpers.rst for the semantics
40 * expectations that are being validated here. All future changes in here
41 * or the documentation need to be in sync.
43 * On s390 platform, the lower 4 bits are used to identify given page table
44 * entry type. But these bits might affect the ability to clear entries with
45 * pxx_clear() because of how dynamic page table folding works on s390. So
46 * while loading up the entries do not change the lower 4 bits. It does not
47 * have affect any other platform. Also avoid the 62nd bit on ppc64 that is
48 * used to mark a pte entry.
50 #define S390_SKIP_MASK GENMASK(3, 0)
51 #if __BITS_PER_LONG == 64
52 #define PPC64_SKIP_MASK GENMASK(62, 62)
54 #define PPC64_SKIP_MASK 0x0
56 #define ARCH_SKIP_MASK (S390_SKIP_MASK | PPC64_SKIP_MASK)
57 #define RANDOM_ORVALUE (GENMASK(BITS_PER_LONG - 1, 0) & ~ARCH_SKIP_MASK)
58 #define RANDOM_NZVALUE GENMASK(7, 0)
60 struct pgtable_debug_args
{
62 struct vm_area_struct
*vma
;
77 pgprot_t page_prot_none
;
79 bool is_contiguous_page
;
80 unsigned long pud_pfn
;
81 unsigned long pmd_pfn
;
82 unsigned long pte_pfn
;
84 unsigned long fixed_alignment
;
85 unsigned long fixed_pgd_pfn
;
86 unsigned long fixed_p4d_pfn
;
87 unsigned long fixed_pud_pfn
;
88 unsigned long fixed_pmd_pfn
;
89 unsigned long fixed_pte_pfn
;
92 static void __init
pte_basic_tests(struct pgtable_debug_args
*args
, int idx
)
94 pgprot_t prot
= vm_get_page_prot(idx
);
95 pte_t pte
= pfn_pte(args
->fixed_pte_pfn
, prot
);
96 unsigned long val
= idx
, *ptr
= &val
;
98 pr_debug("Validating PTE basic (%pGv)\n", ptr
);
101 * This test needs to be executed after the given page table entry
102 * is created with pfn_pte() to make sure that vm_get_page_prot(idx)
103 * does not have the dirty bit enabled from the beginning. This is
104 * important for platforms like arm64 where (!PTE_RDONLY) indicate
105 * dirty bit being set.
107 WARN_ON(pte_dirty(pte_wrprotect(pte
)));
109 WARN_ON(!pte_same(pte
, pte
));
110 WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte
))));
111 WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte
))));
112 WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte
), args
->vma
)));
113 WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte
))));
114 WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte
))));
115 WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte
, args
->vma
))));
116 WARN_ON(pte_dirty(pte_wrprotect(pte_mkclean(pte
))));
117 WARN_ON(!pte_dirty(pte_wrprotect(pte_mkdirty(pte
))));
120 static void __init
pte_advanced_tests(struct pgtable_debug_args
*args
)
126 * Architectures optimize set_pte_at by avoiding TLB flush.
127 * This requires set_pte_at to be not used to update an
128 * existing pte entry. Clear pte before we do set_pte_at
130 * flush_dcache_page() is called after set_pte_at() to clear
131 * PG_arch_1 for the page on ARM64. The page flag isn't cleared
132 * when it's released and page allocation check will fail when
133 * the page is allocated again. For architectures other than ARM64,
134 * the unexpected overhead of cache flushing is acceptable.
136 page
= (args
->pte_pfn
!= ULONG_MAX
) ? pfn_to_page(args
->pte_pfn
) : NULL
;
140 pr_debug("Validating PTE advanced\n");
141 if (WARN_ON(!args
->ptep
))
144 pte
= pfn_pte(args
->pte_pfn
, args
->page_prot
);
145 set_pte_at(args
->mm
, args
->vaddr
, args
->ptep
, pte
);
146 flush_dcache_page(page
);
147 ptep_set_wrprotect(args
->mm
, args
->vaddr
, args
->ptep
);
148 pte
= ptep_get(args
->ptep
);
149 WARN_ON(pte_write(pte
));
150 ptep_get_and_clear(args
->mm
, args
->vaddr
, args
->ptep
);
151 pte
= ptep_get(args
->ptep
);
152 WARN_ON(!pte_none(pte
));
154 pte
= pfn_pte(args
->pte_pfn
, args
->page_prot
);
155 pte
= pte_wrprotect(pte
);
156 pte
= pte_mkclean(pte
);
157 set_pte_at(args
->mm
, args
->vaddr
, args
->ptep
, pte
);
158 flush_dcache_page(page
);
159 pte
= pte_mkwrite(pte
, args
->vma
);
160 pte
= pte_mkdirty(pte
);
161 ptep_set_access_flags(args
->vma
, args
->vaddr
, args
->ptep
, pte
, 1);
162 pte
= ptep_get(args
->ptep
);
163 WARN_ON(!(pte_write(pte
) && pte_dirty(pte
)));
164 ptep_get_and_clear_full(args
->mm
, args
->vaddr
, args
->ptep
, 1);
165 pte
= ptep_get(args
->ptep
);
166 WARN_ON(!pte_none(pte
));
168 pte
= pfn_pte(args
->pte_pfn
, args
->page_prot
);
169 pte
= pte_mkyoung(pte
);
170 set_pte_at(args
->mm
, args
->vaddr
, args
->ptep
, pte
);
171 flush_dcache_page(page
);
172 ptep_test_and_clear_young(args
->vma
, args
->vaddr
, args
->ptep
);
173 pte
= ptep_get(args
->ptep
);
174 WARN_ON(pte_young(pte
));
176 ptep_get_and_clear_full(args
->mm
, args
->vaddr
, args
->ptep
, 1);
179 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
180 static void __init
pmd_basic_tests(struct pgtable_debug_args
*args
, int idx
)
182 pgprot_t prot
= vm_get_page_prot(idx
);
183 unsigned long val
= idx
, *ptr
= &val
;
186 if (!has_transparent_hugepage())
189 pr_debug("Validating PMD basic (%pGv)\n", ptr
);
190 pmd
= pfn_pmd(args
->fixed_pmd_pfn
, prot
);
193 * This test needs to be executed after the given page table entry
194 * is created with pfn_pmd() to make sure that vm_get_page_prot(idx)
195 * does not have the dirty bit enabled from the beginning. This is
196 * important for platforms like arm64 where (!PTE_RDONLY) indicate
197 * dirty bit being set.
199 WARN_ON(pmd_dirty(pmd_wrprotect(pmd
)));
202 WARN_ON(!pmd_same(pmd
, pmd
));
203 WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd
))));
204 WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd
))));
205 WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd
), args
->vma
)));
206 WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd
))));
207 WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd
))));
208 WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd
, args
->vma
))));
209 WARN_ON(pmd_dirty(pmd_wrprotect(pmd_mkclean(pmd
))));
210 WARN_ON(!pmd_dirty(pmd_wrprotect(pmd_mkdirty(pmd
))));
212 * A huge page does not point to next level page table
213 * entry. Hence this must qualify as pmd_bad().
215 WARN_ON(!pmd_bad(pmd_mkhuge(pmd
)));
218 static void __init
pmd_advanced_tests(struct pgtable_debug_args
*args
)
222 unsigned long vaddr
= args
->vaddr
;
224 if (!has_transparent_hugepage())
227 page
= (args
->pmd_pfn
!= ULONG_MAX
) ? pfn_to_page(args
->pmd_pfn
) : NULL
;
232 * flush_dcache_page() is called after set_pmd_at() to clear
233 * PG_arch_1 for the page on ARM64. The page flag isn't cleared
234 * when it's released and page allocation check will fail when
235 * the page is allocated again. For architectures other than ARM64,
236 * the unexpected overhead of cache flushing is acceptable.
238 pr_debug("Validating PMD advanced\n");
239 /* Align the address wrt HPAGE_PMD_SIZE */
240 vaddr
&= HPAGE_PMD_MASK
;
242 pgtable_trans_huge_deposit(args
->mm
, args
->pmdp
, args
->start_ptep
);
244 pmd
= pfn_pmd(args
->pmd_pfn
, args
->page_prot
);
245 set_pmd_at(args
->mm
, vaddr
, args
->pmdp
, pmd
);
246 flush_dcache_page(page
);
247 pmdp_set_wrprotect(args
->mm
, vaddr
, args
->pmdp
);
248 pmd
= READ_ONCE(*args
->pmdp
);
249 WARN_ON(pmd_write(pmd
));
250 pmdp_huge_get_and_clear(args
->mm
, vaddr
, args
->pmdp
);
251 pmd
= READ_ONCE(*args
->pmdp
);
252 WARN_ON(!pmd_none(pmd
));
254 pmd
= pfn_pmd(args
->pmd_pfn
, args
->page_prot
);
255 pmd
= pmd_wrprotect(pmd
);
256 pmd
= pmd_mkclean(pmd
);
257 set_pmd_at(args
->mm
, vaddr
, args
->pmdp
, pmd
);
258 flush_dcache_page(page
);
259 pmd
= pmd_mkwrite(pmd
, args
->vma
);
260 pmd
= pmd_mkdirty(pmd
);
261 pmdp_set_access_flags(args
->vma
, vaddr
, args
->pmdp
, pmd
, 1);
262 pmd
= READ_ONCE(*args
->pmdp
);
263 WARN_ON(!(pmd_write(pmd
) && pmd_dirty(pmd
)));
264 pmdp_huge_get_and_clear_full(args
->vma
, vaddr
, args
->pmdp
, 1);
265 pmd
= READ_ONCE(*args
->pmdp
);
266 WARN_ON(!pmd_none(pmd
));
268 pmd
= pmd_mkhuge(pfn_pmd(args
->pmd_pfn
, args
->page_prot
));
269 pmd
= pmd_mkyoung(pmd
);
270 set_pmd_at(args
->mm
, vaddr
, args
->pmdp
, pmd
);
271 flush_dcache_page(page
);
272 pmdp_test_and_clear_young(args
->vma
, vaddr
, args
->pmdp
);
273 pmd
= READ_ONCE(*args
->pmdp
);
274 WARN_ON(pmd_young(pmd
));
276 /* Clear the pte entries */
277 pmdp_huge_get_and_clear(args
->mm
, vaddr
, args
->pmdp
);
278 pgtable_trans_huge_withdraw(args
->mm
, args
->pmdp
);
281 static void __init
pmd_leaf_tests(struct pgtable_debug_args
*args
)
285 if (!has_transparent_hugepage())
288 pr_debug("Validating PMD leaf\n");
289 pmd
= pfn_pmd(args
->fixed_pmd_pfn
, args
->page_prot
);
292 * PMD based THP is a leaf entry.
294 pmd
= pmd_mkhuge(pmd
);
295 WARN_ON(!pmd_leaf(pmd
));
298 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
299 static void __init
pud_basic_tests(struct pgtable_debug_args
*args
, int idx
)
301 pgprot_t prot
= vm_get_page_prot(idx
);
302 unsigned long val
= idx
, *ptr
= &val
;
305 if (!has_transparent_pud_hugepage())
308 pr_debug("Validating PUD basic (%pGv)\n", ptr
);
309 pud
= pfn_pud(args
->fixed_pud_pfn
, prot
);
312 * This test needs to be executed after the given page table entry
313 * is created with pfn_pud() to make sure that vm_get_page_prot(idx)
314 * does not have the dirty bit enabled from the beginning. This is
315 * important for platforms like arm64 where (!PTE_RDONLY) indicate
316 * dirty bit being set.
318 WARN_ON(pud_dirty(pud_wrprotect(pud
)));
320 WARN_ON(!pud_same(pud
, pud
));
321 WARN_ON(!pud_young(pud_mkyoung(pud_mkold(pud
))));
322 WARN_ON(!pud_dirty(pud_mkdirty(pud_mkclean(pud
))));
323 WARN_ON(pud_dirty(pud_mkclean(pud_mkdirty(pud
))));
324 WARN_ON(!pud_write(pud_mkwrite(pud_wrprotect(pud
))));
325 WARN_ON(pud_write(pud_wrprotect(pud_mkwrite(pud
))));
326 WARN_ON(pud_young(pud_mkold(pud_mkyoung(pud
))));
327 WARN_ON(pud_dirty(pud_wrprotect(pud_mkclean(pud
))));
328 WARN_ON(!pud_dirty(pud_wrprotect(pud_mkdirty(pud
))));
330 if (mm_pmd_folded(args
->mm
))
334 * A huge page does not point to next level page table
335 * entry. Hence this must qualify as pud_bad().
337 WARN_ON(!pud_bad(pud_mkhuge(pud
)));
340 static void __init
pud_advanced_tests(struct pgtable_debug_args
*args
)
343 unsigned long vaddr
= args
->vaddr
;
346 if (!has_transparent_pud_hugepage())
349 page
= (args
->pud_pfn
!= ULONG_MAX
) ? pfn_to_page(args
->pud_pfn
) : NULL
;
354 * flush_dcache_page() is called after set_pud_at() to clear
355 * PG_arch_1 for the page on ARM64. The page flag isn't cleared
356 * when it's released and page allocation check will fail when
357 * the page is allocated again. For architectures other than ARM64,
358 * the unexpected overhead of cache flushing is acceptable.
360 pr_debug("Validating PUD advanced\n");
361 /* Align the address wrt HPAGE_PUD_SIZE */
362 vaddr
&= HPAGE_PUD_MASK
;
364 pud
= pfn_pud(args
->pud_pfn
, args
->page_prot
);
365 set_pud_at(args
->mm
, vaddr
, args
->pudp
, pud
);
366 flush_dcache_page(page
);
367 pudp_set_wrprotect(args
->mm
, vaddr
, args
->pudp
);
368 pud
= READ_ONCE(*args
->pudp
);
369 WARN_ON(pud_write(pud
));
371 #ifndef __PAGETABLE_PMD_FOLDED
372 pudp_huge_get_and_clear(args
->mm
, vaddr
, args
->pudp
);
373 pud
= READ_ONCE(*args
->pudp
);
374 WARN_ON(!pud_none(pud
));
375 #endif /* __PAGETABLE_PMD_FOLDED */
376 pud
= pfn_pud(args
->pud_pfn
, args
->page_prot
);
377 pud
= pud_wrprotect(pud
);
378 pud
= pud_mkclean(pud
);
379 set_pud_at(args
->mm
, vaddr
, args
->pudp
, pud
);
380 flush_dcache_page(page
);
381 pud
= pud_mkwrite(pud
);
382 pud
= pud_mkdirty(pud
);
383 pudp_set_access_flags(args
->vma
, vaddr
, args
->pudp
, pud
, 1);
384 pud
= READ_ONCE(*args
->pudp
);
385 WARN_ON(!(pud_write(pud
) && pud_dirty(pud
)));
387 #ifndef __PAGETABLE_PMD_FOLDED
388 pudp_huge_get_and_clear_full(args
->vma
, vaddr
, args
->pudp
, 1);
389 pud
= READ_ONCE(*args
->pudp
);
390 WARN_ON(!pud_none(pud
));
391 #endif /* __PAGETABLE_PMD_FOLDED */
393 pud
= pfn_pud(args
->pud_pfn
, args
->page_prot
);
394 pud
= pud_mkyoung(pud
);
395 set_pud_at(args
->mm
, vaddr
, args
->pudp
, pud
);
396 flush_dcache_page(page
);
397 pudp_test_and_clear_young(args
->vma
, vaddr
, args
->pudp
);
398 pud
= READ_ONCE(*args
->pudp
);
399 WARN_ON(pud_young(pud
));
401 pudp_huge_get_and_clear(args
->mm
, vaddr
, args
->pudp
);
404 static void __init
pud_leaf_tests(struct pgtable_debug_args
*args
)
408 if (!has_transparent_pud_hugepage())
411 pr_debug("Validating PUD leaf\n");
412 pud
= pfn_pud(args
->fixed_pud_pfn
, args
->page_prot
);
414 * PUD based THP is a leaf entry.
416 pud
= pud_mkhuge(pud
);
417 WARN_ON(!pud_leaf(pud
));
419 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
420 static void __init
pud_basic_tests(struct pgtable_debug_args
*args
, int idx
) { }
421 static void __init
pud_advanced_tests(struct pgtable_debug_args
*args
) { }
422 static void __init
pud_leaf_tests(struct pgtable_debug_args
*args
) { }
423 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
424 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
425 static void __init
pmd_basic_tests(struct pgtable_debug_args
*args
, int idx
) { }
426 static void __init
pud_basic_tests(struct pgtable_debug_args
*args
, int idx
) { }
427 static void __init
pmd_advanced_tests(struct pgtable_debug_args
*args
) { }
428 static void __init
pud_advanced_tests(struct pgtable_debug_args
*args
) { }
429 static void __init
pmd_leaf_tests(struct pgtable_debug_args
*args
) { }
430 static void __init
pud_leaf_tests(struct pgtable_debug_args
*args
) { }
431 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
433 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
434 static void __init
pmd_huge_tests(struct pgtable_debug_args
*args
)
438 if (!arch_vmap_pmd_supported(args
->page_prot
) ||
439 args
->fixed_alignment
< PMD_SIZE
)
442 pr_debug("Validating PMD huge\n");
444 * X86 defined pmd_set_huge() verifies that the given
445 * PMD is not a populated non-leaf entry.
447 WRITE_ONCE(*args
->pmdp
, __pmd(0));
448 WARN_ON(!pmd_set_huge(args
->pmdp
, __pfn_to_phys(args
->fixed_pmd_pfn
), args
->page_prot
));
449 WARN_ON(!pmd_clear_huge(args
->pmdp
));
450 pmd
= READ_ONCE(*args
->pmdp
);
451 WARN_ON(!pmd_none(pmd
));
454 static void __init
pud_huge_tests(struct pgtable_debug_args
*args
)
458 if (!arch_vmap_pud_supported(args
->page_prot
) ||
459 args
->fixed_alignment
< PUD_SIZE
)
462 pr_debug("Validating PUD huge\n");
464 * X86 defined pud_set_huge() verifies that the given
465 * PUD is not a populated non-leaf entry.
467 WRITE_ONCE(*args
->pudp
, __pud(0));
468 WARN_ON(!pud_set_huge(args
->pudp
, __pfn_to_phys(args
->fixed_pud_pfn
), args
->page_prot
));
469 WARN_ON(!pud_clear_huge(args
->pudp
));
470 pud
= READ_ONCE(*args
->pudp
);
471 WARN_ON(!pud_none(pud
));
473 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
474 static void __init
pmd_huge_tests(struct pgtable_debug_args
*args
) { }
475 static void __init
pud_huge_tests(struct pgtable_debug_args
*args
) { }
476 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
478 static void __init
p4d_basic_tests(struct pgtable_debug_args
*args
)
482 pr_debug("Validating P4D basic\n");
483 memset(&p4d
, RANDOM_NZVALUE
, sizeof(p4d_t
));
484 WARN_ON(!p4d_same(p4d
, p4d
));
487 static void __init
pgd_basic_tests(struct pgtable_debug_args
*args
)
491 pr_debug("Validating PGD basic\n");
492 memset(&pgd
, RANDOM_NZVALUE
, sizeof(pgd_t
));
493 WARN_ON(!pgd_same(pgd
, pgd
));
496 #ifndef __PAGETABLE_PUD_FOLDED
497 static void __init
pud_clear_tests(struct pgtable_debug_args
*args
)
499 pud_t pud
= READ_ONCE(*args
->pudp
);
501 if (mm_pmd_folded(args
->mm
))
504 pr_debug("Validating PUD clear\n");
505 pud
= __pud(pud_val(pud
) | RANDOM_ORVALUE
);
506 WRITE_ONCE(*args
->pudp
, pud
);
507 pud_clear(args
->pudp
);
508 pud
= READ_ONCE(*args
->pudp
);
509 WARN_ON(!pud_none(pud
));
512 static void __init
pud_populate_tests(struct pgtable_debug_args
*args
)
516 if (mm_pmd_folded(args
->mm
))
519 pr_debug("Validating PUD populate\n");
521 * This entry points to next level page table page.
522 * Hence this must not qualify as pud_bad().
524 pud_populate(args
->mm
, args
->pudp
, args
->start_pmdp
);
525 pud
= READ_ONCE(*args
->pudp
);
526 WARN_ON(pud_bad(pud
));
528 #else /* !__PAGETABLE_PUD_FOLDED */
529 static void __init
pud_clear_tests(struct pgtable_debug_args
*args
) { }
530 static void __init
pud_populate_tests(struct pgtable_debug_args
*args
) { }
531 #endif /* PAGETABLE_PUD_FOLDED */
533 #ifndef __PAGETABLE_P4D_FOLDED
534 static void __init
p4d_clear_tests(struct pgtable_debug_args
*args
)
536 p4d_t p4d
= READ_ONCE(*args
->p4dp
);
538 if (mm_pud_folded(args
->mm
))
541 pr_debug("Validating P4D clear\n");
542 p4d
= __p4d(p4d_val(p4d
) | RANDOM_ORVALUE
);
543 WRITE_ONCE(*args
->p4dp
, p4d
);
544 p4d_clear(args
->p4dp
);
545 p4d
= READ_ONCE(*args
->p4dp
);
546 WARN_ON(!p4d_none(p4d
));
549 static void __init
p4d_populate_tests(struct pgtable_debug_args
*args
)
553 if (mm_pud_folded(args
->mm
))
556 pr_debug("Validating P4D populate\n");
558 * This entry points to next level page table page.
559 * Hence this must not qualify as p4d_bad().
561 pud_clear(args
->pudp
);
562 p4d_clear(args
->p4dp
);
563 p4d_populate(args
->mm
, args
->p4dp
, args
->start_pudp
);
564 p4d
= READ_ONCE(*args
->p4dp
);
565 WARN_ON(p4d_bad(p4d
));
568 static void __init
pgd_clear_tests(struct pgtable_debug_args
*args
)
570 pgd_t pgd
= READ_ONCE(*(args
->pgdp
));
572 if (mm_p4d_folded(args
->mm
))
575 pr_debug("Validating PGD clear\n");
576 pgd
= __pgd(pgd_val(pgd
) | RANDOM_ORVALUE
);
577 WRITE_ONCE(*args
->pgdp
, pgd
);
578 pgd_clear(args
->pgdp
);
579 pgd
= READ_ONCE(*args
->pgdp
);
580 WARN_ON(!pgd_none(pgd
));
583 static void __init
pgd_populate_tests(struct pgtable_debug_args
*args
)
587 if (mm_p4d_folded(args
->mm
))
590 pr_debug("Validating PGD populate\n");
592 * This entry points to next level page table page.
593 * Hence this must not qualify as pgd_bad().
595 p4d_clear(args
->p4dp
);
596 pgd_clear(args
->pgdp
);
597 pgd_populate(args
->mm
, args
->pgdp
, args
->start_p4dp
);
598 pgd
= READ_ONCE(*args
->pgdp
);
599 WARN_ON(pgd_bad(pgd
));
601 #else /* !__PAGETABLE_P4D_FOLDED */
602 static void __init
p4d_clear_tests(struct pgtable_debug_args
*args
) { }
603 static void __init
pgd_clear_tests(struct pgtable_debug_args
*args
) { }
604 static void __init
p4d_populate_tests(struct pgtable_debug_args
*args
) { }
605 static void __init
pgd_populate_tests(struct pgtable_debug_args
*args
) { }
606 #endif /* PAGETABLE_P4D_FOLDED */
608 static void __init
pte_clear_tests(struct pgtable_debug_args
*args
)
611 pte_t pte
= pfn_pte(args
->pte_pfn
, args
->page_prot
);
613 page
= (args
->pte_pfn
!= ULONG_MAX
) ? pfn_to_page(args
->pte_pfn
) : NULL
;
618 * flush_dcache_page() is called after set_pte_at() to clear
619 * PG_arch_1 for the page on ARM64. The page flag isn't cleared
620 * when it's released and page allocation check will fail when
621 * the page is allocated again. For architectures other than ARM64,
622 * the unexpected overhead of cache flushing is acceptable.
624 pr_debug("Validating PTE clear\n");
625 if (WARN_ON(!args
->ptep
))
629 pte
= __pte(pte_val(pte
) | RANDOM_ORVALUE
);
631 set_pte_at(args
->mm
, args
->vaddr
, args
->ptep
, pte
);
632 flush_dcache_page(page
);
634 ptep_clear(args
->mm
, args
->vaddr
, args
->ptep
);
635 pte
= ptep_get(args
->ptep
);
636 WARN_ON(!pte_none(pte
));
639 static void __init
pmd_clear_tests(struct pgtable_debug_args
*args
)
641 pmd_t pmd
= READ_ONCE(*args
->pmdp
);
643 pr_debug("Validating PMD clear\n");
644 pmd
= __pmd(pmd_val(pmd
) | RANDOM_ORVALUE
);
645 WRITE_ONCE(*args
->pmdp
, pmd
);
646 pmd_clear(args
->pmdp
);
647 pmd
= READ_ONCE(*args
->pmdp
);
648 WARN_ON(!pmd_none(pmd
));
651 static void __init
pmd_populate_tests(struct pgtable_debug_args
*args
)
655 pr_debug("Validating PMD populate\n");
657 * This entry points to next level page table page.
658 * Hence this must not qualify as pmd_bad().
660 pmd_populate(args
->mm
, args
->pmdp
, args
->start_ptep
);
661 pmd
= READ_ONCE(*args
->pmdp
);
662 WARN_ON(pmd_bad(pmd
));
665 static void __init
pte_special_tests(struct pgtable_debug_args
*args
)
667 pte_t pte
= pfn_pte(args
->fixed_pte_pfn
, args
->page_prot
);
669 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL
))
672 pr_debug("Validating PTE special\n");
673 WARN_ON(!pte_special(pte_mkspecial(pte
)));
676 static void __init
pte_protnone_tests(struct pgtable_debug_args
*args
)
678 pte_t pte
= pfn_pte(args
->fixed_pte_pfn
, args
->page_prot_none
);
680 if (!IS_ENABLED(CONFIG_NUMA_BALANCING
))
683 pr_debug("Validating PTE protnone\n");
684 WARN_ON(!pte_protnone(pte
));
685 WARN_ON(!pte_present(pte
));
688 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
689 static void __init
pmd_protnone_tests(struct pgtable_debug_args
*args
)
693 if (!IS_ENABLED(CONFIG_NUMA_BALANCING
))
696 if (!has_transparent_hugepage())
699 pr_debug("Validating PMD protnone\n");
700 pmd
= pmd_mkhuge(pfn_pmd(args
->fixed_pmd_pfn
, args
->page_prot_none
));
701 WARN_ON(!pmd_protnone(pmd
));
702 WARN_ON(!pmd_present(pmd
));
704 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
705 static void __init
pmd_protnone_tests(struct pgtable_debug_args
*args
) { }
706 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
708 #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
709 static void __init
pte_devmap_tests(struct pgtable_debug_args
*args
)
711 pte_t pte
= pfn_pte(args
->fixed_pte_pfn
, args
->page_prot
);
713 pr_debug("Validating PTE devmap\n");
714 WARN_ON(!pte_devmap(pte_mkdevmap(pte
)));
717 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
718 static void __init
pmd_devmap_tests(struct pgtable_debug_args
*args
)
722 if (!has_transparent_hugepage())
725 pr_debug("Validating PMD devmap\n");
726 pmd
= pfn_pmd(args
->fixed_pmd_pfn
, args
->page_prot
);
727 WARN_ON(!pmd_devmap(pmd_mkdevmap(pmd
)));
730 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
731 static void __init
pud_devmap_tests(struct pgtable_debug_args
*args
)
735 if (!has_transparent_pud_hugepage())
738 pr_debug("Validating PUD devmap\n");
739 pud
= pfn_pud(args
->fixed_pud_pfn
, args
->page_prot
);
740 WARN_ON(!pud_devmap(pud_mkdevmap(pud
)));
742 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
743 static void __init
pud_devmap_tests(struct pgtable_debug_args
*args
) { }
744 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
745 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
746 static void __init
pmd_devmap_tests(struct pgtable_debug_args
*args
) { }
747 static void __init
pud_devmap_tests(struct pgtable_debug_args
*args
) { }
748 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
750 static void __init
pte_devmap_tests(struct pgtable_debug_args
*args
) { }
751 static void __init
pmd_devmap_tests(struct pgtable_debug_args
*args
) { }
752 static void __init
pud_devmap_tests(struct pgtable_debug_args
*args
) { }
753 #endif /* CONFIG_ARCH_HAS_PTE_DEVMAP */
755 static void __init
pte_soft_dirty_tests(struct pgtable_debug_args
*args
)
757 pte_t pte
= pfn_pte(args
->fixed_pte_pfn
, args
->page_prot
);
759 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY
))
762 pr_debug("Validating PTE soft dirty\n");
763 WARN_ON(!pte_soft_dirty(pte_mksoft_dirty(pte
)));
764 WARN_ON(pte_soft_dirty(pte_clear_soft_dirty(pte
)));
767 static void __init
pte_swap_soft_dirty_tests(struct pgtable_debug_args
*args
)
769 pte_t pte
= pfn_pte(args
->fixed_pte_pfn
, args
->page_prot
);
771 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY
))
774 pr_debug("Validating PTE swap soft dirty\n");
775 WARN_ON(!pte_swp_soft_dirty(pte_swp_mksoft_dirty(pte
)));
776 WARN_ON(pte_swp_soft_dirty(pte_swp_clear_soft_dirty(pte
)));
779 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
780 static void __init
pmd_soft_dirty_tests(struct pgtable_debug_args
*args
)
784 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY
))
787 if (!has_transparent_hugepage())
790 pr_debug("Validating PMD soft dirty\n");
791 pmd
= pfn_pmd(args
->fixed_pmd_pfn
, args
->page_prot
);
792 WARN_ON(!pmd_soft_dirty(pmd_mksoft_dirty(pmd
)));
793 WARN_ON(pmd_soft_dirty(pmd_clear_soft_dirty(pmd
)));
796 static void __init
pmd_swap_soft_dirty_tests(struct pgtable_debug_args
*args
)
800 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY
) ||
801 !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION
))
804 if (!has_transparent_hugepage())
807 pr_debug("Validating PMD swap soft dirty\n");
808 pmd
= pfn_pmd(args
->fixed_pmd_pfn
, args
->page_prot
);
809 WARN_ON(!pmd_swp_soft_dirty(pmd_swp_mksoft_dirty(pmd
)));
810 WARN_ON(pmd_swp_soft_dirty(pmd_swp_clear_soft_dirty(pmd
)));
812 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
813 static void __init
pmd_soft_dirty_tests(struct pgtable_debug_args
*args
) { }
814 static void __init
pmd_swap_soft_dirty_tests(struct pgtable_debug_args
*args
) { }
815 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
817 static void __init
pte_swap_exclusive_tests(struct pgtable_debug_args
*args
)
819 unsigned long max_swap_offset
;
820 swp_entry_t entry
, entry2
;
823 pr_debug("Validating PTE swap exclusive\n");
825 /* See generic_max_swapfile_size(): probe the maximum offset */
826 max_swap_offset
= swp_offset(pte_to_swp_entry(swp_entry_to_pte(swp_entry(0, ~0UL))));
828 /* Create a swp entry with all possible bits set */
829 entry
= swp_entry((1 << MAX_SWAPFILES_SHIFT
) - 1, max_swap_offset
);
831 pte
= swp_entry_to_pte(entry
);
832 WARN_ON(pte_swp_exclusive(pte
));
833 WARN_ON(!is_swap_pte(pte
));
834 entry2
= pte_to_swp_entry(pte
);
835 WARN_ON(memcmp(&entry
, &entry2
, sizeof(entry
)));
837 pte
= pte_swp_mkexclusive(pte
);
838 WARN_ON(!pte_swp_exclusive(pte
));
839 WARN_ON(!is_swap_pte(pte
));
840 WARN_ON(pte_swp_soft_dirty(pte
));
841 entry2
= pte_to_swp_entry(pte
);
842 WARN_ON(memcmp(&entry
, &entry2
, sizeof(entry
)));
844 pte
= pte_swp_clear_exclusive(pte
);
845 WARN_ON(pte_swp_exclusive(pte
));
846 WARN_ON(!is_swap_pte(pte
));
847 entry2
= pte_to_swp_entry(pte
);
848 WARN_ON(memcmp(&entry
, &entry2
, sizeof(entry
)));
851 static void __init
pte_swap_tests(struct pgtable_debug_args
*args
)
856 pr_debug("Validating PTE swap\n");
857 pte
= pfn_pte(args
->fixed_pte_pfn
, args
->page_prot
);
858 swp
= __pte_to_swp_entry(pte
);
859 pte
= __swp_entry_to_pte(swp
);
860 WARN_ON(args
->fixed_pte_pfn
!= pte_pfn(pte
));
863 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
864 static void __init
pmd_swap_tests(struct pgtable_debug_args
*args
)
869 if (!has_transparent_hugepage())
872 pr_debug("Validating PMD swap\n");
873 pmd
= pfn_pmd(args
->fixed_pmd_pfn
, args
->page_prot
);
874 swp
= __pmd_to_swp_entry(pmd
);
875 pmd
= __swp_entry_to_pmd(swp
);
876 WARN_ON(args
->fixed_pmd_pfn
!= pmd_pfn(pmd
));
878 #else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */
879 static void __init
pmd_swap_tests(struct pgtable_debug_args
*args
) { }
880 #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
882 static void __init
swap_migration_tests(struct pgtable_debug_args
*args
)
887 if (!IS_ENABLED(CONFIG_MIGRATION
))
891 * swap_migration_tests() requires a dedicated page as it needs to
892 * be locked before creating a migration entry from it. Locking the
893 * page that actually maps kernel text ('start_kernel') can be real
894 * problematic. Lets use the allocated page explicitly for this
897 page
= (args
->pte_pfn
!= ULONG_MAX
) ? pfn_to_page(args
->pte_pfn
) : NULL
;
901 pr_debug("Validating swap migration\n");
904 * make_[readable|writable]_migration_entry() expects given page to
905 * be locked, otherwise it stumbles upon a BUG_ON().
907 __SetPageLocked(page
);
908 swp
= make_writable_migration_entry(page_to_pfn(page
));
909 WARN_ON(!is_migration_entry(swp
));
910 WARN_ON(!is_writable_migration_entry(swp
));
912 swp
= make_readable_migration_entry(swp_offset(swp
));
913 WARN_ON(!is_migration_entry(swp
));
914 WARN_ON(is_writable_migration_entry(swp
));
916 swp
= make_readable_migration_entry(page_to_pfn(page
));
917 WARN_ON(!is_migration_entry(swp
));
918 WARN_ON(is_writable_migration_entry(swp
));
919 __ClearPageLocked(page
);
922 #ifdef CONFIG_HUGETLB_PAGE
923 static void __init
hugetlb_basic_tests(struct pgtable_debug_args
*args
)
928 pr_debug("Validating HugeTLB basic\n");
930 * Accessing the page associated with the pfn is safe here,
931 * as it was previously derived from a real kernel symbol.
933 page
= pfn_to_page(args
->fixed_pmd_pfn
);
934 pte
= mk_huge_pte(page
, args
->page_prot
);
936 WARN_ON(!huge_pte_dirty(huge_pte_mkdirty(pte
)));
937 WARN_ON(!huge_pte_write(huge_pte_mkwrite(huge_pte_wrprotect(pte
))));
938 WARN_ON(huge_pte_write(huge_pte_wrprotect(huge_pte_mkwrite(pte
))));
940 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
941 pte
= pfn_pte(args
->fixed_pmd_pfn
, args
->page_prot
);
943 WARN_ON(!pte_huge(arch_make_huge_pte(pte
, PMD_SHIFT
, VM_ACCESS_FLAGS
)));
944 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
946 #else /* !CONFIG_HUGETLB_PAGE */
947 static void __init
hugetlb_basic_tests(struct pgtable_debug_args
*args
) { }
948 #endif /* CONFIG_HUGETLB_PAGE */
950 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
951 static void __init
pmd_thp_tests(struct pgtable_debug_args
*args
)
955 if (!has_transparent_hugepage())
958 pr_debug("Validating PMD based THP\n");
960 * pmd_trans_huge() and pmd_present() must return positive after
961 * MMU invalidation with pmd_mkinvalid(). This behavior is an
962 * optimization for transparent huge page. pmd_trans_huge() must
963 * be true if pmd_page() returns a valid THP to avoid taking the
964 * pmd_lock when others walk over non transhuge pmds (i.e. there
965 * are no THP allocated). Especially when splitting a THP and
966 * removing the present bit from the pmd, pmd_trans_huge() still
967 * needs to return true. pmd_present() should be true whenever
968 * pmd_trans_huge() returns true.
970 pmd
= pfn_pmd(args
->fixed_pmd_pfn
, args
->page_prot
);
971 WARN_ON(!pmd_trans_huge(pmd_mkhuge(pmd
)));
973 #ifndef __HAVE_ARCH_PMDP_INVALIDATE
974 WARN_ON(!pmd_trans_huge(pmd_mkinvalid(pmd_mkhuge(pmd
))));
975 WARN_ON(!pmd_present(pmd_mkinvalid(pmd_mkhuge(pmd
))));
976 #endif /* __HAVE_ARCH_PMDP_INVALIDATE */
979 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
980 static void __init
pud_thp_tests(struct pgtable_debug_args
*args
)
984 if (!has_transparent_pud_hugepage())
987 pr_debug("Validating PUD based THP\n");
988 pud
= pfn_pud(args
->fixed_pud_pfn
, args
->page_prot
);
989 WARN_ON(!pud_trans_huge(pud_mkhuge(pud
)));
992 * pud_mkinvalid() has been dropped for now. Enable back
993 * these tests when it comes back with a modified pud_present().
995 * WARN_ON(!pud_trans_huge(pud_mkinvalid(pud_mkhuge(pud))));
996 * WARN_ON(!pud_present(pud_mkinvalid(pud_mkhuge(pud))));
999 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
1000 static void __init
pud_thp_tests(struct pgtable_debug_args
*args
) { }
1001 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
1002 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
1003 static void __init
pmd_thp_tests(struct pgtable_debug_args
*args
) { }
1004 static void __init
pud_thp_tests(struct pgtable_debug_args
*args
) { }
1005 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1007 static unsigned long __init
get_random_vaddr(void)
1009 unsigned long random_vaddr
, random_pages
, total_user_pages
;
1011 total_user_pages
= (TASK_SIZE
- FIRST_USER_ADDRESS
) / PAGE_SIZE
;
1013 random_pages
= get_random_long() % total_user_pages
;
1014 random_vaddr
= FIRST_USER_ADDRESS
+ random_pages
* PAGE_SIZE
;
1016 return random_vaddr
;
1019 static void __init
destroy_args(struct pgtable_debug_args
*args
)
1021 struct page
*page
= NULL
;
1023 /* Free (huge) page */
1024 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE
) &&
1025 has_transparent_pud_hugepage() &&
1026 args
->pud_pfn
!= ULONG_MAX
) {
1027 if (args
->is_contiguous_page
) {
1028 free_contig_range(args
->pud_pfn
,
1029 (1 << (HPAGE_PUD_SHIFT
- PAGE_SHIFT
)));
1031 page
= pfn_to_page(args
->pud_pfn
);
1032 __free_pages(page
, HPAGE_PUD_SHIFT
- PAGE_SHIFT
);
1035 args
->pud_pfn
= ULONG_MAX
;
1036 args
->pmd_pfn
= ULONG_MAX
;
1037 args
->pte_pfn
= ULONG_MAX
;
1040 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE
) &&
1041 has_transparent_hugepage() &&
1042 args
->pmd_pfn
!= ULONG_MAX
) {
1043 if (args
->is_contiguous_page
) {
1044 free_contig_range(args
->pmd_pfn
, (1 << HPAGE_PMD_ORDER
));
1046 page
= pfn_to_page(args
->pmd_pfn
);
1047 __free_pages(page
, HPAGE_PMD_ORDER
);
1050 args
->pmd_pfn
= ULONG_MAX
;
1051 args
->pte_pfn
= ULONG_MAX
;
1054 if (args
->pte_pfn
!= ULONG_MAX
) {
1055 page
= pfn_to_page(args
->pte_pfn
);
1058 args
->pte_pfn
= ULONG_MAX
;
1061 /* Free page table entries */
1062 if (args
->start_ptep
) {
1063 pte_free(args
->mm
, args
->start_ptep
);
1064 mm_dec_nr_ptes(args
->mm
);
1067 if (args
->start_pmdp
) {
1068 pmd_free(args
->mm
, args
->start_pmdp
);
1069 mm_dec_nr_pmds(args
->mm
);
1072 if (args
->start_pudp
) {
1073 pud_free(args
->mm
, args
->start_pudp
);
1074 mm_dec_nr_puds(args
->mm
);
1077 if (args
->start_p4dp
)
1078 p4d_free(args
->mm
, args
->start_p4dp
);
1080 /* Free vma and mm struct */
1082 vm_area_free(args
->vma
);
1088 static struct page
* __init
1089 debug_vm_pgtable_alloc_huge_page(struct pgtable_debug_args
*args
, int order
)
1091 struct page
*page
= NULL
;
1093 #ifdef CONFIG_CONTIG_ALLOC
1094 if (order
> MAX_ORDER
) {
1095 page
= alloc_contig_pages((1 << order
), GFP_KERNEL
,
1096 first_online_node
, NULL
);
1098 args
->is_contiguous_page
= true;
1104 if (order
<= MAX_ORDER
)
1105 page
= alloc_pages(GFP_KERNEL
, order
);
1111 * Check if a physical memory range described by <pstart, pend> contains
1112 * an area that is of size psize, and aligned to psize.
1114 * Don't use address 0, an all-zeroes physical address might mask bugs, and
1115 * it's not used on x86.
1117 static void __init
phys_align_check(phys_addr_t pstart
,
1118 phys_addr_t pend
, unsigned long psize
,
1119 phys_addr_t
*physp
, unsigned long *alignp
)
1121 phys_addr_t aligned_start
, aligned_end
;
1126 aligned_start
= ALIGN(pstart
, psize
);
1127 aligned_end
= aligned_start
+ psize
;
1129 if (aligned_end
> aligned_start
&& aligned_end
<= pend
) {
1131 *physp
= aligned_start
;
1135 static void __init
init_fixed_pfns(struct pgtable_debug_args
*args
)
1138 phys_addr_t phys
, pstart
, pend
;
1141 * Initialize the fixed pfns. To do this, try to find a
1142 * valid physical range, preferably aligned to PUD_SIZE,
1143 * but settling for aligned to PMD_SIZE as a fallback. If
1144 * neither of those is found, use the physical address of
1145 * the start_kernel symbol.
1147 * The memory doesn't need to be allocated, it just needs to exist
1148 * as usable memory. It won't be touched.
1150 * The alignment is recorded, and can be checked to see if we
1151 * can run the tests that require an actual valid physical
1152 * address range on some architectures ({pmd,pud}_huge_test
1156 phys
= __pa_symbol(&start_kernel
);
1157 args
->fixed_alignment
= PAGE_SIZE
;
1159 for_each_mem_range(idx
, &pstart
, &pend
) {
1160 /* First check for a PUD-aligned area */
1161 phys_align_check(pstart
, pend
, PUD_SIZE
, &phys
,
1162 &args
->fixed_alignment
);
1164 /* If a PUD-aligned area is found, we're done */
1165 if (args
->fixed_alignment
== PUD_SIZE
)
1169 * If no PMD-aligned area found yet, check for one,
1170 * but continue the loop to look for a PUD-aligned area.
1172 if (args
->fixed_alignment
< PMD_SIZE
)
1173 phys_align_check(pstart
, pend
, PMD_SIZE
, &phys
,
1174 &args
->fixed_alignment
);
1177 args
->fixed_pgd_pfn
= __phys_to_pfn(phys
& PGDIR_MASK
);
1178 args
->fixed_p4d_pfn
= __phys_to_pfn(phys
& P4D_MASK
);
1179 args
->fixed_pud_pfn
= __phys_to_pfn(phys
& PUD_MASK
);
1180 args
->fixed_pmd_pfn
= __phys_to_pfn(phys
& PMD_MASK
);
1181 args
->fixed_pte_pfn
= __phys_to_pfn(phys
& PAGE_MASK
);
1182 WARN_ON(!pfn_valid(args
->fixed_pte_pfn
));
1186 static int __init
init_args(struct pgtable_debug_args
*args
)
1188 struct page
*page
= NULL
;
1192 * Initialize the debugging data.
1194 * vm_get_page_prot(VM_NONE) or vm_get_page_prot(VM_SHARED|VM_NONE)
1195 * will help create page table entries with PROT_NONE permission as
1196 * required for pxx_protnone_tests().
1198 memset(args
, 0, sizeof(*args
));
1199 args
->vaddr
= get_random_vaddr();
1200 args
->page_prot
= vm_get_page_prot(VM_ACCESS_FLAGS
);
1201 args
->page_prot_none
= vm_get_page_prot(VM_NONE
);
1202 args
->is_contiguous_page
= false;
1203 args
->pud_pfn
= ULONG_MAX
;
1204 args
->pmd_pfn
= ULONG_MAX
;
1205 args
->pte_pfn
= ULONG_MAX
;
1206 args
->fixed_pgd_pfn
= ULONG_MAX
;
1207 args
->fixed_p4d_pfn
= ULONG_MAX
;
1208 args
->fixed_pud_pfn
= ULONG_MAX
;
1209 args
->fixed_pmd_pfn
= ULONG_MAX
;
1210 args
->fixed_pte_pfn
= ULONG_MAX
;
1212 /* Allocate mm and vma */
1213 args
->mm
= mm_alloc();
1215 pr_err("Failed to allocate mm struct\n");
1220 args
->vma
= vm_area_alloc(args
->mm
);
1222 pr_err("Failed to allocate vma\n");
1228 * Allocate page table entries. They will be modified in the tests.
1229 * Lets save the page table entries so that they can be released
1230 * when the tests are completed.
1232 args
->pgdp
= pgd_offset(args
->mm
, args
->vaddr
);
1233 args
->p4dp
= p4d_alloc(args
->mm
, args
->pgdp
, args
->vaddr
);
1235 pr_err("Failed to allocate p4d entries\n");
1239 args
->start_p4dp
= p4d_offset(args
->pgdp
, 0UL);
1240 WARN_ON(!args
->start_p4dp
);
1242 args
->pudp
= pud_alloc(args
->mm
, args
->p4dp
, args
->vaddr
);
1244 pr_err("Failed to allocate pud entries\n");
1248 args
->start_pudp
= pud_offset(args
->p4dp
, 0UL);
1249 WARN_ON(!args
->start_pudp
);
1251 args
->pmdp
= pmd_alloc(args
->mm
, args
->pudp
, args
->vaddr
);
1253 pr_err("Failed to allocate pmd entries\n");
1257 args
->start_pmdp
= pmd_offset(args
->pudp
, 0UL);
1258 WARN_ON(!args
->start_pmdp
);
1260 if (pte_alloc(args
->mm
, args
->pmdp
)) {
1261 pr_err("Failed to allocate pte entries\n");
1265 args
->start_ptep
= pmd_pgtable(READ_ONCE(*args
->pmdp
));
1266 WARN_ON(!args
->start_ptep
);
1268 init_fixed_pfns(args
);
1271 * Allocate (huge) pages because some of the tests need to access
1272 * the data in the pages. The corresponding tests will be skipped
1273 * if we fail to allocate (huge) pages.
1275 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE
) &&
1276 has_transparent_pud_hugepage()) {
1277 page
= debug_vm_pgtable_alloc_huge_page(args
,
1278 HPAGE_PUD_SHIFT
- PAGE_SHIFT
);
1280 args
->pud_pfn
= page_to_pfn(page
);
1281 args
->pmd_pfn
= args
->pud_pfn
;
1282 args
->pte_pfn
= args
->pud_pfn
;
1287 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE
) &&
1288 has_transparent_hugepage()) {
1289 page
= debug_vm_pgtable_alloc_huge_page(args
, HPAGE_PMD_ORDER
);
1291 args
->pmd_pfn
= page_to_pfn(page
);
1292 args
->pte_pfn
= args
->pmd_pfn
;
1297 page
= alloc_page(GFP_KERNEL
);
1299 args
->pte_pfn
= page_to_pfn(page
);
1308 static int __init
debug_vm_pgtable(void)
1310 struct pgtable_debug_args args
;
1311 spinlock_t
*ptl
= NULL
;
1314 pr_info("Validating architecture page table helpers\n");
1315 ret
= init_args(&args
);
1320 * Iterate over each possible vm_flags to make sure that all
1321 * the basic page table transformation validations just hold
1322 * true irrespective of the starting protection value for a
1323 * given page table entry.
1325 * Protection based vm_flags combinatins are always linear
1326 * and increasing i.e starting from VM_NONE and going upto
1327 * (VM_SHARED | READ | WRITE | EXEC).
1329 #define VM_FLAGS_START (VM_NONE)
1330 #define VM_FLAGS_END (VM_SHARED | VM_EXEC | VM_WRITE | VM_READ)
1332 for (idx
= VM_FLAGS_START
; idx
<= VM_FLAGS_END
; idx
++) {
1333 pte_basic_tests(&args
, idx
);
1334 pmd_basic_tests(&args
, idx
);
1335 pud_basic_tests(&args
, idx
);
1339 * Both P4D and PGD level tests are very basic which do not
1340 * involve creating page table entries from the protection
1341 * value and the given pfn. Hence just keep them out from
1342 * the above iteration for now to save some test execution
1345 p4d_basic_tests(&args
);
1346 pgd_basic_tests(&args
);
1348 pmd_leaf_tests(&args
);
1349 pud_leaf_tests(&args
);
1351 pte_special_tests(&args
);
1352 pte_protnone_tests(&args
);
1353 pmd_protnone_tests(&args
);
1355 pte_devmap_tests(&args
);
1356 pmd_devmap_tests(&args
);
1357 pud_devmap_tests(&args
);
1359 pte_soft_dirty_tests(&args
);
1360 pmd_soft_dirty_tests(&args
);
1361 pte_swap_soft_dirty_tests(&args
);
1362 pmd_swap_soft_dirty_tests(&args
);
1364 pte_swap_exclusive_tests(&args
);
1366 pte_swap_tests(&args
);
1367 pmd_swap_tests(&args
);
1369 swap_migration_tests(&args
);
1371 pmd_thp_tests(&args
);
1372 pud_thp_tests(&args
);
1374 hugetlb_basic_tests(&args
);
1377 * Page table modifying tests. They need to hold
1378 * proper page table lock.
1381 args
.ptep
= pte_offset_map_lock(args
.mm
, args
.pmdp
, args
.vaddr
, &ptl
);
1382 pte_clear_tests(&args
);
1383 pte_advanced_tests(&args
);
1385 pte_unmap_unlock(args
.ptep
, ptl
);
1387 ptl
= pmd_lock(args
.mm
, args
.pmdp
);
1388 pmd_clear_tests(&args
);
1389 pmd_advanced_tests(&args
);
1390 pmd_huge_tests(&args
);
1391 pmd_populate_tests(&args
);
1394 ptl
= pud_lock(args
.mm
, args
.pudp
);
1395 pud_clear_tests(&args
);
1396 pud_advanced_tests(&args
);
1397 pud_huge_tests(&args
);
1398 pud_populate_tests(&args
);
1401 spin_lock(&(args
.mm
->page_table_lock
));
1402 p4d_clear_tests(&args
);
1403 pgd_clear_tests(&args
);
1404 p4d_populate_tests(&args
);
1405 pgd_populate_tests(&args
);
1406 spin_unlock(&(args
.mm
->page_table_lock
));
1408 destroy_args(&args
);
1411 late_initcall(debug_vm_pgtable
);