mm: actually mark kernel page table pages

commit 977870522af34359b461060597ee3a86f27450d6 upstream.

Now that the API is in place, mark kernel page table pages just after they
are allocated.  Unmark them just before they are freed.

Note: Unconditionally clearing the 'kernel' marking (via
ptdesc_clear_kernel()) would be functionally identical to what is here.
But having the if() makes it logically clear that this function can be
used for kernel and non-kernel page tables.

Link: https://lkml.kernel.org/r/20251022082635.2462433-4-baolu.lu@linux.intel.com
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Acked-by: David Hildenbrand <david@redhat.com>
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Betkov <bp@alien8.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jann Horn <jannh@google.com>
Cc: Jean-Philippe Brucker <jean-philippe@linaro.org>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Robin Murohy <robin.murphy@arm.com>
Cc: Thomas Gleinxer <tglx@linutronix.de>
Cc: "Uladzislau Rezki (Sony)" <urezki@gmail.com>
Cc: Vasant Hegde <vasant.hegde@amd.com>
Cc: Vinicius Costa Gomes <vinicius.gomes@intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Cc: Yi Lai <yi1.lai@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
index 3c8ec3bfea447874df48fbe81427be91a798d65e..b9d2a7c79b93a13223ed0a9c5a91f28b82d7f58f 100644 (file)
--- a/include/asm-generic/pgalloc.h
+++ b/include/asm-generic/pgalloc.h
@@ -28,6 +28,8 @@ static inline pte_t *__pte_alloc_one_kernel_noprof(struct mm_struct *mm)
                return NULL;
        }
 
+       ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define __pte_alloc_one_kernel(...)    alloc_hooks(__pte_alloc_one_kernel_noprof(__VA_ARGS__))
@@ -146,6 +148,10 @@ static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long ad
                pagetable_free(ptdesc);
                return NULL;
        }
+
+       if (mm == &init_mm)
+               ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define pmd_alloc_one(...)     alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__))
@@ -179,6 +185,10 @@ static inline pud_t *__pud_alloc_one_noprof(struct mm_struct *mm, unsigned long
                return NULL;
 
        pagetable_pud_ctor(ptdesc);
+
+       if (mm == &init_mm)
+               ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define __pud_alloc_one(...)   alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS__))
@@ -233,6 +243,10 @@ static inline p4d_t *__p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long
                return NULL;
 
        pagetable_p4d_ctor(ptdesc);
+
+       if (mm == &init_mm)
+               ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define __p4d_alloc_one(...)   alloc_hooks(__p4d_alloc_one_noprof(__VA_ARGS__))
@@ -277,6 +291,10 @@ static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order
                return NULL;
 
        pagetable_pgd_ctor(ptdesc);
+
+       if (mm == &init_mm)
+               ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define __pgd_alloc(...)       alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__))

diff --git a/include/linux/mm.h b/include/linux/mm.h
index d622756f4e38b9e70d1ccabbf8ac30e7113dcd5d..1f4305693d0ff1ec0488e8758bd5428be68851dd 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3042,6 +3042,9 @@ static inline void pagetable_free(struct ptdesc *pt)
 {
        struct page *page = ptdesc_page(pt);
 
+       if (ptdesc_test_kernel(pt))
+               ptdesc_clear_kernel(pt);
+
        __free_pages(page, compound_order(page));
 }