mm: actually mark kernel page table pages

Now that the API is in place, mark kernel page table pages just after they
are allocated.  Unmark them just before they are freed.

Note: Unconditionally clearing the 'kernel' marking (via
ptdesc_clear_kernel()) would be functionally identical to what is here.
But having the if() makes it logically clear that this function can be
used for kernel and non-kernel page tables.

Link: https://lkml.kernel.org/r/20251022082635.2462433-4-baolu.lu@linux.intel.com
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Acked-by: David Hildenbrand <david@redhat.com>
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Betkov <bp@alien8.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jann Horn <jannh@google.com>
Cc: Jean-Philippe Brucker <jean-philippe@linaro.org>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Robin Murohy <robin.murphy@arm.com>
Cc: Thomas Gleinxer <tglx@linutronix.de>
Cc: "Uladzislau Rezki (Sony)" <urezki@gmail.com>
Cc: Vasant Hegde <vasant.hegde@amd.com>
Cc: Vinicius Costa Gomes <vinicius.gomes@intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Cc: Yi Lai <yi1.lai@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
index 3c8ec3bfea447874df48fbe81427be91a798d65e..b9d2a7c79b93a13223ed0a9c5a91f28b82d7f58f 100644 (file)
--- a/include/asm-generic/pgalloc.h
+++ b/include/asm-generic/pgalloc.h
@@ -28,6 +28,8 @@ static inline pte_t *__pte_alloc_one_kernel_noprof(struct mm_struct *mm)
                return NULL;
        }
 
+       ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define __pte_alloc_one_kernel(...)    alloc_hooks(__pte_alloc_one_kernel_noprof(__VA_ARGS__))
@@ -146,6 +148,10 @@ static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long ad
                pagetable_free(ptdesc);
                return NULL;
        }
+
+       if (mm == &init_mm)
+               ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define pmd_alloc_one(...)     alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__))
@@ -179,6 +185,10 @@ static inline pud_t *__pud_alloc_one_noprof(struct mm_struct *mm, unsigned long
                return NULL;
 
        pagetable_pud_ctor(ptdesc);
+
+       if (mm == &init_mm)
+               ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define __pud_alloc_one(...)   alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS__))
@@ -233,6 +243,10 @@ static inline p4d_t *__p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long
                return NULL;
 
        pagetable_p4d_ctor(ptdesc);
+
+       if (mm == &init_mm)
+               ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define __p4d_alloc_one(...)   alloc_hooks(__p4d_alloc_one_noprof(__VA_ARGS__))
@@ -277,6 +291,10 @@ static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order
                return NULL;
 
        pagetable_pgd_ctor(ptdesc);
+
+       if (mm == &init_mm)
+               ptdesc_set_kernel(ptdesc);
+
        return ptdesc_address(ptdesc);
 }
 #define __pgd_alloc(...)       alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__))

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 5c887c4ea29ea3bd4e2605259d41ef11d50fdd83..8f46048875a729163405cc3a1654d5da1e2617ee 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3057,6 +3057,9 @@ static inline void pagetable_free(struct ptdesc *pt)
 {
        struct page *page = ptdesc_page(pt);
 
+       if (ptdesc_test_kernel(pt))
+               ptdesc_clear_kernel(pt);
+
        __free_pages(page, compound_order(page));
 }