x86/mm: make register_page_bootmem_memmap handle PTE mappings

register_page_bootmem_memmap expects that vmemmap pages handed to it are
PMD-mapped, and that the number of pages to call get_page_bootmem on is
PMD-aligned.

This is currently a correct assumption, but will no longer be true once
pre-HVO of hugetlb pages is implemented.

Make it handle PTE-mapped vmemmap pages and a nr_pages argument that is
not necessarily PAGES_PER_SECTION.

Link: https://lkml.kernel.org/r/20250228182928.2645936-9-fvdl@google.com
Signed-off-by: Frank van der Linden <fvdl@google.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Dan Carpenter <dan.carpenter@linaro.org>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: David Hildenbrand <david@redhat.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Joao Martins <joao.m.martins@oracle.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Roman Gushchin (Cruise) <roman.gushchin@linux.dev>
Cc: Usama Arif <usamaarif642@gmail.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 01ea7c6df3036bd185cdb3f54ddf244b79cbce8c..6e8e4ef5312ae5322709a95f8fb2033a277ad228 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1599,11 +1599,14 @@ void register_page_bootmem_memmap(unsigned long section_nr,
                }
                get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO);
 
-               if (!boot_cpu_has(X86_FEATURE_PSE)) {
+               pmd = pmd_offset(pud, addr);
+               if (pmd_none(*pmd)) {
+                       next = (addr + PAGE_SIZE) & PAGE_MASK;
+                       continue;
+               }
+
+               if (!boot_cpu_has(X86_FEATURE_PSE) || !pmd_leaf(*pmd)) {
                        next = (addr + PAGE_SIZE) & PAGE_MASK;
-                       pmd = pmd_offset(pud, addr);
-                       if (pmd_none(*pmd))
-                               continue;
                        get_page_bootmem(section_nr, pmd_page(*pmd),
                                         MIX_SECTION_INFO);
 
@@ -1614,12 +1617,7 @@ void register_page_bootmem_memmap(unsigned long section_nr,
                                         SECTION_INFO);
                } else {
                        next = pmd_addr_end(addr, end);
-
-                       pmd = pmd_offset(pud, addr);
-                       if (pmd_none(*pmd))
-                               continue;
-
-                       nr_pmd_pages = 1 << get_order(PMD_SIZE);
+                       nr_pmd_pages = (next - addr) >> PAGE_SHIFT;
                        page = pmd_page(*pmd);
                        while (nr_pmd_pages--)
                                get_page_bootmem(section_nr, page++,