mm/sparse-vmemmap: pass @pgmap argument to memory deactivation paths

Currently, the memory hot-remove call chain -- arch_remove_memory(),
__remove_pages(), sparse_remove_section() and section_deactivate() -- does
not carry the struct dev_pagemap pointer.  This prevents the lower levels
from knowing whether the section was originally populated with vmemmap
optimizations (e.g., DAX with vmemmap optimization enabled).

Without this information, we cannot call vmemmap_can_optimize() to
determine if the vmemmap pages were optimized.  As a result, the vmemmap
page accounting during teardown will mistakenly assume a non-optimized
allocation, leading to incorrect memmap statistics.

To lay the groundwork for fixing the vmemmap page accounting, we need to
pass the @pgmap pointer down to the deactivation location.  Plumb the
@pgmap argument through the APIs of arch_remove_memory(), __remove_pages()
and sparse_remove_section(), mirroring the corresponding *_activate()
paths.

Link: https://lore.kernel.org/20260428081855.1249045-4-songmuchun@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Reviewed-by: Oscar Salvador <osalvador@suse.de>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Acked-by: Liam R. Howlett <liam@infradead.org>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
Cc: Joao Martins <joao.m.martins@oracle.com>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index dd85e093ffdb1773f5f1f665eaff2b3ae74564e9..e5a42b7a01609e2444af66acb0947adeeb7ecb70 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -2024,12 +2024,13 @@ err:
        return ret;
 }
 
-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+                       struct dev_pagemap *pgmap)
 {
        unsigned long start_pfn = start >> PAGE_SHIFT;
        unsigned long nr_pages = size >> PAGE_SHIFT;
 
-       __remove_pages(start_pfn, nr_pages, altmap);
+       __remove_pages(start_pfn, nr_pages, altmap, pgmap);
        __remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size);
 }
 

diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index 031b39eb081c57c57499fc0657f34f3c227cafed..687980b6e91f05159552636f3f0513ad81c66490 100644 (file)
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -119,12 +119,13 @@ int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params)
        return ret;
 }
 
-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+                       struct dev_pagemap *pgmap)
 {
        unsigned long start_pfn = start >> PAGE_SHIFT;
        unsigned long nr_pages = size >> PAGE_SHIFT;
 
-       __remove_pages(start_pfn, nr_pages, altmap);
+       __remove_pages(start_pfn, nr_pages, altmap, pgmap);
 }
 #endif
 

diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 648d0c5602ec8b3f8137e9172518aee3d7fa7d6c..4c1afab9199637b8c37744cbe11782d9a09ff892 100644 (file)
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -158,12 +158,13 @@ int __ref arch_add_memory(int nid, u64 start, u64 size,
        return rc;
 }
 
-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+                             struct dev_pagemap *pgmap)
 {
        unsigned long start_pfn = start >> PAGE_SHIFT;
        unsigned long nr_pages = size >> PAGE_SHIFT;
 
-       __remove_pages(start_pfn, nr_pages, altmap);
+       __remove_pages(start_pfn, nr_pages, altmap, pgmap);
        arch_remove_linear_mapping(start, size);
 }
 #endif

diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index fa8d2f6f554b57a6e438c9caaa04637017633166..885f1db4e9bfd4ae9a94e3bff4252f5e72bc9ea5 100644 (file)
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -1742,9 +1742,10 @@ int __ref arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *param
        return ret;
 }
 
-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+                             struct dev_pagemap *pgmap)
 {
-       __remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap);
+       __remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap, pgmap);
        remove_linear_mapping(start, size);
        flush_tlb_all();
 }

diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 1f72efc2a579faefe272ff3116341ff7ff3cfd0d..11a689423440fa100d1938a5617d84dd56468f7e 100644 (file)
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -276,12 +276,13 @@ int arch_add_memory(int nid, u64 start, u64 size,
        return rc;
 }
 
-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+                       struct dev_pagemap *pgmap)
 {
        unsigned long start_pfn = start >> PAGE_SHIFT;
        unsigned long nr_pages = size >> PAGE_SHIFT;
 
-       __remove_pages(start_pfn, nr_pages, altmap);
+       __remove_pages(start_pfn, nr_pages, altmap, pgmap);
        vmem_remove_mapping(start, size);
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index df2261fa4f985b24422ca3d2bca5b82c621b51ce..77b889b71cf38ad81199866461c5c4c0a13ee95e 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1288,12 +1288,13 @@ kernel_physical_mapping_remove(unsigned long start, unsigned long end)
        remove_pagetable(start, end, true, NULL);
 }
 
-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+                             struct dev_pagemap *pgmap)
 {
        unsigned long start_pfn = start >> PAGE_SHIFT;
        unsigned long nr_pages = size >> PAGE_SHIFT;
 
-       __remove_pages(start_pfn, nr_pages, altmap);
+       __remove_pages(start_pfn, nr_pages, altmap, pgmap);
        kernel_physical_mapping_remove(start, start + size);
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */

diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index 815e908c4135b1f34aeecab5174d49e77d9e1d18..7c9d66729c6095f328b29b03c7e83cd7bfa697b0 100644 (file)
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -135,9 +135,10 @@ static inline bool movable_node_is_enabled(void)
        return movable_node_enabled;
 }
 
-extern void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap);
+extern void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+                              struct dev_pagemap *pgmap);
 extern void __remove_pages(unsigned long start_pfn, unsigned long nr_pages,
-                          struct vmem_altmap *altmap);
+                          struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
 
 /* reasonably generic interface to expand the physical pages */
 extern int __add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages,
@@ -307,7 +308,8 @@ extern int sparse_add_section(int nid, unsigned long pfn,
                unsigned long nr_pages, struct vmem_altmap *altmap,
                struct dev_pagemap *pgmap);
 extern void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
-                                 struct vmem_altmap *altmap);
+                                 struct vmem_altmap *altmap,
+                                 struct dev_pagemap *pgmap);
 extern struct zone *zone_for_pfn_range(enum mmop online_type,
                int nid, struct memory_group *group, unsigned long start_pfn,
                unsigned long nr_pages);

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index cf4f77108c43fc52fcb28e4ad6cb5477cf1372a2..462d8dcd636dc9dd1a4a3683a1c9c24aaa89c897 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -576,6 +576,7 @@ void remove_pfn_range_from_zone(struct zone *zone,
  * @pfn: starting pageframe (must be aligned to start of a section)
  * @nr_pages: number of pages to remove (must be multiple of section size)
  * @altmap: alternative device page map or %NULL if default memmap is used
+ * @pgmap: device page map or %NULL if not ZONE_DEVICE
  *
  * Generic helper function to remove section mappings and sysfs entries
  * for the section of the memory we are removing. Caller needs to make
@@ -583,7 +584,7 @@ void remove_pfn_range_from_zone(struct zone *zone,
  * calling offline_pages().
  */
 void __remove_pages(unsigned long pfn, unsigned long nr_pages,
-                   struct vmem_altmap *altmap)
+                   struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
 {
        const unsigned long end_pfn = pfn + nr_pages;
        unsigned long cur_nr_pages;
@@ -598,7 +599,7 @@ void __remove_pages(unsigned long pfn, unsigned long nr_pages,
                /* Select all remaining pages up to the next section boundary */
                cur_nr_pages = min(end_pfn - pfn,
                                   SECTION_ALIGN_UP(pfn + 1) - pfn);
-               sparse_remove_section(pfn, cur_nr_pages, altmap);
+               sparse_remove_section(pfn, cur_nr_pages, altmap, pgmap);
        }
 }
 
@@ -1427,7 +1428,7 @@ static void remove_memory_blocks_and_altmaps(u64 start, u64 size)
 
                remove_memory_block_devices(cur_start, memblock_size);
 
-               arch_remove_memory(cur_start, memblock_size, altmap);
+               arch_remove_memory(cur_start, memblock_size, altmap, NULL);
 
                /* Verify that all vmemmap pages have actually been freed. */
                WARN(altmap->alloc, "Altmap not fully unmapped");
@@ -1470,7 +1471,7 @@ static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group,
                ret = create_memory_block_devices(cur_start, memblock_size, nid,
                                                  params.altmap, group);
                if (ret) {
-                       arch_remove_memory(cur_start, memblock_size, params.altmap);
+                       arch_remove_memory(cur_start, memblock_size, params.altmap, NULL);
                        kfree(params.altmap);
                        goto out;
                }
@@ -1556,7 +1557,7 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
                /* create memory block devices after memory was added */
                ret = create_memory_block_devices(start, size, nid, NULL, group);
                if (ret) {
-                       arch_remove_memory(start, size, params.altmap);
+                       arch_remove_memory(start, size, params.altmap, NULL);
                        goto error;
                }
        }
@@ -2268,7 +2269,7 @@ static int try_remove_memory(u64 start, u64 size)
                 * No altmaps present, do the removal directly
                 */
                remove_memory_block_devices(start, size);
-               arch_remove_memory(start, size, NULL);
+               arch_remove_memory(start, size, NULL, NULL);
        } else {
                /* all memblocks in the range have altmaps */
                remove_memory_blocks_and_altmaps(start, size);

diff --git a/mm/memremap.c b/mm/memremap.c
index 053842d45cb1097bdfe05f049d00ec508ab152f3..81766d82240096adc6dc8875442ab3e3743f2819 100644 (file)
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -97,10 +97,10 @@ static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
                                   PHYS_PFN(range_len(range)));
        if (pgmap->type == MEMORY_DEVICE_PRIVATE) {
                __remove_pages(PHYS_PFN(range->start),
-                              PHYS_PFN(range_len(range)), NULL);
+                              PHYS_PFN(range_len(range)), NULL, pgmap);
        } else {
                arch_remove_memory(range->start, range_len(range),
-                               pgmap_altmap(pgmap));
+                               pgmap_altmap(pgmap), pgmap);
                kasan_remove_zero_shadow(__va(range->start), range_len(range));
        }
        mem_hotplug_done();

diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 60e55e78d7ffb5ed7dcc19f9e0d549fdcf3eafd9..eafb7c6eb71ed9a4d6245a4e648deb42dfaa3372 100644 (file)
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -660,7 +660,7 @@ static struct page * __meminit populate_section_memmap(unsigned long pfn,
 }
 
 static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
-               struct vmem_altmap *altmap)
+               struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
 {
        unsigned long start = (unsigned long) pfn_to_page(pfn);
        unsigned long end = start + nr_pages * sizeof(struct page);
@@ -741,7 +741,7 @@ static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages)
  * usage map, but still need to free the vmemmap range.
  */
 static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
-               struct vmem_altmap *altmap)
+               struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
 {
        struct mem_section *ms = __pfn_to_section(pfn);
        bool section_is_early = early_section(ms);
@@ -779,7 +779,7 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
         * section_activate() and pfn_valid() .
         */
        if (!section_is_early)
-               depopulate_section_memmap(pfn, nr_pages, altmap);
+               depopulate_section_memmap(pfn, nr_pages, altmap, pgmap);
        else if (memmap)
                free_map_bootmem(memmap);
 
@@ -823,7 +823,7 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn,
 
        memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
        if (!memmap) {
-               section_deactivate(pfn, nr_pages, altmap);
+               section_deactivate(pfn, nr_pages, altmap, pgmap);
                return ERR_PTR(-ENOMEM);
        }
 
@@ -884,13 +884,13 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn,
 }
 
 void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
-                          struct vmem_altmap *altmap)
+               struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
 {
        struct mem_section *ms = __pfn_to_section(pfn);
 
        if (WARN_ON_ONCE(!valid_section(ms)))
                return;
 
-       section_deactivate(pfn, nr_pages, altmap);
+       section_deactivate(pfn, nr_pages, altmap, pgmap);
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */