mm/page_alloc: add support for initializing pageblock as isolated

MIGRATE_ISOLATE is a standalone bit, so a pageblock cannot be initialized
to just MIGRATE_ISOLATE.  Add init_pageblock_migratetype() to enable
initialize a pageblock with a migratetype and isolated.

Link: https://lkml.kernel.org/r/20250617021115.2331563-4-ziy@nvidia.com
Signed-off-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: David Hildenbrand <david@redhat.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kirill A. Shuemov <kirill.shutemov@linux.intel.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Richard Chang <richardycc@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index eaac5ae8c05c8ee2f2868d5bc1b04d1f68235b3f..23f038a162319aa662e4eedd094e5fb986ae5452 100644 (file)
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -314,7 +314,8 @@ extern int add_memory_driver_managed(int nid, u64 start, u64 size,
                                     mhp_t mhp_flags);
 extern void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
                                   unsigned long nr_pages,
-                                  struct vmem_altmap *altmap, int migratetype);
+                                  struct vmem_altmap *altmap, int migratetype,
+                                  bool isolate_pageblock);
 extern void remove_pfn_range_from_zone(struct zone *zone,
                                       unsigned long start_pfn,
                                       unsigned long nr_pages);

diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h
index fc021d3f95ca77cb7cf490e2a4709e91c56f44be..14c6a5f691c24cb0010ce9ddb6ae19e054c29a72 100644 (file)
--- a/include/linux/page-isolation.h
+++ b/include/linux/page-isolation.h
@@ -41,6 +41,9 @@ static inline void set_pageblock_isolate(struct page *page)
 #define MEMORY_OFFLINE 0x1
 #define REPORT_FAILURE 0x2
 
+void __meminit init_pageblock_migratetype(struct page *page,
+                                         enum migratetype migratetype,
+                                         bool isolate);
 void set_pageblock_migratetype(struct page *page, enum migratetype migratetype);
 
 bool move_freepages_block_isolate(struct zone *zone, struct page *page,

diff --git a/kernel/kexec_handover.c b/kernel/kexec_handover.c
index 5a21dbe179505a47a69e789c720fe404065d108c..49634cc3fb43775f73e6ac6b759c8c7cd129272e 100644 (file)
--- a/kernel/kexec_handover.c
+++ b/kernel/kexec_handover.c
@@ -1100,8 +1100,8 @@ static void __init kho_release_scratch(void)
                ulong pfn;
 
                for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages)
-                       set_pageblock_migratetype(pfn_to_page(pfn),
-                                                 MIGRATE_CMA);
+                       init_pageblock_migratetype(pfn_to_page(pfn),
+                                                  MIGRATE_CMA, false);
        }
 }
 

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index c03896375749f9767b73fe22fff41fb163fc7c5a..11d5668ff6e7b76c4856b4c1178088420b052917 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3297,8 +3297,8 @@ static void __init hugetlb_bootmem_init_migratetype(struct folio *folio,
                if (folio_test_hugetlb_cma(folio))
                        init_cma_pageblock(folio_page(folio, i));
                else
-                       set_pageblock_migratetype(folio_page(folio, i),
-                                         MIGRATE_MOVABLE);
+                       init_pageblock_migratetype(folio_page(folio, i),
+                                         MIGRATE_MOVABLE, false);
        }
 }
 

diff --git a/mm/internal.h b/mm/internal.h
index fe83dfca3c72666a94ac4b0073fcb2c67efd2b62..22a95a2b7fa19e9857e089fff2dbfa96fe2d2d0f 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -820,7 +820,8 @@ extern void *memmap_alloc(phys_addr_t size, phys_addr_t align,
                          int nid, bool exact_nid);
 
 void memmap_init_range(unsigned long, int, unsigned long, unsigned long,
-               unsigned long, enum meminit_context, struct vmem_altmap *, int);
+               unsigned long, enum meminit_context, struct vmem_altmap *, int,
+               bool);
 
 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
 

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 403221982c2ea762932aae9eaa11d33e6f90ac4f..a3c2b0784070328078911a4d034ed95c1de9e4d7 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -747,7 +747,8 @@ static inline void section_taint_zone_device(unsigned long pfn)
  */
 void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
                                  unsigned long nr_pages,
-                                 struct vmem_altmap *altmap, int migratetype)
+                                 struct vmem_altmap *altmap, int migratetype,
+                                 bool isolate_pageblock)
 {
        struct pglist_data *pgdat = zone->zone_pgdat;
        int nid = pgdat->node_id;
@@ -779,7 +780,8 @@ void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
         * are reserved so nobody should be touching them so we should be safe
         */
        memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0,
-                        MEMINIT_HOTPLUG, altmap, migratetype);
+                        MEMINIT_HOTPLUG, altmap, migratetype,
+                        isolate_pageblock);
 
        set_zone_contiguous(zone);
 }
@@ -1104,7 +1106,8 @@ int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages,
        if (mhp_off_inaccessible)
                page_init_poison(pfn_to_page(pfn), sizeof(struct page) * nr_pages);
 
-       move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_UNMOVABLE);
+       move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_UNMOVABLE,
+                              false);
 
        for (i = 0; i < nr_pages; i++) {
                struct page *page = pfn_to_page(pfn + i);
@@ -1175,7 +1178,8 @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 
 
        /* associate pfn range with the zone */
-       move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE);
+       move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_MOVABLE,
+                              true);
 
        if (!node_state(nid, N_MEMORY)) {
                /* Adding memory to the node for the first time */

diff --git a/mm/memremap.c b/mm/memremap.c
index f75078c14839ef26e1b40a2b3f0c1c6407ec7d1d..b0ce0d8254bd8feee76290cba55434010c0f4c17 100644 (file)
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -228,7 +228,7 @@ static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params,
                zone = &NODE_DATA(nid)->node_zones[ZONE_DEVICE];
                move_pfn_range_to_zone(zone, PHYS_PFN(range->start),
                                PHYS_PFN(range_len(range)), params->altmap,
-                               MIGRATE_MOVABLE);
+                               MIGRATE_MOVABLE, false);
        }
 
        mem_hotplug_done();

diff --git a/mm/mm_init.c b/mm/mm_init.c
index 02f41e2bdf60fe9d5f2dd9a8ccf5e6cae2fedb39..5c21b3af216b212a2f4642f1a503ae4206e15e5e 100644 (file)
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -685,7 +685,8 @@ void __meminit __init_page_from_nid(unsigned long pfn, int nid)
        __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
 
        if (pageblock_aligned(pfn))
-               set_pageblock_migratetype(pfn_to_page(pfn), MIGRATE_MOVABLE);
+               init_pageblock_migratetype(pfn_to_page(pfn), MIGRATE_MOVABLE,
+                               false);
 }
 
 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
@@ -874,7 +875,8 @@ static void __init init_unavailable_range(unsigned long spfn,
 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone,
                unsigned long start_pfn, unsigned long zone_end_pfn,
                enum meminit_context context,
-               struct vmem_altmap *altmap, int migratetype)
+               struct vmem_altmap *altmap, int migratetype,
+               bool isolate_pageblock)
 {
        unsigned long pfn, end_pfn = start_pfn + size;
        struct page *page;
@@ -931,7 +933,8 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
                 * over the place during system boot.
                 */
                if (pageblock_aligned(pfn)) {
-                       set_pageblock_migratetype(page, migratetype);
+                       init_pageblock_migratetype(page, migratetype,
+                                       isolate_pageblock);
                        cond_resched();
                }
                pfn++;
@@ -954,7 +957,8 @@ static void __init memmap_init_zone_range(struct zone *zone,
                return;
 
        memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
-                         zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
+                         zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE,
+                         false);
 
        if (*hole_pfn < start_pfn)
                init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
@@ -1035,7 +1039,7 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
         * because this is done early in section_activate()
         */
        if (pageblock_aligned(pfn)) {
-               set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+               init_pageblock_migratetype(page, MIGRATE_MOVABLE, false);
                cond_resched();
        }
 
@@ -1996,7 +2000,8 @@ static void __init deferred_free_pages(unsigned long pfn,
        /* Free a large naturally-aligned chunk if possible */
        if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) {
                for (i = 0; i < nr_pages; i += pageblock_nr_pages)
-                       set_pageblock_migratetype(page + i, MIGRATE_MOVABLE);
+                       init_pageblock_migratetype(page + i, MIGRATE_MOVABLE,
+                                       false);
                __free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY);
                return;
        }
@@ -2006,7 +2011,8 @@ static void __init deferred_free_pages(unsigned long pfn,
 
        for (i = 0; i < nr_pages; i++, page++, pfn++) {
                if (pageblock_aligned(pfn))
-                       set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+                       init_pageblock_migratetype(page, MIGRATE_MOVABLE,
+                                       false);
                __free_pages_core(page, 0, MEMINIT_EARLY);
        }
 }
@@ -2305,7 +2311,7 @@ void __init init_cma_reserved_pageblock(struct page *page)
                set_page_count(p, 0);
        } while (++p, --i);
 
-       set_pageblock_migratetype(page, MIGRATE_CMA);
+       init_pageblock_migratetype(page, MIGRATE_CMA, false);
        set_page_refcounted(page);
        /* pages were reserved and not allocated */
        clear_page_tag_ref(page);
@@ -2319,7 +2325,7 @@ void __init init_cma_reserved_pageblock(struct page *page)
  */
 void __init init_cma_pageblock(struct page *page)
 {
-       set_pageblock_migratetype(page, MIGRATE_CMA);
+       init_pageblock_migratetype(page, MIGRATE_CMA, false);
        adjust_managed_page_count(page, pageblock_nr_pages);
        page_zone(page)->cma_pages += pageblock_nr_pages;
 }

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 61dd34102c14a03aca642c8743583efa705c410c..c7730264bf5f638181dcdd1acb62b1324196c19d 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -544,6 +544,32 @@ __always_inline void set_pageblock_migratetype(struct page *page,
                                  MIGRATETYPE_AND_ISO_MASK);
 }
 
+void __meminit init_pageblock_migratetype(struct page *page,
+                                         enum migratetype migratetype,
+                                         bool isolate)
+{
+       unsigned long flags;
+
+       if (unlikely(page_group_by_mobility_disabled &&
+                    migratetype < MIGRATE_PCPTYPES))
+               migratetype = MIGRATE_UNMOVABLE;
+
+       flags = migratetype;
+
+#ifdef CONFIG_MEMORY_ISOLATION
+       if (migratetype == MIGRATE_ISOLATE) {
+               VM_WARN_ONCE(
+                       1,
+                       "Set isolate=true to isolate pageblock with a migratetype");
+               return;
+       }
+       if (isolate)
+               flags |= BIT(PB_migrate_isolate);
+#endif
+       __set_pfnblock_flags_mask(page, page_to_pfn(page), flags,
+                                 MIGRATETYPE_AND_ISO_MASK);
+}
+
 #ifdef CONFIG_DEBUG_VM
 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 {