mm/page_isolation: make page isolation a standalone bit

During page isolation, the original migratetype is overwritten, since
MIGRATE_* are enums and stored in pageblock bitmaps.  Change
MIGRATE_ISOLATE to be stored a standalone bit, PB_migrate_isolate, like
PB_compact_skip, so that migratetype is not lost during pageblock
isolation.

Link: https://lkml.kernel.org/r/20250617021115.2331563-3-ziy@nvidia.com
Signed-off-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: David Hildenbrand <david@redhat.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kirill A. Shuemov <kirill.shutemov@linux.intel.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Richard Chang <richardycc@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 76d66c07b673a0f510f6cca3142477fc0701e665..1d1bb2b7f40d25b430932c9ef9096d97bf1c29de 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -79,6 +79,9 @@ enum migratetype {
         * __free_pageblock_cma() function.
         */
        MIGRATE_CMA,
+       __MIGRATE_TYPE_END = MIGRATE_CMA,
+#else
+       __MIGRATE_TYPE_END = MIGRATE_HIGHATOMIC,
 #endif
 #ifdef CONFIG_MEMORY_ISOLATION
        MIGRATE_ISOLATE,        /* can't allocate from here */

diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h
index 277d8d92980c2d338f9665fee79afd244b15df3e..fc021d3f95ca77cb7cf490e2a4709e91c56f44be 100644 (file)
--- a/include/linux/page-isolation.h
+++ b/include/linux/page-isolation.h
@@ -11,6 +11,12 @@ static inline bool is_migrate_isolate(int migratetype)
 {
        return migratetype == MIGRATE_ISOLATE;
 }
+#define get_pageblock_isolate(page) \
+       get_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)
+#define clear_pageblock_isolate(page) \
+       clear_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)
+#define set_pageblock_isolate(page) \
+       set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)
 #else
 static inline bool is_migrate_isolate_page(struct page *page)
 {
@@ -20,6 +26,16 @@ static inline bool is_migrate_isolate(int migratetype)
 {
        return false;
 }
+static inline bool get_pageblock_isolate(struct page *page)
+{
+       return false;
+}
+static inline void clear_pageblock_isolate(struct page *page)
+{
+}
+static inline void set_pageblock_isolate(struct page *page)
+{
+}
 #endif
 
 #define MEMORY_OFFLINE 0x1

diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h
index c240c7a1fb038eae91dd0e97d94981c25793498c..6a44be0f39f45f0857019435cfef4b1ba0eae8d4 100644 (file)
--- a/include/linux/pageblock-flags.h
+++ b/include/linux/pageblock-flags.h
@@ -21,6 +21,13 @@ enum pageblock_bits {
                        /* 3 bits required for migrate types */
        PB_compact_skip,/* If set the block is skipped by compaction */
 
+#ifdef CONFIG_MEMORY_ISOLATION
+       /*
+        * Pageblock isolation is represented with a separate bit, so that
+        * the migratetype of a block is not overwritten by isolation.
+        */
+       PB_migrate_isolate, /* If set the block is isolated */
+#endif
        /*
         * Assume the bits will always align on a word. If this assumption
         * changes then get/set pageblock needs updating.
@@ -32,6 +39,13 @@ enum pageblock_bits {
 
 #define MIGRATETYPE_MASK ((1UL << (PB_migrate_end + 1)) - 1)
 
+#ifdef CONFIG_MEMORY_ISOLATION
+#define MIGRATETYPE_AND_ISO_MASK \
+       (((1UL << (PB_migrate_end + 1)) - 1) | BIT(PB_migrate_isolate))
+#else
+#define MIGRATETYPE_AND_ISO_MASK MIGRATETYPE_MASK
+#endif
+
 #if defined(CONFIG_HUGETLB_PAGE)
 
 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 43a4c15327217d4201efbafccda86ff4cdc337a6..61dd34102c14a03aca642c8743583efa705c410c 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -365,8 +365,12 @@ get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn,
        unsigned long *bitmap;
        unsigned long word_bitidx;
 
+#ifdef CONFIG_MEMORY_ISOLATION
+       BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8);
+#else
        BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
-       BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
+#endif
+       BUILD_BUG_ON(__MIGRATE_TYPE_END >= (1 << PB_migratetype_bits));
        VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
 
        bitmap = get_pageblock_bitmap(page, pfn);
@@ -439,7 +443,16 @@ bool get_pfnblock_bit(const struct page *page, unsigned long pfn,
 __always_inline enum migratetype
 get_pfnblock_migratetype(const struct page *page, unsigned long pfn)
 {
-       return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
+       unsigned long mask = MIGRATETYPE_AND_ISO_MASK;
+       unsigned long flags;
+
+       flags = __get_pfnblock_flags_mask(page, pfn, mask);
+
+#ifdef CONFIG_MEMORY_ISOLATION
+       if (flags & BIT(PB_migrate_isolate))
+               return MIGRATE_ISOLATE;
+#endif
+       return flags & MIGRATETYPE_MASK;
 }
 
 /**
@@ -519,8 +532,16 @@ __always_inline void set_pageblock_migratetype(struct page *page,
                     migratetype < MIGRATE_PCPTYPES))
                migratetype = MIGRATE_UNMOVABLE;
 
+#ifdef CONFIG_MEMORY_ISOLATION
+       if (migratetype == MIGRATE_ISOLATE) {
+               set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
+               return;
+       }
+       /* MIGRATETYPE_AND_ISO_MASK clears PB_migrate_isolate if it is set */
+#endif
        __set_pfnblock_flags_mask(page, page_to_pfn(page),
-                                 (unsigned long)migratetype, MIGRATETYPE_MASK);
+                                 (unsigned long)migratetype,
+                                 MIGRATETYPE_AND_ISO_MASK);
 }
 
 #ifdef CONFIG_DEBUG_VM