mm/page_isolation: remove migratetype from move_freepages_block_isolate()

Since migratetype is no longer overwritten during pageblock isolation,
moving a pageblock out of MIGRATE_ISOLATE no longer needs a new
migratetype.

Add pageblock_isolate_and_move_free_pages() and
pageblock_unisolate_and_move_free_pages() to be explicit about the page
isolation operations.  Both share the common code in
__move_freepages_block_isolate(), which is renamed from
move_freepages_block_isolate().

Add toggle_pageblock_isolate() to flip pageblock isolation bit in
__move_freepages_block_isolate().

Make set_pageblock_migratetype() only accept non MIGRATE_ISOLATE types, so
that one should use set_pageblock_isolate() to isolate pageblocks.  As a
result, move pageblock migratetype code out of __move_freepages_block().

Link: https://lkml.kernel.org/r/20250617021115.2331563-5-ziy@nvidia.com
Signed-off-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: David Hildenbrand <david@redhat.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kirill A. Shuemov <kirill.shutemov@linux.intel.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Richard Chang <richardycc@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h
index 14c6a5f691c24cb0010ce9ddb6ae19e054c29a72..7241a67196182c161b52bf59e6b1ef2c2477218a 100644 (file)
--- a/include/linux/page-isolation.h
+++ b/include/linux/page-isolation.h
@@ -44,10 +44,9 @@ static inline void set_pageblock_isolate(struct page *page)
 void __meminit init_pageblock_migratetype(struct page *page,
                                          enum migratetype migratetype,
                                          bool isolate);
-void set_pageblock_migratetype(struct page *page, enum migratetype migratetype);
 
-bool move_freepages_block_isolate(struct zone *zone, struct page *page,
-                                 int migratetype);
+bool pageblock_isolate_and_move_free_pages(struct zone *zone, struct page *page);
+bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *page);
 
 int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
                             int migratetype, int flags);

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index c7730264bf5f638181dcdd1acb62b1324196c19d..938b01bed1f681b9d64f33bf1332b42a911c8250 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -525,8 +525,8 @@ void clear_pfnblock_bit(const struct page *page, unsigned long pfn,
  * @page: The page within the block of interest
  * @migratetype: migratetype to set
  */
-__always_inline void set_pageblock_migratetype(struct page *page,
-                                              enum migratetype migratetype)
+static void set_pageblock_migratetype(struct page *page,
+                                     enum migratetype migratetype)
 {
        if (unlikely(page_group_by_mobility_disabled &&
                     migratetype < MIGRATE_PCPTYPES))
@@ -534,9 +534,13 @@ __always_inline void set_pageblock_migratetype(struct page *page,
 
 #ifdef CONFIG_MEMORY_ISOLATION
        if (migratetype == MIGRATE_ISOLATE) {
-               set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
+               VM_WARN_ONCE(1,
+                       "Use set_pageblock_isolate() for pageblock isolation");
                return;
        }
+       VM_WARN_ONCE(get_pfnblock_bit(page, page_to_pfn(page),
+                                     PB_migrate_isolate),
+                    "Use clear_pageblock_isolate() to unisolate pageblock");
        /* MIGRATETYPE_AND_ISO_MASK clears PB_migrate_isolate if it is set */
 #endif
        __set_pfnblock_flags_mask(page, page_to_pfn(page),
@@ -1921,8 +1925,8 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
 #endif
 
 /*
- * Change the type of a block and move all its free pages to that
- * type's freelist.
+ * Move all free pages of a block to new type's freelist. Caller needs to
+ * change the block type.
  */
 static int __move_freepages_block(struct zone *zone, unsigned long start_pfn,
                                  int old_mt, int new_mt)
@@ -1954,8 +1958,6 @@ static int __move_freepages_block(struct zone *zone, unsigned long start_pfn,
                pages_moved += 1 << order;
        }
 
-       set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt);
-
        return pages_moved;
 }
 
@@ -2013,11 +2015,16 @@ static int move_freepages_block(struct zone *zone, struct page *page,
                                int old_mt, int new_mt)
 {
        unsigned long start_pfn;
+       int res;
 
        if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))
                return -1;
 
-       return __move_freepages_block(zone, start_pfn, old_mt, new_mt);
+       res = __move_freepages_block(zone, start_pfn, old_mt, new_mt);
+       set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt);
+
+       return res;
+
 }
 
 #ifdef CONFIG_MEMORY_ISOLATION
@@ -2045,11 +2052,19 @@ static unsigned long find_large_buddy(unsigned long start_pfn)
        return start_pfn;
 }
 
+static inline void toggle_pageblock_isolate(struct page *page, bool isolate)
+{
+       if (isolate)
+               set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
+       else
+               clear_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
+}
+
 /**
- * move_freepages_block_isolate - move free pages in block for page isolation
+ * __move_freepages_block_isolate - move free pages in block for page isolation
  * @zone: the zone
  * @page: the pageblock page
- * @migratetype: migratetype to set on the pageblock
+ * @isolate: to isolate the given pageblock or unisolate it
  *
  * This is similar to move_freepages_block(), but handles the special
  * case encountered in page isolation, where the block of interest
@@ -2064,10 +2079,18 @@ static unsigned long find_large_buddy(unsigned long start_pfn)
  *
  * Returns %true if pages could be moved, %false otherwise.
  */
-bool move_freepages_block_isolate(struct zone *zone, struct page *page,
-                                 int migratetype)
+static bool __move_freepages_block_isolate(struct zone *zone,
+               struct page *page, bool isolate)
 {
        unsigned long start_pfn, pfn;
+       int from_mt;
+       int to_mt;
+
+       if (isolate == get_pageblock_isolate(page)) {
+               VM_WARN_ONCE(1, "%s a pageblock that is already in that state",
+                            isolate ? "Isolate" : "Unisolate");
+               return false;
+       }
 
        if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))
                return false;
@@ -2084,7 +2107,7 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page,
 
                del_page_from_free_list(buddy, zone, order,
                                        get_pfnblock_migratetype(buddy, pfn));
-               set_pageblock_migratetype(page, migratetype);
+               toggle_pageblock_isolate(page, isolate);
                split_large_buddy(zone, buddy, pfn, order, FPI_NONE);
                return true;
        }
@@ -2095,16 +2118,38 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page,
 
                del_page_from_free_list(page, zone, order,
                                        get_pfnblock_migratetype(page, pfn));
-               set_pageblock_migratetype(page, migratetype);
+               toggle_pageblock_isolate(page, isolate);
                split_large_buddy(zone, page, pfn, order, FPI_NONE);
                return true;
        }
 move:
-       __move_freepages_block(zone, start_pfn,
-                              get_pfnblock_migratetype(page, start_pfn),
-                              migratetype);
+       /* Use MIGRATETYPE_MASK to get non-isolate migratetype */
+       if (isolate) {
+               from_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),
+                                                   MIGRATETYPE_MASK);
+               to_mt = MIGRATE_ISOLATE;
+       } else {
+               from_mt = MIGRATE_ISOLATE;
+               to_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),
+                                                 MIGRATETYPE_MASK);
+       }
+
+       __move_freepages_block(zone, start_pfn, from_mt, to_mt);
+       toggle_pageblock_isolate(pfn_to_page(start_pfn), isolate);
+
        return true;
 }
+
+bool pageblock_isolate_and_move_free_pages(struct zone *zone, struct page *page)
+{
+       return __move_freepages_block_isolate(zone, page, true);
+}
+
+bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *page)
+{
+       return __move_freepages_block_isolate(zone, page, false);
+}
+
 #endif /* CONFIG_MEMORY_ISOLATION */
 
 static void change_pageblock_range(struct page *pageblock_page,
@@ -2296,6 +2341,7 @@ try_to_claim_block(struct zone *zone, struct page *page,
        if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
                        page_group_by_mobility_disabled) {
                __move_freepages_block(zone, start_pfn, block_type, start_type);
+               set_pageblock_migratetype(pfn_to_page(start_pfn), start_type);
                return __rmqueue_smallest(zone, order, start_type);
        }
 

diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index b2fc5266e3d264def0896f7057d6564624b28592..08f627a5032f8d6c91b66d4a49561857d27b58bb 100644 (file)
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -188,7 +188,7 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_
        unmovable = has_unmovable_pages(check_unmovable_start, check_unmovable_end,
                        migratetype, isol_flags);
        if (!unmovable) {
-               if (!move_freepages_block_isolate(zone, page, MIGRATE_ISOLATE)) {
+               if (!pageblock_isolate_and_move_free_pages(zone, page)) {
                        spin_unlock_irqrestore(&zone->lock, flags);
                        return -EBUSY;
                }
@@ -209,7 +209,7 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_
        return -EBUSY;
 }
 
-static void unset_migratetype_isolate(struct page *page, int migratetype)
+static void unset_migratetype_isolate(struct page *page)
 {
        struct zone *zone;
        unsigned long flags;
@@ -262,10 +262,10 @@ static void unset_migratetype_isolate(struct page *page, int migratetype)
                 * Isolating this block already succeeded, so this
                 * should not fail on zone boundaries.
                 */
-               WARN_ON_ONCE(!move_freepages_block_isolate(zone, page, migratetype));
+               WARN_ON_ONCE(!pageblock_unisolate_and_move_free_pages(zone, page));
        } else {
-               set_pageblock_migratetype(page, migratetype);
-               __putback_isolated_page(page, order, migratetype);
+               clear_pageblock_isolate(page);
+               __putback_isolated_page(page, order, get_pageblock_migratetype(page));
        }
        zone->nr_isolate_pageblock--;
 out:
@@ -383,7 +383,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
                if (PageBuddy(page)) {
                        int order = buddy_order(page);
 
-                       /* move_freepages_block_isolate() handled this */
+                       /* pageblock_isolate_and_move_free_pages() handled this */
                        VM_WARN_ON_ONCE(pfn + (1 << order) > boundary_pfn);
 
                        pfn += 1UL << order;
@@ -433,7 +433,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
 failed:
        /* restore the original migratetype */
        if (!skip_isolation)
-               unset_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype);
+               unset_migratetype_isolate(pfn_to_page(isolate_pageblock));
        return -EBUSY;
 }
 
@@ -504,7 +504,7 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
        ret = isolate_single_pageblock(isolate_end, flags, true,
                        skip_isolation, migratetype);
        if (ret) {
-               unset_migratetype_isolate(pfn_to_page(isolate_start), migratetype);
+               unset_migratetype_isolate(pfn_to_page(isolate_start));
                return ret;
        }
 
@@ -517,8 +517,7 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
                                        start_pfn, end_pfn)) {
                        undo_isolate_page_range(isolate_start, pfn, migratetype);
                        unset_migratetype_isolate(
-                               pfn_to_page(isolate_end - pageblock_nr_pages),
-                               migratetype);
+                               pfn_to_page(isolate_end - pageblock_nr_pages));
                        return -EBUSY;
                }
        }
@@ -548,7 +547,7 @@ void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
                page = __first_valid_page(pfn, pageblock_nr_pages);
                if (!page || !is_migrate_isolate_page(page))
                        continue;
-               unset_migratetype_isolate(page, migratetype);
+               unset_migratetype_isolate(page);
        }
 }
 /*