mm: hugetlb: optimize replace_free_hugepage_folios()

If no free hugepage folios are available, there is no need to perform any
replacement operations.  Additionally, gigantic folios should not be
replaced under any circumstances.  Therefore, we only check for the
presence of non-gigantic folios, also adding the gigantic folio check to
avoid accidental replacement.

To optimize performance, we skip unnecessary iterations over pfn for
compound pages and high-order buddy pages to save processing time.

A simple test on machine with 114G free memory, allocate 120 * 1G HugeTLB
folios(104 successfully returned),

  time echo 120 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages

Before: 0m0.602s
After:  0m0.431s

[wangkefeng.wang@huawei.com: v2]
Link: https://lkml.kernel.org/r/20260114135512.2159799-1-wangkefeng.wang@huawei.com
[akpm@linux-foundation.org: use single-return-point style, tweak comment]
Link: https://lkml.kernel.org/r/20260112150954.1802953-4-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Jane Chu <jane.chu@oracle.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 9c7efad6fa485b020c03955ba91e7512b26b9282..120ebd448b4231c490512e18afb0993ceffb9b02 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -2810,23 +2810,62 @@ int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list)
  */
 int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn)
 {
-       struct folio *folio;
+       unsigned long nr = 0;
+       struct page *page;
+       struct hstate *h;
+       LIST_HEAD(list);
        int ret = 0;
 
-       LIST_HEAD(isolate_list);
+       /* Avoid pfn iterations if no free non-gigantic huge pages */
+       for_each_hstate(h) {
+               if (hstate_is_gigantic(h))
+                       continue;
+
+               nr += h->free_huge_pages;
+               if (nr)
+                       break;
+       }
+
+       if (!nr)
+               return 0;
 
        while (start_pfn < end_pfn) {
-               folio = pfn_folio(start_pfn);
+               page = pfn_to_page(start_pfn);
+               nr = 1;
 
-               /* Not to disrupt normal path by vainly holding hugetlb_lock */
-               if (folio_test_hugetlb(folio) && !folio_ref_count(folio)) {
-                       ret = alloc_and_dissolve_hugetlb_folio(folio, &isolate_list);
-                       if (ret)
-                               break;
+               if (PageHuge(page) || PageCompound(page)) {
+                       struct folio *folio = page_folio(page);
+
+                       nr = folio_nr_pages(folio) - folio_page_idx(folio, page);
+
+                       /*
+                        * Don't disrupt normal path by vainly holding
+                        * hugetlb_lock
+                        */
+                       if (folio_test_hugetlb(folio) && !folio_ref_count(folio)) {
+                               if (order_is_gigantic(folio_order(folio))) {
+                                       ret = -ENOMEM;
+                                       break;
+                               }
+
+                               ret = alloc_and_dissolve_hugetlb_folio(folio, &list);
+                               if (ret)
+                                       break;
+
+                               putback_movable_pages(&list);
+                       }
+               } else if (PageBuddy(page)) {
+                       /*
+                        * Buddy order check without zone lock is unsafe and
+                        * the order is maybe invalid, but race should be
+                        * small, and the worst thing is skipping free hugetlb.
+                        */
+                       const unsigned int order = buddy_order_unsafe(page);
 
-                       putback_movable_pages(&isolate_list);
+                       if (order <= MAX_PAGE_ORDER)
+                               nr = 1UL << order;
                }
-               start_pfn++;
+               start_pfn += nr;
        }
 
        return ret;