mm/khugepaged: fix race with folio split/free using temporary reference

hpage_collapse_scan_file() calls is_refcount_suitable(), which in turn
calls folio_mapcount().  folio_mapcount() checks folio_test_large() before
proceeding to folio_large_mapcount(), but there is a race window where the
folio may get split/freed between these checks, triggering:

  VM_WARN_ON_FOLIO(!folio_test_large(folio), folio)

Take a temporary reference to the folio in hpage_collapse_scan_file().
This stabilizes the folio during refcount check and prevents incorrect
large folio detection due to concurrent split/free.  Use helper
folio_expected_ref_count() + 1 to compare with folio_ref_count() instead
of using is_refcount_suitable().

Link: https://lkml.kernel.org/r/20250526182818.37978-1-shivankg@amd.com
Fixes: 05c5323b2a34 ("mm: track mapcount of large folios in single value")
Signed-off-by: Shivank Garg <shivankg@amd.com>
Reported-by: syzbot+2b99589e33edbe9475ca@syzkaller.appspotmail.com
Closes: https://lore.kernel.org/all/6828470d.a70a0220.38f255.000c.GAE@google.com
Suggested-by: David Hildenbrand <david@redhat.com>
Acked-by: David Hildenbrand <david@redhat.com>
Acked-by: Dev Jain <dev.jain@arm.com>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Bharata B Rao <bharata@amd.com>
Cc: Fengwei Yin <fengwei.yin@intel.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mariano Pache <npache@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index cdf5a581368bf4f7f9241f4e301401a08d15543f..7731a162a1a7a4bbc2343bc8c90315a360eac4e0 100644 (file)
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -2293,6 +2293,17 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,
                        continue;
                }
 
+               if (!folio_try_get(folio)) {
+                       xas_reset(&xas);
+                       continue;
+               }
+
+               if (unlikely(folio != xas_reload(&xas))) {
+                       folio_put(folio);
+                       xas_reset(&xas);
+                       continue;
+               }
+
                if (folio_order(folio) == HPAGE_PMD_ORDER &&
                    folio->index == start) {
                        /* Maybe PMD-mapped */
@@ -2303,23 +2314,27 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,
                         * it's safe to skip LRU and refcount checks before
                         * returning.
                         */
+                       folio_put(folio);
                        break;
                }
 
                node = folio_nid(folio);
                if (hpage_collapse_scan_abort(node, cc)) {
                        result = SCAN_SCAN_ABORT;
+                       folio_put(folio);
                        break;
                }
                cc->node_load[node]++;
 
                if (!folio_test_lru(folio)) {
                        result = SCAN_PAGE_LRU;
+                       folio_put(folio);
                        break;
                }
 
-               if (!is_refcount_suitable(folio)) {
+               if (folio_expected_ref_count(folio) + 1 != folio_ref_count(folio)) {
                        result = SCAN_PAGE_COUNT;
+                       folio_put(folio);
                        break;
                }
 
@@ -2331,6 +2346,7 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,
                 */
 
                present += folio_nr_pages(folio);
+               folio_put(folio);
 
                if (need_resched()) {
                        xas_pause(&xas);