mm/rmap: use folio_large_nr_pages() in add/remove functions

Let's just use the "large" variant in code where we are sure that we have
a large folio in our hands: this way we are sure that we don't perform any
unnecessary "large" checks.

While at it, convert the VM_BUG_ON_VMA to a VM_WARN_ON_ONCE.

Maybe in the future there will not be a difference in that regard between
large and small folios; in that case, unifying the handling again will be
easy.  E.g., folio_large_nr_pages() will simply translate to
folio_nr_pages() until we replace all instances.

Link: https://lkml.kernel.org/r/20250303163014.1128035-12-david@redhat.com
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andy Lutomirks^H^Hski <luto@kernel.org>
Cc: Borislav Betkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Lance Yang <ioworker0@gmail.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcow (Oracle) <willy@infradead.org>
Cc: Michal Koutn <mkoutny@suse.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: tejun heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Zefan Li <lizefan.x@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/rmap.c b/mm/rmap.c
index 08846b7eced6059c50148850ffbd72a7fd5d1173..c9922928616ee71cc81b7a9dfb5f7c6350b42f71 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1274,7 +1274,7 @@ static __always_inline unsigned int __folio_add_rmap(struct folio *folio,
                if (first) {
                        nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped);
                        if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) {
-                               nr_pages = folio_nr_pages(folio);
+                               nr_pages = folio_large_nr_pages(folio);
                                /*
                                 * We only track PMD mappings of PMD-sized
                                 * folios separately.
@@ -1522,14 +1522,11 @@ void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
 void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
                unsigned long address, rmap_t flags)
 {
-       const int nr = folio_nr_pages(folio);
        const bool exclusive = flags & RMAP_EXCLUSIVE;
-       int nr_pmdmapped = 0;
+       int nr = 1, nr_pmdmapped = 0;
 
        VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
        VM_WARN_ON_FOLIO(!exclusive && !folio_test_locked(folio), folio);
-       VM_BUG_ON_VMA(address < vma->vm_start ||
-                       address + (nr << PAGE_SHIFT) > vma->vm_end, vma);
 
        /*
         * VM_DROPPABLE mappings don't swap; instead they're just dropped when
@@ -1547,6 +1544,7 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
        } else if (!folio_test_pmd_mappable(folio)) {
                int i;
 
+               nr = folio_large_nr_pages(folio);
                for (i = 0; i < nr; i++) {
                        struct page *page = folio_page(folio, i);
 
@@ -1559,6 +1557,7 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
                folio_set_large_mapcount(folio, nr, vma);
                atomic_set(&folio->_nr_pages_mapped, nr);
        } else {
+               nr = folio_large_nr_pages(folio);
                /* increment count (starts at -1) */
                atomic_set(&folio->_entire_mapcount, 0);
                folio_set_large_mapcount(folio, 1, vma);
@@ -1568,6 +1567,9 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
                nr_pmdmapped = nr;
        }
 
+       VM_WARN_ON_ONCE(address < vma->vm_start ||
+                       address + (nr << PAGE_SHIFT) > vma->vm_end);
+
        __folio_mod_stat(folio, nr, nr_pmdmapped);
        mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
 }
@@ -1681,7 +1683,7 @@ static __always_inline void __folio_remove_rmap(struct folio *folio,
                if (last) {
                        nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped);
                        if (likely(nr < ENTIRELY_MAPPED)) {
-                               nr_pages = folio_nr_pages(folio);
+                               nr_pages = folio_large_nr_pages(folio);
                                if (level == RMAP_LEVEL_PMD)
                                        nr_pmdmapped = nr_pages;
                                nr = nr_pages - (nr & FOLIO_PAGES_MAPPED);