mm: extend rmap flags arguments for folio_add_new_anon_rmap

Patch series "mm: clarify folio_add_new_anon_rmap() and
__folio_add_anon_rmap()", v2.

This patchset is preparatory work for mTHP swapin.

folio_add_new_anon_rmap() assumes that new anon rmaps are always
exclusive.  However, this assumption doesn’t hold true for cases like
do_swap_page(), where a new anon might be added to the swapcache and is
not necessarily exclusive.

The patchset extends the rmap flags to allow folio_add_new_anon_rmap() to
handle both exclusive and non-exclusive new anon folios.  The
do_swap_page() function is updated to use this extended API with rmap
flags.  Consequently, all new anon folios now consistently use
folio_add_new_anon_rmap().  The special case for !folio_test_anon() in
__folio_add_anon_rmap() can be safely removed.

In conclusion, new anon folios always use folio_add_new_anon_rmap(),
regardless of exclusivity.  Old anon folios continue to use
__folio_add_anon_rmap() via folio_add_anon_rmap_pmd() and
folio_add_anon_rmap_ptes().

This patch (of 3):

In the case of a swap-in, a new anonymous folio is not necessarily
exclusive.  This patch updates the rmap flags to allow a new anonymous
folio to be treated as either exclusive or non-exclusive.  To maintain the
existing behavior, we always use EXCLUSIVE as the default setting.

[akpm@linux-foundation.org: cleanup and constifications per David and akpm]
[v-songbaohua@oppo.com: fix missing doc for flags of folio_add_new_anon_rmap()]
Link: https://lkml.kernel.org/r/20240619210641.62542-1-21cnbao@gmail.com
[v-songbaohua@oppo.com: enhance doc for extend rmap flags arguments for folio_add_new_anon_rmap]
Link: https://lkml.kernel.org/r/20240622030256.43775-1-21cnbao@gmail.com
Link: https://lkml.kernel.org/r/20240617231137.80726-1-21cnbao@gmail.com
Link: https://lkml.kernel.org/r/20240617231137.80726-2-21cnbao@gmail.com
Signed-off-by: Barry Song <v-songbaohua@oppo.com>
Suggested-by: David Hildenbrand <david@redhat.com>
Tested-by: Shuai Yuan <yuanshuai@oppo.com>
Acked-by: David Hildenbrand <david@redhat.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Chris Li <chrisl@kernel.org>
Cc: "Huang, Ying" <ying.huang@intel.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 980fa5d75d69c2ddaf9fae7e3e405afaf53a2966..0978c64f49d833f19dd2559ab3042c243708300c 100644 (file)
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -244,7 +244,7 @@ void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages,
 void folio_add_anon_rmap_pmd(struct folio *, struct page *,
                struct vm_area_struct *, unsigned long address, rmap_t flags);
 void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *,
-               unsigned long address);
+               unsigned long address, rmap_t flags);
 void folio_add_file_rmap_ptes(struct folio *, struct page *, int nr_pages,
                struct vm_area_struct *);
 #define folio_add_file_rmap_pte(folio, page, vma) \

diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 2c83ba776fc7b297b7efaf662a8adb0c0a73befd..c20368aa33ddd81e948e1abb932e618b039962e6 100644 (file)
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -181,7 +181,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 
        if (new_page) {
                folio_get(new_folio);
-               folio_add_new_anon_rmap(new_folio, vma, addr);
+               folio_add_new_anon_rmap(new_folio, vma, addr, RMAP_EXCLUSIVE);
                folio_add_lru_vma(new_folio, vma);
        } else
                /* no new page, just dec_mm_counter for old_page */

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 46ba81240d9694c655e0914b2c2726aeb4d90efc..14a05c64380655bd038fb20a217cc7bca6f62ab4 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -974,7 +974,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,
 
                entry = mk_huge_pmd(page, vma->vm_page_prot);
                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
-               folio_add_new_anon_rmap(folio, vma, haddr);
+               folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, vma);
                pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
                set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index f8d08b49420cc6b6411fbcc8d163f8f0197d674e..409f67a817f101edf33e0693d1d08bf62ba3dbd7 100644 (file)
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1210,7 +1210,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address,
 
        spin_lock(pmd_ptl);
        BUG_ON(!pmd_none(*pmd));
-       folio_add_new_anon_rmap(folio, vma, address);
+       folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);
        folio_add_lru_vma(folio, vma);
        pgtable_trans_huge_deposit(mm, pmd, pgtable);
        set_pmd_at(mm, address, pmd, _pmd);

diff --git a/mm/memory.c b/mm/memory.c
index d8a0b7d2e15bca1d85ce5b94b9e136bece037a4f..a4fc6e632d2c2a787443bf64aef30fe368d0a9d1 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -930,7 +930,7 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
        *prealloc = NULL;
        copy_user_highpage(&new_folio->page, page, addr, src_vma);
        __folio_mark_uptodate(new_folio);
-       folio_add_new_anon_rmap(new_folio, dst_vma, addr);
+       folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE);
        folio_add_lru_vma(new_folio, dst_vma);
        rss[MM_ANONPAGES]++;
 
@@ -3402,7 +3402,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
                 * some TLBs while the old PTE remains in others.
                 */
                ptep_clear_flush(vma, vmf->address, vmf->pte);
-               folio_add_new_anon_rmap(new_folio, vma, vmf->address);
+               folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE);
                folio_add_lru_vma(new_folio, vma);
                BUG_ON(unshare && pte_write(entry));
                set_pte_at(mm, vmf->address, vmf->pte, entry);
@@ -4339,7 +4339,7 @@ check_folio:
 
        /* ksm created a completely new copy */
        if (unlikely(folio != swapcache && swapcache)) {
-               folio_add_new_anon_rmap(folio, vma, address);
+               folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, vma);
        } else {
                folio_add_anon_rmap_ptes(folio, page, nr_pages, vma, address,
@@ -4594,7 +4594,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
        count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_FAULT_ALLOC);
 #endif
-       folio_add_new_anon_rmap(folio, vma, addr);
+       folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
        folio_add_lru_vma(folio, vma);
 setpte:
        if (vmf_orig_pte_uffd_wp(vmf))
@@ -4792,7 +4792,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio,
        /* copy-on-write page */
        if (write && !(vma->vm_flags & VM_SHARED)) {
                VM_BUG_ON_FOLIO(nr != 1, folio);
-               folio_add_new_anon_rmap(folio, vma, addr);
+               folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, vma);
        } else {
                folio_add_file_rmap_ptes(folio, page, nr, vma);

diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 051d0a3ccbee02a561ec9aac7d99d152e1ae46ec..6d66dc1c6ffa062a082baefc5894b1cc4bc6466b 100644 (file)
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -658,7 +658,7 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
                goto unlock_abort;
 
        inc_mm_counter(mm, MM_ANONPAGES);
-       folio_add_new_anon_rmap(folio, vma, addr);
+       folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
        if (!folio_is_zone_device(folio))
                folio_add_lru_vma(folio, vma);
        folio_get(folio);

diff --git a/mm/rmap.c b/mm/rmap.c
index 69cbd7ac2a5c87921c9c830d56b0307c941d5684..c0c99f91ade1b1f759995efa6d78674b4d116c71 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1401,30 +1401,35 @@ void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
  * @folio:     The folio to add the mapping to.
  * @vma:       the vm area in which the mapping is added
  * @address:   the user virtual address mapped
+ * @flags:     The rmap flags
  *
  * Like folio_add_anon_rmap_*() but must only be called on *new* folios.
  * This means the inc-and-test can be bypassed.
- * The folio does not have to be locked.
+ * The folio doesn't necessarily need to be locked while it's exclusive
+ * unless two threads map it concurrently. However, the folio must be
+ * locked if it's shared.
  *
- * If the folio is pmd-mappable, it is accounted as a THP.  As the folio
- * is new, it's assumed to be mapped exclusively by a single process.
+ * If the folio is pmd-mappable, it is accounted as a THP.
  */
 void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
-               unsigned long address)
+               unsigned long address, rmap_t flags)
 {
-       int nr = folio_nr_pages(folio);
+       const int nr = folio_nr_pages(folio);
+       const bool exclusive = flags & RMAP_EXCLUSIVE;
        int nr_pmdmapped = 0;
 
        VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
+       VM_WARN_ON_FOLIO(!exclusive && !folio_test_locked(folio), folio);
        VM_BUG_ON_VMA(address < vma->vm_start ||
                        address + (nr << PAGE_SHIFT) > vma->vm_end, vma);
        __folio_set_swapbacked(folio);
-       __folio_set_anon(folio, vma, address, true);
+       __folio_set_anon(folio, vma, address, exclusive);
 
        if (likely(!folio_test_large(folio))) {
                /* increment count (starts at -1) */
                atomic_set(&folio->_mapcount, 0);
-               SetPageAnonExclusive(&folio->page);
+               if (exclusive)
+                       SetPageAnonExclusive(&folio->page);
        } else if (!folio_test_pmd_mappable(folio)) {
                int i;
 
@@ -1433,7 +1438,8 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
 
                        /* increment count (starts at -1) */
                        atomic_set(&page->_mapcount, 0);
-                       SetPageAnonExclusive(page);
+                       if (exclusive)
+                               SetPageAnonExclusive(page);
                }
 
                /* increment count (starts at -1) */
@@ -1445,7 +1451,8 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
                /* increment count (starts at -1) */
                atomic_set(&folio->_large_mapcount, 0);
                atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED);
-               SetPageAnonExclusive(&folio->page);
+               if (exclusive)
+                       SetPageAnonExclusive(&folio->page);
                nr_pmdmapped = nr;
        }
 

diff --git a/mm/swapfile.c b/mm/swapfile.c
index 9c6d8e557c0fd5ab5f3cf62cdf84ade21a8192f4..ae1d2700f6a3e2d1c2a98b7bba477daee51203de 100644 (file)
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1911,7 +1911,7 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
 
                folio_add_anon_rmap_pte(folio, page, vma, addr, rmap_flags);
        } else { /* ksm created a completely new copy */
-               folio_add_new_anon_rmap(folio, vma, addr);
+               folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, vma);
        }
        new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot));

diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 5e7f2801698ac5c634ad8464a98fa0b2108fab6c..8dedaec0048634bc7deafe3402263c29d6d1b13d 100644 (file)
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -216,7 +216,7 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd,
                        folio_add_lru(folio);
                folio_add_file_rmap_pte(folio, page, dst_vma);
        } else {
-               folio_add_new_anon_rmap(folio, dst_vma, dst_addr);
+               folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, dst_vma);
        }