mm: convert FPB_IGNORE_* into FPB_RESPECT_*

Patch series "mm: folio_pte_batch() improvements", v2.

Ever since we added folio_pte_batch() for fork() + munmap() purposes, a
lot more users appeared (and more are being proposed), and more
functionality was added.

Most of the users only need basic functionality, and could benefit from a
non-inlined version.

So let's clean up folio_pte_batch() and split it into a basic
folio_pte_batch() (no flags) and a more advanced folio_pte_batch_ext().
Using either variant will now look much cleaner.

This series will likely conflict with some changes in some (old+new)
folio_pte_batch() users, but conflicts should be trivial to resolve.

This patch (of 4):

Respecting these PTE bits is the exception, so let's invert the meaning.

With this change, most callers don't have to pass any flags.  This is a
preparation for splitting folio_pte_batch() into a non-inlined variant
that doesn't consume any flags.

Long-term, we want folio_pte_batch() to probably ignore most common PTE
bits (e.g., write/dirty/young/soft-dirty) that are not relevant for most
page table walkers: uffd-wp and protnone might be bits to consider in the
future.  Only walkers that care about them can opt-in to respect them.

No functional change intended.

Link: https://lkml.kernel.org/r/20250702104926.212243-2-david@redhat.com
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Oscar Salvador <osalvador@suse.de>
Reviewed-by: Dev Jain <dev.jain@arm.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Jann Horn <jannh@google.com>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mathew Brost <matthew.brost@intel.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/internal.h b/mm/internal.h
index 2e235740128ad84561dc334295e77b812d04e72d..e530809ef7d278550b946b06df71c3afbe1d1369 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -202,17 +202,17 @@ static inline void vma_close(struct vm_area_struct *vma)
 /* Flags for folio_pte_batch(). */
 typedef int __bitwise fpb_t;
 
-/* Compare PTEs after pte_mkclean(), ignoring the dirty bit. */
-#define FPB_IGNORE_DIRTY               ((__force fpb_t)BIT(0))
+/* Compare PTEs respecting the dirty bit. */
+#define FPB_RESPECT_DIRTY              ((__force fpb_t)BIT(0))
 
-/* Compare PTEs after pte_clear_soft_dirty(), ignoring the soft-dirty bit. */
-#define FPB_IGNORE_SOFT_DIRTY          ((__force fpb_t)BIT(1))
+/* Compare PTEs respecting the soft-dirty bit. */
+#define FPB_RESPECT_SOFT_DIRTY         ((__force fpb_t)BIT(1))
 
 static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)
 {
-       if (flags & FPB_IGNORE_DIRTY)
+       if (!(flags & FPB_RESPECT_DIRTY))
                pte = pte_mkclean(pte);
-       if (likely(flags & FPB_IGNORE_SOFT_DIRTY))
+       if (likely(!(flags & FPB_RESPECT_SOFT_DIRTY)))
                pte = pte_clear_soft_dirty(pte);
        return pte_wrprotect(pte_mkold(pte));
 }
@@ -236,8 +236,8 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)
  * pages of the same large folio.
  *
  * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN,
- * the accessed bit, writable bit, dirty bit (with FPB_IGNORE_DIRTY) and
- * soft-dirty bit (with FPB_IGNORE_SOFT_DIRTY).
+ * the accessed bit, writable bit, dirty bit (unless FPB_RESPECT_DIRTY is set)
+ * and soft-dirty bit (unless FPB_RESPECT_SOFT_DIRTY is set).
  *
  * start_ptep must map any page of the folio. max_nr must be at least one and
  * must be limited by the caller so scanning cannot exceed a single page table.

diff --git a/mm/madvise.c b/mm/madvise.c
index a34c2c89a53bb9ef8e659d8b27a11bcf60b07294..e7f1d4caad81eae47201decdaaf625978a32846c 100644 (file)
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -346,10 +346,9 @@ static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end,
                                          pte_t pte, bool *any_young,
                                          bool *any_dirty)
 {
-       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        int max_nr = (end - addr) / PAGE_SIZE;
 
-       return folio_pte_batch(folio, addr, ptep, pte, max_nr, fpb_flags, NULL,
+       return folio_pte_batch(folio, addr, ptep, pte, max_nr, 0, NULL,
                               any_young, any_dirty);
 }
 

diff --git a/mm/memory.c b/mm/memory.c
index 9944380e947d47702851ecaedcc23354caba8db7..a03f1964db3388be9d8ba292bf35a50573131cb7 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -990,10 +990,10 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
         * by keeping the batching logic separate.
         */
        if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) {
-               if (src_vma->vm_flags & VM_SHARED)
-                       flags |= FPB_IGNORE_DIRTY;
-               if (!vma_soft_dirty_enabled(src_vma))
-                       flags |= FPB_IGNORE_SOFT_DIRTY;
+               if (!(src_vma->vm_flags & VM_SHARED))
+                       flags |= FPB_RESPECT_DIRTY;
+               if (vma_soft_dirty_enabled(src_vma))
+                       flags |= FPB_RESPECT_SOFT_DIRTY;
 
                nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags,
                                     &any_writable, NULL, NULL);
@@ -1535,7 +1535,6 @@ static inline int zap_present_ptes(struct mmu_gather *tlb,
                struct zap_details *details, int *rss, bool *force_flush,
                bool *force_break, bool *any_skipped)
 {
-       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        struct mm_struct *mm = tlb->mm;
        struct folio *folio;
        struct page *page;
@@ -1565,7 +1564,7 @@ static inline int zap_present_ptes(struct mmu_gather *tlb,
         * by keeping the batching logic separate.
         */
        if (unlikely(folio_test_large(folio) && max_nr != 1)) {
-               nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags,
+               nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, 0,
                                     NULL, NULL, NULL);
 
                zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 1ff7b2174eb77866591633ec87c7577d1923cb5e..2a25eedc3b1c0a3d4e7f2cf332be03bb3c539784 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -675,7 +675,6 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
 static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
                        unsigned long end, struct mm_walk *walk)
 {
-       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        struct vm_area_struct *vma = walk->vma;
        struct folio *folio;
        struct queue_pages *qp = walk->private;
@@ -713,8 +712,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
                        continue;
                if (folio_test_large(folio) && max_nr != 1)
                        nr = folio_pte_batch(folio, addr, pte, ptent,
-                                            max_nr, fpb_flags,
-                                            NULL, NULL, NULL);
+                                            max_nr, 0, NULL, NULL, NULL);
                /*
                 * vm_normal_folio() filters out zero pages, but there might
                 * still be reserved folios to skip, perhaps in a VDSO.

diff --git a/mm/mlock.c b/mm/mlock.c
index 3cb72b579ffd3304f52579e22f61e5c1a8e6b08f..2238cdc5eb1c1e65b24223d8df930e8fac36699a 100644 (file)
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -307,14 +307,13 @@ void munlock_folio(struct folio *folio)
 static inline unsigned int folio_mlock_step(struct folio *folio,
                pte_t *pte, unsigned long addr, unsigned long end)
 {
-       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        unsigned int count = (end - addr) >> PAGE_SHIFT;
        pte_t ptent = ptep_get(pte);
 
        if (!folio_test_large(folio))
                return 1;
 
-       return folio_pte_batch(folio, addr, pte, ptent, count, fpb_flags, NULL,
+       return folio_pte_batch(folio, addr, pte, ptent, count, 0, NULL,
                               NULL, NULL);
 }
 

diff --git a/mm/mremap.c b/mm/mremap.c
index 36585041c760d1369cc2fe7428c705490437b7c9..d4d3ffc931502760b9c229fccb58e8783ffffc08 100644 (file)
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -173,7 +173,6 @@ static pte_t move_soft_dirty_pte(pte_t pte)
 static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
                pte_t *ptep, pte_t pte, int max_nr)
 {
-       const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        struct folio *folio;
 
        if (max_nr == 1)
@@ -183,7 +182,7 @@ static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr
        if (!folio || !folio_test_large(folio))
                return 1;
 
-       return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+       return folio_pte_batch(folio, addr, ptep, pte, max_nr, 0, NULL,
                               NULL, NULL);
 }
 

diff --git a/mm/rmap.c b/mm/rmap.c
index 4b1a2a33e39f0b1de02527b7c25c24024019dcbe..366e66651c88f5dc7cbc108b16c3ebceba1a4b69 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1849,7 +1849,6 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
                        struct page_vma_mapped_walk *pvmw,
                        enum ttu_flags flags, pte_t pte)
 {
-       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        unsigned long end_addr, addr = pvmw->address;
        struct vm_area_struct *vma = pvmw->vma;
        unsigned int max_nr;
@@ -1869,7 +1868,7 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
        if (pte_unused(pte))
                return 1;
 
-       return folio_pte_batch(folio, addr, pvmw->pte, pte, max_nr, fpb_flags,
+       return folio_pte_batch(folio, addr, pvmw->pte, pte, max_nr, 0,
                               NULL, NULL, NULL);
 }