mm: split folio_pte_batch() into folio_pte_batch() and folio_pte_batch_flags()

Many users (including upcoming ones) don't really need the flags etc, and
can live with the possible overhead of a function call.

So let's provide a basic, non-inlined folio_pte_batch(), to avoid code
bloat while still providing a variant that optimizes out all flag checks
at runtime.  folio_pte_batch_flags() will get inlined into
folio_pte_batch(), optimizing out any conditionals that depend on input
flags.

folio_pte_batch() will behave like folio_pte_batch_flags() when no flags
are specified.  It's okay to add new users of folio_pte_batch_flags(), but
using folio_pte_batch() if applicable is preferred.

So, before this change, folio_pte_batch() was inlined into the C file
optimized by propagating constants within the resulting object file.

With this change, we now also have a folio_pte_batch() that is optimized
by propagating all constants.  But instead of having one instance per
object file, we have a single shared one.

In zap_present_ptes(), where we care about performance, the compiler
already seem to generate a call to a common inlined folio_pte_batch()
variant, shared with fork() code.  So calling the new non-inlined variant
should not make a difference.

While at it, drop the "addr" parameter that is unused.

Link: https://lkml.kernel.org/r/20250702104926.212243-4-david@redhat.com
Signed-off-by: David Hildenbrand <david@redhat.com>
Suggested-by: Andrew Morton <akpm@linux-foundation.org>
Link: https://lore.kernel.org/linux-mm/20250503182858.5a02729fcffd6d4723afcfc2@linux-foundation.org/
Reviewed-by: Oscar Salvador <osalvador@suse.de>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Dev Jain <dev.jain@arm.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Jann Horn <jannh@google.com>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mathew Brost <matthew.brost@intel.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/internal.h b/mm/internal.h
index 40ee7200e510bda72ac31ee3adb70ab8557f76da..c7d18f608c3f1a70a47608a675df52717cc6ee64 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -218,9 +218,8 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)
 }
 
 /**
- * folio_pte_batch - detect a PTE batch for a large folio
+ * folio_pte_batch_flags - detect a PTE batch for a large folio
  * @folio: The large folio to detect a PTE batch for.
- * @addr: The user virtual address the first page is mapped at.
  * @ptep: Page table pointer for the first entry.
  * @pte: Page table entry for the first page.
  * @max_nr: The maximum number of table entries to consider.
@@ -243,9 +242,12 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)
  * must be limited by the caller so scanning cannot exceed a single VMA and
  * a single page table.
  *
+ * This function will be inlined to optimize based on the input parameters;
+ * consider using folio_pte_batch() instead if applicable.
+ *
  * Return: the number of table entries in the batch.
  */
-static inline unsigned int folio_pte_batch(struct folio *folio, unsigned long addr,
+static inline unsigned int folio_pte_batch_flags(struct folio *folio,
                pte_t *ptep, pte_t pte, unsigned int max_nr, fpb_t flags,
                bool *any_writable, bool *any_young, bool *any_dirty)
 {
@@ -293,6 +295,9 @@ static inline unsigned int folio_pte_batch(struct folio *folio, unsigned long ad
        return min(nr, max_nr);
 }
 
+unsigned int folio_pte_batch(struct folio *folio, pte_t *ptep, pte_t pte,
+               unsigned int max_nr);
+
 /**
  * pte_move_swp_offset - Move the swap entry offset field of a swap pte
  *      forward or backward by delta

diff --git a/mm/madvise.c b/mm/madvise.c
index e7f1d4caad81eae47201decdaaf625978a32846c..7c4958f694b40565fe615856aa6ff44619f2ad52 100644 (file)
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -348,8 +348,8 @@ static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end,
 {
        int max_nr = (end - addr) / PAGE_SIZE;
 
-       return folio_pte_batch(folio, addr, ptep, pte, max_nr, 0, NULL,
-                              any_young, any_dirty);
+       return folio_pte_batch_flags(folio, ptep, pte, max_nr, 0, NULL,
+                                    any_young, any_dirty);
 }
 
 static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,

diff --git a/mm/memory.c b/mm/memory.c
index a03f1964db3388be9d8ba292bf35a50573131cb7..042088340b73c56821fd0164e5cf033712d36430 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -995,8 +995,8 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
                if (vma_soft_dirty_enabled(src_vma))
                        flags |= FPB_RESPECT_SOFT_DIRTY;
 
-               nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags,
-                                    &any_writable, NULL, NULL);
+               nr = folio_pte_batch_flags(folio, src_pte, pte, max_nr, flags,
+                                          &any_writable, NULL, NULL);
                folio_ref_add(folio, nr);
                if (folio_test_anon(folio)) {
                        if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page,
@@ -1564,9 +1564,7 @@ static inline int zap_present_ptes(struct mmu_gather *tlb,
         * by keeping the batching logic separate.
         */
        if (unlikely(folio_test_large(folio) && max_nr != 1)) {
-               nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, 0,
-                                    NULL, NULL, NULL);
-
+               nr = folio_pte_batch(folio, pte, ptent, max_nr);
                zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,
                                       addr, details, rss, force_flush,
                                       force_break, any_skipped);

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 2a25eedc3b1c0a3d4e7f2cf332be03bb3c539784..eb83cff7db8c35708296e5216e07783dc1bcb9b6 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -711,8 +711,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
                if (!folio || folio_is_zone_device(folio))
                        continue;
                if (folio_test_large(folio) && max_nr != 1)
-                       nr = folio_pte_batch(folio, addr, pte, ptent,
-                                            max_nr, 0, NULL, NULL, NULL);
+                       nr = folio_pte_batch(folio, pte, ptent, max_nr);
                /*
                 * vm_normal_folio() filters out zero pages, but there might
                 * still be reserved folios to skip, perhaps in a VDSO.

diff --git a/mm/mlock.c b/mm/mlock.c
index 2238cdc5eb1c1e65b24223d8df930e8fac36699a..a1d93ad33c6db5cb9782f84d58cbb7a893b3373d 100644 (file)
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -313,8 +313,7 @@ static inline unsigned int folio_mlock_step(struct folio *folio,
        if (!folio_test_large(folio))
                return 1;
 
-       return folio_pte_batch(folio, addr, pte, ptent, count, 0, NULL,
-                              NULL, NULL);
+       return folio_pte_batch(folio, pte, ptent, count);
 }
 
 static inline bool allow_mlock_munlock(struct folio *folio,

diff --git a/mm/mremap.c b/mm/mremap.c
index d4d3ffc931502760b9c229fccb58e8783ffffc08..1f5bebbb9c0cbe565f1fae7c47c7d9f68cc7b078 100644 (file)
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -182,8 +182,7 @@ static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr
        if (!folio || !folio_test_large(folio))
                return 1;
 
-       return folio_pte_batch(folio, addr, ptep, pte, max_nr, 0, NULL,
-                              NULL, NULL);
+       return folio_pte_batch(folio, ptep, pte, max_nr);
 }
 
 static int move_ptes(struct pagetable_move_control *pmc,

diff --git a/mm/rmap.c b/mm/rmap.c
index 366e66651c88f5dc7cbc108b16c3ebceba1a4b69..4c833b43fef971f677e0f3f837647b0c643ede84 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1868,8 +1868,7 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
        if (pte_unused(pte))
                return 1;
 
-       return folio_pte_batch(folio, addr, pvmw->pte, pte, max_nr, 0,
-                              NULL, NULL, NULL);
+       return folio_pte_batch(folio, pvmw->pte, pte, max_nr);
 }
 
 /*

diff --git a/mm/util.c b/mm/util.c
index 20bbfe4ce1b8b1155201d34722f8f27f784c9afd..f134cefc906240bc7f3a476ef9ed3c40d4f86200 100644 (file)
--- a/mm/util.c
+++ b/mm/util.c
@@ -1171,3 +1171,32 @@ int compat_vma_mmap_prepare(struct file *file, struct vm_area_struct *vma)
        return 0;
 }
 EXPORT_SYMBOL(compat_vma_mmap_prepare);
+
+#ifdef CONFIG_MMU
+/**
+ * folio_pte_batch - detect a PTE batch for a large folio
+ * @folio: The large folio to detect a PTE batch for.
+ * @ptep: Page table pointer for the first entry.
+ * @pte: Page table entry for the first page.
+ * @max_nr: The maximum number of table entries to consider.
+ *
+ * This is a simplified variant of folio_pte_batch_flags().
+ *
+ * Detect a PTE batch: consecutive (present) PTEs that map consecutive
+ * pages of the same large folio in a single VMA and a single page table.
+ *
+ * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN,
+ * the accessed bit, writable bit, dirt-bit and soft-dirty bit.
+ *
+ * ptep must map any page of the folio. max_nr must be at least one and
+ * must be limited by the caller so scanning cannot exceed a single VMA and
+ * a single page table.
+ *
+ * Return: the number of table entries in the batch.
+ */
+unsigned int folio_pte_batch(struct folio *folio, pte_t *ptep, pte_t pte,
+               unsigned int max_nr)
+{
+       return folio_pte_batch_flags(folio, ptep, pte, max_nr, 0, NULL, NULL, NULL);
+}
+#endif /* CONFIG_MMU */