mm/damon/vaddr: attempt per-vma lock during page table walk

Currently, DAMON virtual address operations use mmap_read_lock during page
table walks, which can cause unnecessary contention under high
concurrency.

Introduce damon_va_walk_page_range() to first attempt acquiring a per-vma
lock.  If the VMA is found and the range is fully contained within it, the
page table walk proceeds with the per-vma lock instead of mmap_read_lock.

This optimization is expected to be particularly effective for
damon_va_young() and damon_va_mkold(), which are frequently called and
typically operate within a single VMA.

Link: https://lore.kernel.org/20260512151523.2092638-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Reviewed-by: SeongJae Park <sj@kernel.org>
Cc: Nanyong Sun <sunnanyong@huawei.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index 1b0ebe3b6951e84d046ca0a48f2800ad210e23ac..d271476035641bba84a01a471c6dc52d6070877a 100644 (file)
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -237,6 +237,35 @@ static void damon_va_update(struct damon_ctx *ctx)
        }
 }
 
+static void damon_va_walk_page_range(struct mm_struct *mm, unsigned long start,
+               unsigned long end, struct mm_walk_ops *ops, void *private)
+{
+       struct vm_area_struct *vma;
+
+       vma = lock_vma_under_rcu(mm, start);
+       if (!vma)
+               goto lock_mmap;
+
+       if (end > vma->vm_end) {
+               vma_end_read(vma);
+               goto lock_mmap;
+       }
+
+       if (!(vma->vm_flags & VM_PFNMAP)) {
+               ops->walk_lock = PGWALK_VMA_RDLOCK_VERIFY;
+               walk_page_range_vma(vma, start, end, ops, private);
+       }
+
+       vma_end_read(vma);
+       return;
+
+lock_mmap:
+       mmap_read_lock(mm);
+       ops->walk_lock = PGWALK_RDLOCK;
+       walk_page_range(mm, start, end, ops, private);
+       mmap_read_unlock(mm);
+}
+
 static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
                unsigned long next, struct mm_walk *walk)
 {
@@ -315,17 +344,14 @@ out:
 #define damon_mkold_hugetlb_entry NULL
 #endif /* CONFIG_HUGETLB_PAGE */
 
-static const struct mm_walk_ops damon_mkold_ops = {
-       .pmd_entry = damon_mkold_pmd_entry,
-       .hugetlb_entry = damon_mkold_hugetlb_entry,
-       .walk_lock = PGWALK_RDLOCK,
-};
-
 static void damon_va_mkold(struct mm_struct *mm, unsigned long addr)
 {
-       mmap_read_lock(mm);
-       walk_page_range(mm, addr, addr + 1, &damon_mkold_ops, NULL);
-       mmap_read_unlock(mm);
+       struct mm_walk_ops damon_mkold_ops = {
+               .pmd_entry = damon_mkold_pmd_entry,
+               .hugetlb_entry = damon_mkold_hugetlb_entry,
+       };
+
+       damon_va_walk_page_range(mm, addr, addr + 1, &damon_mkold_ops, NULL);
 }
 
 /*
@@ -445,12 +471,6 @@ out:
 #define damon_young_hugetlb_entry NULL
 #endif /* CONFIG_HUGETLB_PAGE */
 
-static const struct mm_walk_ops damon_young_ops = {
-       .pmd_entry = damon_young_pmd_entry,
-       .hugetlb_entry = damon_young_hugetlb_entry,
-       .walk_lock = PGWALK_RDLOCK,
-};
-
 static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
                unsigned long *folio_sz)
 {
@@ -459,9 +479,12 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
                .young = false,
        };
 
-       mmap_read_lock(mm);
-       walk_page_range(mm, addr, addr + 1, &damon_young_ops, &arg);
-       mmap_read_unlock(mm);
+       struct mm_walk_ops damon_young_ops = {
+               .pmd_entry = damon_young_pmd_entry,
+               .hugetlb_entry = damon_young_hugetlb_entry,
+       };
+
+       damon_va_walk_page_range(mm, addr, addr + 1, &damon_young_ops, &arg);
        return arg.young;
 }
 
@@ -750,7 +773,6 @@ static unsigned long damos_va_migrate(struct damon_target *target,
        struct mm_walk_ops walk_ops = {
                .pmd_entry = damos_va_migrate_pmd_entry,
                .pte_entry = NULL,
-               .walk_lock = PGWALK_RDLOCK,
        };
 
        use_target_nid = dests->nr_dests == 0;
@@ -768,9 +790,7 @@ static unsigned long damos_va_migrate(struct damon_target *target,
        if (!mm)
                goto free_lists;
 
-       mmap_read_lock(mm);
-       walk_page_range(mm, r->ar.start, r->ar.end, &walk_ops, &priv);
-       mmap_read_unlock(mm);
+       damon_va_walk_page_range(mm, r->ar.start, r->ar.end, &walk_ops, &priv);
        mmput(mm);
 
        for (int i = 0; i < nr_dests; i++) {
@@ -862,7 +882,6 @@ static unsigned long damos_va_stat(struct damon_target *target,
        struct mm_struct *mm;
        struct mm_walk_ops walk_ops = {
                .pmd_entry = damos_va_stat_pmd_entry,
-               .walk_lock = PGWALK_RDLOCK,
        };
 
        priv.scheme = s;
@@ -875,9 +894,7 @@ static unsigned long damos_va_stat(struct damon_target *target,
        if (!mm)
                return 0;
 
-       mmap_read_lock(mm);
-       walk_page_range(mm, r->ar.start, r->ar.end, &walk_ops, &priv);
-       mmap_read_unlock(mm);
+       damon_va_walk_page_range(mm, r->ar.start, r->ar.end, &walk_ops, &priv);
        mmput(mm);
        return 0;
 }