mm/vma: use new VMA flags for sticky flags logic

Use the new vma_flags_t flags implementation to perform the logic around
sticky flags and what flags are ignored on VMA merge.

We make use of the new vma_flags_empty(), vma_flags_diff_pair(), and
vma_flags_and_mask() functionality.

Also update the VMA tests accordingly.

Link: https://lkml.kernel.org/r/369574f06360ffa44707047e3b58eb4897345fba.1774034900.git.ljs@kernel.org
Signed-off-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Acked-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: "Borislav Petkov (AMD)" <bp@alien8.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christian Brauner <brauner@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: Kees Cook <kees@kernel.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Ondrej Mosnacek <omosnace@redhat.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Moore <paul@paul-moore.com>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: Richard Weinberger <richard@nod.at>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stephen Smalley <stephen.smalley.work@gmail.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vineet Gupta <vgupta@kernel.org>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Will Deacon <will@kernel.org>
Cc: xu xin <xu.xin16@zte.com.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 7954a7a2b811addcc65bac206403024d37235f77..d7e647e31742a3cfe59980eb5f50254dbf7b356e 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -540,6 +540,7 @@ enum {
 
 /* VMA basic access permission flags */
 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
+#define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT)
 
 /*
  * Special vmas that are non-mergable, non-mlock()able.
@@ -585,27 +586,32 @@ enum {
  * possesses it but the other does not, the merged VMA should nonetheless have
  * applied to it:
  *
- *   VM_SOFTDIRTY - if a VMA is marked soft-dirty, that is has not had its
- *                  references cleared via /proc/$pid/clear_refs, any merged VMA
- *                  should be considered soft-dirty also as it operates at a VMA
- *                  granularity.
+ *   VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its
+ *                       references cleared via /proc/$pid/clear_refs, any
+ *                       merged VMA should be considered soft-dirty also as it
+ *                       operates at a VMA granularity.
  *
- * VM_MAYBE_GUARD - If a VMA may have guard regions in place it implies that
- *                  mapped page tables may contain metadata not described by the
- *                  VMA and thus any merged VMA may also contain this metadata,
- *                  and thus we must make this flag sticky.
+ * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies
+ *                       that mapped page tables may contain metadata not
+ *                       described by the VMA and thus any merged VMA may also
+ *                       contain this metadata, and thus we must make this flag
+ *                       sticky.
  */
-#define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD)
+#ifdef CONFIG_MEM_SOFT_DIRTY
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)
+#else
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)
+#endif
 
 /*
  * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one
  * of these flags and the other not does not preclude a merge.
  *
- *    VM_STICKY - When merging VMAs, VMA flags must match, unless they are
- *                'sticky'. If any sticky flags exist in either VMA, we simply
- *                set all of them on the merged VMA.
+ *    VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they
+ *                       are 'sticky'. If any sticky flags exist in either VMA,
+ *                       we simply set all of them on the merged VMA.
  */
-#define VM_IGNORE_MERGE VM_STICKY
+#define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS
 
 /*
  * Flags which should result in page tables being copied on fork. These are

diff --git a/mm/vma.c b/mm/vma.c
index 4d21e7d8e93c47053c8367bcf11db1f0471cb4ed..6af26619e020be9037fac175a322d8c8d3396d55 100644 (file)
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -86,10 +86,15 @@ static bool vma_is_fork_child(struct vm_area_struct *vma)
 static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
 {
        struct vm_area_struct *vma = merge_next ? vmg->next : vmg->prev;
+       vma_flags_t diff;
 
        if (!mpol_equal(vmg->policy, vma_policy(vma)))
                return false;
-       if ((vma->vm_flags ^ vmg->vm_flags) & ~VM_IGNORE_MERGE)
+
+       diff = vma_flags_diff_pair(&vma->flags, &vmg->vma_flags);
+       vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS);
+
+       if (!vma_flags_empty(&diff))
                return false;
        if (vma->vm_file != vmg->file)
                return false;
@@ -805,7 +810,8 @@ static bool can_merge_remove_vma(struct vm_area_struct *vma)
 static __must_check struct vm_area_struct *vma_merge_existing_range(
                struct vma_merge_struct *vmg)
 {
-       vm_flags_t sticky_flags = vmg->vm_flags & VM_STICKY;
+       vma_flags_t sticky_flags = vma_flags_and_mask(&vmg->vma_flags,
+                                                     VMA_STICKY_FLAGS);
        struct vm_area_struct *middle = vmg->middle;
        struct vm_area_struct *prev = vmg->prev;
        struct vm_area_struct *next;
@@ -898,15 +904,22 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
        vma_start_write(middle);
 
        if (merge_right) {
+               vma_flags_t next_sticky;
+
                vma_start_write(next);
                vmg->target = next;
-               sticky_flags |= (next->vm_flags & VM_STICKY);
+               next_sticky = vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS);
+               vma_flags_set_mask(&sticky_flags, next_sticky);
        }
 
        if (merge_left) {
+               vma_flags_t prev_sticky;
+
                vma_start_write(prev);
                vmg->target = prev;
-               sticky_flags |= (prev->vm_flags & VM_STICKY);
+
+               prev_sticky = vma_flags_and_mask(&prev->flags, VMA_STICKY_FLAGS);
+               vma_flags_set_mask(&sticky_flags, prev_sticky);
        }
 
        if (merge_both) {
@@ -976,7 +989,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
        if (err || commit_merge(vmg))
                goto abort;
 
-       vm_flags_set(vmg->target, sticky_flags);
+       vma_set_flags_mask(vmg->target, sticky_flags);
        khugepaged_enter_vma(vmg->target, vmg->vm_flags);
        vmg->state = VMA_MERGE_SUCCESS;
        return vmg->target;
@@ -1154,12 +1167,16 @@ int vma_expand(struct vma_merge_struct *vmg)
        struct vm_area_struct *target = vmg->target;
        struct vm_area_struct *next = vmg->next;
        bool remove_next = false;
-       vm_flags_t sticky_flags;
+       vma_flags_t sticky_flags =
+               vma_flags_and_mask(&vmg->vma_flags, VMA_STICKY_FLAGS);
+       vma_flags_t target_sticky;
        int ret = 0;
 
        mmap_assert_write_locked(vmg->mm);
        vma_start_write(target);
 
+       target_sticky = vma_flags_and_mask(&target->flags, VMA_STICKY_FLAGS);
+
        if (next && target != next && vmg->end == next->vm_end)
                remove_next = true;
 
@@ -1174,10 +1191,7 @@ int vma_expand(struct vma_merge_struct *vmg)
        VM_WARN_ON_VMG(target->vm_start < vmg->start ||
                       target->vm_end > vmg->end, vmg);
 
-       sticky_flags = vmg->vm_flags & VM_STICKY;
-       sticky_flags |= target->vm_flags & VM_STICKY;
-       if (remove_next)
-               sticky_flags |= next->vm_flags & VM_STICKY;
+       vma_flags_set_mask(&sticky_flags, target_sticky);
 
        /*
         * If we are removing the next VMA or copying from a VMA
@@ -1194,13 +1208,18 @@ int vma_expand(struct vma_merge_struct *vmg)
                return ret;
 
        if (remove_next) {
+               vma_flags_t next_sticky;
+
                vma_start_write(next);
                vmg->__remove_next = true;
+
+               next_sticky = vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS);
+               vma_flags_set_mask(&sticky_flags, next_sticky);
        }
        if (commit_merge(vmg))
                goto nomem;
 
-       vm_flags_set(target, sticky_flags);
+       vma_set_flags_mask(target, sticky_flags);
        return 0;
 
 nomem:
@@ -1950,10 +1969,15 @@ out:
  */
 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
 {
+       vma_flags_t diff = vma_flags_diff_pair(&a->flags, &b->flags);
+
+       vma_flags_clear_mask(&diff, VMA_ACCESS_FLAGS);
+       vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS);
+
        return a->vm_end == b->vm_start &&
                mpol_equal(vma_policy(a), vma_policy(b)) &&
                a->vm_file == b->vm_file &&
-               !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_IGNORE_MERGE)) &&
+               vma_flags_empty(&diff) &&
                b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
 }
 

diff --git a/tools/testing/vma/include/custom.h b/tools/testing/vma/include/custom.h
index 6200f938e586205052c4f575ff075800539e75bf..7cdd0f60600a551b896f7f0d1c4b6b4a530cef27 100644 (file)
--- a/tools/testing/vma/include/custom.h
+++ b/tools/testing/vma/include/custom.h
@@ -134,8 +134,3 @@ static __always_inline bool vma_flags_same_mask(vma_flags_t *flags,
        vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__))
 #define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \
                                       VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT)
-#ifdef CONFIG_MEM_SOFT_DIRTY
-#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)
-#else
-#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)
-#endif

diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h
index 1dee78c34872815c4030796f6a0d5bc153bfe6b5..65134303b645ff872966a24afddf862045eceb25 100644 (file)
--- a/tools/testing/vma/include/dup.h
+++ b/tools/testing/vma/include/dup.h
@@ -338,6 +338,7 @@ enum {
 
 /* VMA basic access permission flags */
 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
+#define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT)
 
 /*
  * Special vmas that are non-mergable, non-mlock()able.
@@ -363,9 +364,13 @@ enum {
 
 #define CAP_IPC_LOCK         14
 
-#define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD)
+#ifdef CONFIG_MEM_SOFT_DIRTY
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)
+#else
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)
+#endif
 
-#define VM_IGNORE_MERGE VM_STICKY
+#define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS
 
 #define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD)