mm: allow vma_start_read_locked/vma_start_read_locked_nested to fail

With upcoming replacement of vm_lock with vm_refcnt, we need to handle a
possibility of vma_start_read_locked/vma_start_read_locked_nested failing
due to refcount overflow.  Prepare for such possibility by changing these
APIs and adjusting their users.

Link: https://lkml.kernel.org/r/20250213224655.1680278-8-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Cc: Lokesh Gidra <lokeshgidra@google.com>
Tested-by: Shivank Garg <shivankg@amd.com>
Link: https://lkml.kernel.org/r/5e19ec93-8307-47c2-bb13-3ddf7150624e@amd.com
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christian Brauner <brauner@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Klara Modin <klarasmodin@gmail.com>
Cc: Liam R. Howlett <Liam.Howlett@Oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mateusz Guzik <mjguzik@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Minchan Kim <minchan@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: "Paul E . McKenney" <paulmck@kernel.org>
Cc: Peter Xu <peterx@redhat.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Sourav Panda <souravpanda@google.com>
Cc: Wei Yang <richard.weiyang@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 003c3e5c0a960b798cf7c6f232bd278f7ca64f11..09b48af68699ad56c76a0fd26e3cd1ad57328137 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -747,10 +747,11 @@ static inline bool vma_start_read(struct vm_area_struct *vma)
  * not be used in such cases because it might fail due to mm_lock_seq overflow.
  * This functionality is used to obtain vma read lock and drop the mmap read lock.
  */
-static inline void vma_start_read_locked_nested(struct vm_area_struct *vma, int subclass)
+static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int subclass)
 {
        mmap_assert_locked(vma->vm_mm);
        down_read_nested(&vma->vm_lock.lock, subclass);
+       return true;
 }
 
 /*
@@ -759,10 +760,11 @@ static inline void vma_start_read_locked_nested(struct vm_area_struct *vma, int
  * not be used in such cases because it might fail due to mm_lock_seq overflow.
  * This functionality is used to obtain vma read lock and drop the mmap read lock.
  */
-static inline void vma_start_read_locked(struct vm_area_struct *vma)
+static inline bool vma_start_read_locked(struct vm_area_struct *vma)
 {
        mmap_assert_locked(vma->vm_mm);
        down_read(&vma->vm_lock.lock);
+       return true;
 }
 
 static inline void vma_end_read(struct vm_area_struct *vma)

diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 48ac81bbfee60cb531e882a9f38bc21949623af6..fbf2cf62ab9f53bcc48b41a1251171ad8ec599c6 100644 (file)
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -85,8 +85,12 @@ static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm,
 
        mmap_read_lock(mm);
        vma = find_vma_and_prepare_anon(mm, address);
-       if (!IS_ERR(vma))
-               vma_start_read_locked(vma);
+       if (!IS_ERR(vma)) {
+               bool locked = vma_start_read_locked(vma);
+
+               if (!locked)
+                       vma = ERR_PTR(-EAGAIN);
+       }
 
        mmap_read_unlock(mm);
        return vma;
@@ -1555,12 +1559,24 @@ static int uffd_move_lock(struct mm_struct *mm,
 
        mmap_read_lock(mm);
        err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap);
-       if (!err) {
-               vma_start_read_locked(*dst_vmap);
-               if (*dst_vmap != *src_vmap)
-                       vma_start_read_locked_nested(*src_vmap,
-                                               SINGLE_DEPTH_NESTING);
+       if (err)
+               goto out;
+
+       if (!vma_start_read_locked(*dst_vmap)) {
+               err = -EAGAIN;
+               goto out;
        }
+
+       /* Nothing further to do if both vmas are locked. */
+       if (*dst_vmap == *src_vmap)
+               goto out;
+
+       if (!vma_start_read_locked_nested(*src_vmap, SINGLE_DEPTH_NESTING)) {
+               /* Undo dst_vmap locking if src_vmap failed to lock */
+               vma_end_read(*dst_vmap);
+               err = -EAGAIN;
+       }
+out:
        mmap_read_unlock(mm);
        return err;
 }