2 * mm/rmap.c - physical to virtual reverse mappings
4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5 * Released under the General Public License (GPL).
7 * Simple, low overhead reverse mapping scheme.
8 * Please try to keep this thing as modular as possible.
10 * Provides methods for unmapping each kind of mapped page:
11 * the anon methods track anonymous pages, and
12 * the file methods track pages belonging to an inode.
14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001
15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
17 * Contributions by Hugh Dickins 2003, 2004
21 * Lock ordering in mm:
23 * inode->i_rwsem (while writing or truncating, not reading or faulting)
25 * mapping->invalidate_lock (in filemap_fault)
27 * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below)
29 * mapping->i_mmap_rwsem
31 * mm->page_table_lock or pte_lock
32 * swap_lock (in swap_duplicate, swap_info_get)
33 * mmlist_lock (in mmput, drain_mmlist and others)
34 * mapping->private_lock (in block_dirty_folio)
35 * i_pages lock (widely used)
36 * lruvec->lru_lock (in folio_lruvec_lock_irq)
37 * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
38 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
39 * sb_lock (within inode_lock in fs/fs-writeback.c)
40 * i_pages lock (widely used, in set_page_dirty,
41 * in arch-dependent flush_dcache_mmap_lock,
42 * within bdi.wb->list_lock in __sync_single_inode)
44 * anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon)
48 * hugetlbfs PageHuge() take locks in this order:
49 * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
50 * vma_lock (hugetlb specific lock for pmd_sharing)
51 * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing)
56 #include <linux/sched/mm.h>
57 #include <linux/sched/task.h>
58 #include <linux/pagemap.h>
59 #include <linux/swap.h>
60 #include <linux/swapops.h>
61 #include <linux/slab.h>
62 #include <linux/init.h>
63 #include <linux/ksm.h>
64 #include <linux/rmap.h>
65 #include <linux/rcupdate.h>
66 #include <linux/export.h>
67 #include <linux/memcontrol.h>
68 #include <linux/mmu_notifier.h>
69 #include <linux/migrate.h>
70 #include <linux/hugetlb.h>
71 #include <linux/huge_mm.h>
72 #include <linux/backing-dev.h>
73 #include <linux/page_idle.h>
74 #include <linux/memremap.h>
75 #include <linux/userfaultfd_k.h>
76 #include <linux/mm_inline.h>
77 #include <linux/oom.h>
79 #include <asm/tlbflush.h>
81 #define CREATE_TRACE_POINTS
82 #include <trace/events/tlb.h>
83 #include <trace/events/migrate.h>
87 static struct kmem_cache
*anon_vma_cachep
;
88 static struct kmem_cache
*anon_vma_chain_cachep
;
90 static inline struct anon_vma
*anon_vma_alloc(void)
92 struct anon_vma
*anon_vma
;
94 anon_vma
= kmem_cache_alloc(anon_vma_cachep
, GFP_KERNEL
);
96 atomic_set(&anon_vma
->refcount
, 1);
97 anon_vma
->num_children
= 0;
98 anon_vma
->num_active_vmas
= 0;
99 anon_vma
->parent
= anon_vma
;
101 * Initialise the anon_vma root to point to itself. If called
102 * from fork, the root will be reset to the parents anon_vma.
104 anon_vma
->root
= anon_vma
;
110 static inline void anon_vma_free(struct anon_vma
*anon_vma
)
112 VM_BUG_ON(atomic_read(&anon_vma
->refcount
));
115 * Synchronize against folio_lock_anon_vma_read() such that
116 * we can safely hold the lock without the anon_vma getting
119 * Relies on the full mb implied by the atomic_dec_and_test() from
120 * put_anon_vma() against the acquire barrier implied by
121 * down_read_trylock() from folio_lock_anon_vma_read(). This orders:
123 * folio_lock_anon_vma_read() VS put_anon_vma()
124 * down_read_trylock() atomic_dec_and_test()
126 * atomic_read() rwsem_is_locked()
128 * LOCK should suffice since the actual taking of the lock must
129 * happen _before_ what follows.
132 if (rwsem_is_locked(&anon_vma
->root
->rwsem
)) {
133 anon_vma_lock_write(anon_vma
);
134 anon_vma_unlock_write(anon_vma
);
137 kmem_cache_free(anon_vma_cachep
, anon_vma
);
140 static inline struct anon_vma_chain
*anon_vma_chain_alloc(gfp_t gfp
)
142 return kmem_cache_alloc(anon_vma_chain_cachep
, gfp
);
145 static void anon_vma_chain_free(struct anon_vma_chain
*anon_vma_chain
)
147 kmem_cache_free(anon_vma_chain_cachep
, anon_vma_chain
);
150 static void anon_vma_chain_link(struct vm_area_struct
*vma
,
151 struct anon_vma_chain
*avc
,
152 struct anon_vma
*anon_vma
)
155 avc
->anon_vma
= anon_vma
;
156 list_add(&avc
->same_vma
, &vma
->anon_vma_chain
);
157 anon_vma_interval_tree_insert(avc
, &anon_vma
->rb_root
);
161 * __anon_vma_prepare - attach an anon_vma to a memory region
162 * @vma: the memory region in question
164 * This makes sure the memory mapping described by 'vma' has
165 * an 'anon_vma' attached to it, so that we can associate the
166 * anonymous pages mapped into it with that anon_vma.
168 * The common case will be that we already have one, which
169 * is handled inline by anon_vma_prepare(). But if
170 * not we either need to find an adjacent mapping that we
171 * can re-use the anon_vma from (very common when the only
172 * reason for splitting a vma has been mprotect()), or we
173 * allocate a new one.
175 * Anon-vma allocations are very subtle, because we may have
176 * optimistically looked up an anon_vma in folio_lock_anon_vma_read()
177 * and that may actually touch the rwsem even in the newly
178 * allocated vma (it depends on RCU to make sure that the
179 * anon_vma isn't actually destroyed).
181 * As a result, we need to do proper anon_vma locking even
182 * for the new allocation. At the same time, we do not want
183 * to do any locking for the common case of already having
186 int __anon_vma_prepare(struct vm_area_struct
*vma
)
188 struct mm_struct
*mm
= vma
->vm_mm
;
189 struct anon_vma
*anon_vma
, *allocated
;
190 struct anon_vma_chain
*avc
;
192 mmap_assert_locked(mm
);
195 avc
= anon_vma_chain_alloc(GFP_KERNEL
);
199 anon_vma
= find_mergeable_anon_vma(vma
);
202 anon_vma
= anon_vma_alloc();
203 if (unlikely(!anon_vma
))
204 goto out_enomem_free_avc
;
205 anon_vma
->num_children
++; /* self-parent link for new root */
206 allocated
= anon_vma
;
209 anon_vma_lock_write(anon_vma
);
210 /* page_table_lock to protect against threads */
211 spin_lock(&mm
->page_table_lock
);
212 if (likely(!vma
->anon_vma
)) {
213 vma
->anon_vma
= anon_vma
;
214 anon_vma_chain_link(vma
, avc
, anon_vma
);
215 anon_vma
->num_active_vmas
++;
219 spin_unlock(&mm
->page_table_lock
);
220 anon_vma_unlock_write(anon_vma
);
222 if (unlikely(allocated
))
223 put_anon_vma(allocated
);
225 anon_vma_chain_free(avc
);
230 anon_vma_chain_free(avc
);
236 * This is a useful helper function for locking the anon_vma root as
237 * we traverse the vma->anon_vma_chain, looping over anon_vma's that
240 * Such anon_vma's should have the same root, so you'd expect to see
241 * just a single mutex_lock for the whole traversal.
243 static inline struct anon_vma
*lock_anon_vma_root(struct anon_vma
*root
, struct anon_vma
*anon_vma
)
245 struct anon_vma
*new_root
= anon_vma
->root
;
246 if (new_root
!= root
) {
247 if (WARN_ON_ONCE(root
))
248 up_write(&root
->rwsem
);
250 down_write(&root
->rwsem
);
255 static inline void unlock_anon_vma_root(struct anon_vma
*root
)
258 up_write(&root
->rwsem
);
262 * Attach the anon_vmas from src to dst.
263 * Returns 0 on success, -ENOMEM on failure.
265 * anon_vma_clone() is called by vma_expand(), vma_merge(), __split_vma(),
266 * copy_vma() and anon_vma_fork(). The first four want an exact copy of src,
267 * while the last one, anon_vma_fork(), may try to reuse an existing anon_vma to
268 * prevent endless growth of anon_vma. Since dst->anon_vma is set to NULL before
269 * call, we can identify this case by checking (!dst->anon_vma &&
272 * If (!dst->anon_vma && src->anon_vma) is true, this function tries to find
273 * and reuse existing anon_vma which has no vmas and only one child anon_vma.
274 * This prevents degradation of anon_vma hierarchy to endless linear chain in
275 * case of constantly forking task. On the other hand, an anon_vma with more
276 * than one child isn't reused even if there was no alive vma, thus rmap
277 * walker has a good chance of avoiding scanning the whole hierarchy when it
278 * searches where page is mapped.
280 int anon_vma_clone(struct vm_area_struct
*dst
, struct vm_area_struct
*src
)
282 struct anon_vma_chain
*avc
, *pavc
;
283 struct anon_vma
*root
= NULL
;
285 list_for_each_entry_reverse(pavc
, &src
->anon_vma_chain
, same_vma
) {
286 struct anon_vma
*anon_vma
;
288 avc
= anon_vma_chain_alloc(GFP_NOWAIT
| __GFP_NOWARN
);
289 if (unlikely(!avc
)) {
290 unlock_anon_vma_root(root
);
292 avc
= anon_vma_chain_alloc(GFP_KERNEL
);
296 anon_vma
= pavc
->anon_vma
;
297 root
= lock_anon_vma_root(root
, anon_vma
);
298 anon_vma_chain_link(dst
, avc
, anon_vma
);
301 * Reuse existing anon_vma if it has no vma and only one
304 * Root anon_vma is never reused:
305 * it has self-parent reference and at least one child.
307 if (!dst
->anon_vma
&& src
->anon_vma
&&
308 anon_vma
->num_children
< 2 &&
309 anon_vma
->num_active_vmas
== 0)
310 dst
->anon_vma
= anon_vma
;
313 dst
->anon_vma
->num_active_vmas
++;
314 unlock_anon_vma_root(root
);
319 * dst->anon_vma is dropped here otherwise its num_active_vmas can
320 * be incorrectly decremented in unlink_anon_vmas().
321 * We can safely do this because callers of anon_vma_clone() don't care
322 * about dst->anon_vma if anon_vma_clone() failed.
324 dst
->anon_vma
= NULL
;
325 unlink_anon_vmas(dst
);
330 * Attach vma to its own anon_vma, as well as to the anon_vmas that
331 * the corresponding VMA in the parent process is attached to.
332 * Returns 0 on success, non-zero on failure.
334 int anon_vma_fork(struct vm_area_struct
*vma
, struct vm_area_struct
*pvma
)
336 struct anon_vma_chain
*avc
;
337 struct anon_vma
*anon_vma
;
340 /* Don't bother if the parent process has no anon_vma here. */
344 /* Drop inherited anon_vma, we'll reuse existing or allocate new. */
345 vma
->anon_vma
= NULL
;
348 * First, attach the new VMA to the parent VMA's anon_vmas,
349 * so rmap can find non-COWed pages in child processes.
351 error
= anon_vma_clone(vma
, pvma
);
355 /* An existing anon_vma has been reused, all done then. */
359 /* Then add our own anon_vma. */
360 anon_vma
= anon_vma_alloc();
363 anon_vma
->num_active_vmas
++;
364 avc
= anon_vma_chain_alloc(GFP_KERNEL
);
366 goto out_error_free_anon_vma
;
369 * The root anon_vma's rwsem is the lock actually used when we
370 * lock any of the anon_vmas in this anon_vma tree.
372 anon_vma
->root
= pvma
->anon_vma
->root
;
373 anon_vma
->parent
= pvma
->anon_vma
;
375 * With refcounts, an anon_vma can stay around longer than the
376 * process it belongs to. The root anon_vma needs to be pinned until
377 * this anon_vma is freed, because the lock lives in the root.
379 get_anon_vma(anon_vma
->root
);
380 /* Mark this anon_vma as the one where our new (COWed) pages go. */
381 vma
->anon_vma
= anon_vma
;
382 anon_vma_lock_write(anon_vma
);
383 anon_vma_chain_link(vma
, avc
, anon_vma
);
384 anon_vma
->parent
->num_children
++;
385 anon_vma_unlock_write(anon_vma
);
389 out_error_free_anon_vma
:
390 put_anon_vma(anon_vma
);
392 unlink_anon_vmas(vma
);
396 void unlink_anon_vmas(struct vm_area_struct
*vma
)
398 struct anon_vma_chain
*avc
, *next
;
399 struct anon_vma
*root
= NULL
;
402 * Unlink each anon_vma chained to the VMA. This list is ordered
403 * from newest to oldest, ensuring the root anon_vma gets freed last.
405 list_for_each_entry_safe(avc
, next
, &vma
->anon_vma_chain
, same_vma
) {
406 struct anon_vma
*anon_vma
= avc
->anon_vma
;
408 root
= lock_anon_vma_root(root
, anon_vma
);
409 anon_vma_interval_tree_remove(avc
, &anon_vma
->rb_root
);
412 * Leave empty anon_vmas on the list - we'll need
413 * to free them outside the lock.
415 if (RB_EMPTY_ROOT(&anon_vma
->rb_root
.rb_root
)) {
416 anon_vma
->parent
->num_children
--;
420 list_del(&avc
->same_vma
);
421 anon_vma_chain_free(avc
);
424 vma
->anon_vma
->num_active_vmas
--;
427 * vma would still be needed after unlink, and anon_vma will be prepared
430 vma
->anon_vma
= NULL
;
432 unlock_anon_vma_root(root
);
435 * Iterate the list once more, it now only contains empty and unlinked
436 * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
437 * needing to write-acquire the anon_vma->root->rwsem.
439 list_for_each_entry_safe(avc
, next
, &vma
->anon_vma_chain
, same_vma
) {
440 struct anon_vma
*anon_vma
= avc
->anon_vma
;
442 VM_WARN_ON(anon_vma
->num_children
);
443 VM_WARN_ON(anon_vma
->num_active_vmas
);
444 put_anon_vma(anon_vma
);
446 list_del(&avc
->same_vma
);
447 anon_vma_chain_free(avc
);
451 static void anon_vma_ctor(void *data
)
453 struct anon_vma
*anon_vma
= data
;
455 init_rwsem(&anon_vma
->rwsem
);
456 atomic_set(&anon_vma
->refcount
, 0);
457 anon_vma
->rb_root
= RB_ROOT_CACHED
;
460 void __init
anon_vma_init(void)
462 anon_vma_cachep
= kmem_cache_create("anon_vma", sizeof(struct anon_vma
),
463 0, SLAB_TYPESAFE_BY_RCU
|SLAB_PANIC
|SLAB_ACCOUNT
,
465 anon_vma_chain_cachep
= KMEM_CACHE(anon_vma_chain
,
466 SLAB_PANIC
|SLAB_ACCOUNT
);
470 * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
472 * Since there is no serialization what so ever against folio_remove_rmap_*()
473 * the best this function can do is return a refcount increased anon_vma
474 * that might have been relevant to this page.
476 * The page might have been remapped to a different anon_vma or the anon_vma
477 * returned may already be freed (and even reused).
479 * In case it was remapped to a different anon_vma, the new anon_vma will be a
480 * child of the old anon_vma, and the anon_vma lifetime rules will therefore
481 * ensure that any anon_vma obtained from the page will still be valid for as
482 * long as we observe page_mapped() [ hence all those page_mapped() tests ].
484 * All users of this function must be very careful when walking the anon_vma
485 * chain and verify that the page in question is indeed mapped in it
486 * [ something equivalent to page_mapped_in_vma() ].
488 * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
489 * folio_remove_rmap_*() that the anon_vma pointer from page->mapping is valid
490 * if there is a mapcount, we can dereference the anon_vma after observing
493 * NOTE: the caller should normally hold folio lock when calling this. If
494 * not, the caller needs to double check the anon_vma didn't change after
495 * taking the anon_vma lock for either read or write (UFFDIO_MOVE can modify it
496 * concurrently without folio lock protection). See folio_lock_anon_vma_read()
497 * which has already covered that, and comment above remap_pages().
499 struct anon_vma
*folio_get_anon_vma(const struct folio
*folio
)
501 struct anon_vma
*anon_vma
= NULL
;
502 unsigned long anon_mapping
;
505 anon_mapping
= (unsigned long)READ_ONCE(folio
->mapping
);
506 if ((anon_mapping
& PAGE_MAPPING_FLAGS
) != PAGE_MAPPING_ANON
)
508 if (!folio_mapped(folio
))
511 anon_vma
= (struct anon_vma
*) (anon_mapping
- PAGE_MAPPING_ANON
);
512 if (!atomic_inc_not_zero(&anon_vma
->refcount
)) {
518 * If this folio is still mapped, then its anon_vma cannot have been
519 * freed. But if it has been unmapped, we have no security against the
520 * anon_vma structure being freed and reused (for another anon_vma:
521 * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
522 * above cannot corrupt).
524 if (!folio_mapped(folio
)) {
526 put_anon_vma(anon_vma
);
536 * Similar to folio_get_anon_vma() except it locks the anon_vma.
538 * Its a little more complex as it tries to keep the fast path to a single
539 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
540 * reference like with folio_get_anon_vma() and then block on the mutex
541 * on !rwc->try_lock case.
543 struct anon_vma
*folio_lock_anon_vma_read(const struct folio
*folio
,
544 struct rmap_walk_control
*rwc
)
546 struct anon_vma
*anon_vma
= NULL
;
547 struct anon_vma
*root_anon_vma
;
548 unsigned long anon_mapping
;
552 anon_mapping
= (unsigned long)READ_ONCE(folio
->mapping
);
553 if ((anon_mapping
& PAGE_MAPPING_FLAGS
) != PAGE_MAPPING_ANON
)
555 if (!folio_mapped(folio
))
558 anon_vma
= (struct anon_vma
*) (anon_mapping
- PAGE_MAPPING_ANON
);
559 root_anon_vma
= READ_ONCE(anon_vma
->root
);
560 if (down_read_trylock(&root_anon_vma
->rwsem
)) {
562 * folio_move_anon_rmap() might have changed the anon_vma as we
563 * might not hold the folio lock here.
565 if (unlikely((unsigned long)READ_ONCE(folio
->mapping
) !=
567 up_read(&root_anon_vma
->rwsem
);
573 * If the folio is still mapped, then this anon_vma is still
574 * its anon_vma, and holding the mutex ensures that it will
575 * not go away, see anon_vma_free().
577 if (!folio_mapped(folio
)) {
578 up_read(&root_anon_vma
->rwsem
);
584 if (rwc
&& rwc
->try_lock
) {
586 rwc
->contended
= true;
590 /* trylock failed, we got to sleep */
591 if (!atomic_inc_not_zero(&anon_vma
->refcount
)) {
596 if (!folio_mapped(folio
)) {
598 put_anon_vma(anon_vma
);
602 /* we pinned the anon_vma, its safe to sleep */
604 anon_vma_lock_read(anon_vma
);
607 * folio_move_anon_rmap() might have changed the anon_vma as we might
608 * not hold the folio lock here.
610 if (unlikely((unsigned long)READ_ONCE(folio
->mapping
) !=
612 anon_vma_unlock_read(anon_vma
);
613 put_anon_vma(anon_vma
);
618 if (atomic_dec_and_test(&anon_vma
->refcount
)) {
620 * Oops, we held the last refcount, release the lock
621 * and bail -- can't simply use put_anon_vma() because
622 * we'll deadlock on the anon_vma_lock_write() recursion.
624 anon_vma_unlock_read(anon_vma
);
625 __put_anon_vma(anon_vma
);
636 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
638 * Flush TLB entries for recently unmapped pages from remote CPUs. It is
639 * important if a PTE was dirty when it was unmapped that it's flushed
640 * before any IO is initiated on the page to prevent lost writes. Similarly,
641 * it must be flushed before freeing to prevent data leakage.
643 void try_to_unmap_flush(void)
645 struct tlbflush_unmap_batch
*tlb_ubc
= ¤t
->tlb_ubc
;
647 if (!tlb_ubc
->flush_required
)
650 arch_tlbbatch_flush(&tlb_ubc
->arch
);
651 tlb_ubc
->flush_required
= false;
652 tlb_ubc
->writable
= false;
655 /* Flush iff there are potentially writable TLB entries that can race with IO */
656 void try_to_unmap_flush_dirty(void)
658 struct tlbflush_unmap_batch
*tlb_ubc
= ¤t
->tlb_ubc
;
660 if (tlb_ubc
->writable
)
661 try_to_unmap_flush();
665 * Bits 0-14 of mm->tlb_flush_batched record pending generations.
666 * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations.
668 #define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16
669 #define TLB_FLUSH_BATCH_PENDING_MASK \
670 ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1)
671 #define TLB_FLUSH_BATCH_PENDING_LARGE \
672 (TLB_FLUSH_BATCH_PENDING_MASK / 2)
674 static void set_tlb_ubc_flush_pending(struct mm_struct
*mm
, pte_t pteval
,
675 unsigned long start
, unsigned long end
)
677 struct tlbflush_unmap_batch
*tlb_ubc
= ¤t
->tlb_ubc
;
679 bool writable
= pte_dirty(pteval
);
681 if (!pte_accessible(mm
, pteval
))
684 arch_tlbbatch_add_pending(&tlb_ubc
->arch
, mm
, start
, end
);
685 tlb_ubc
->flush_required
= true;
688 * Ensure compiler does not re-order the setting of tlb_flush_batched
689 * before the PTE is cleared.
692 batch
= atomic_read(&mm
->tlb_flush_batched
);
694 if ((batch
& TLB_FLUSH_BATCH_PENDING_MASK
) > TLB_FLUSH_BATCH_PENDING_LARGE
) {
696 * Prevent `pending' from catching up with `flushed' because of
697 * overflow. Reset `pending' and `flushed' to be 1 and 0 if
698 * `pending' becomes large.
700 if (!atomic_try_cmpxchg(&mm
->tlb_flush_batched
, &batch
, 1))
703 atomic_inc(&mm
->tlb_flush_batched
);
707 * If the PTE was dirty then it's best to assume it's writable. The
708 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
709 * before the page is queued for IO.
712 tlb_ubc
->writable
= true;
716 * Returns true if the TLB flush should be deferred to the end of a batch of
717 * unmap operations to reduce IPIs.
719 static bool should_defer_flush(struct mm_struct
*mm
, enum ttu_flags flags
)
721 if (!(flags
& TTU_BATCH_FLUSH
))
724 return arch_tlbbatch_should_defer(mm
);
728 * Reclaim unmaps pages under the PTL but do not flush the TLB prior to
729 * releasing the PTL if TLB flushes are batched. It's possible for a parallel
730 * operation such as mprotect or munmap to race between reclaim unmapping
731 * the page and flushing the page. If this race occurs, it potentially allows
732 * access to data via a stale TLB entry. Tracking all mm's that have TLB
733 * batching in flight would be expensive during reclaim so instead track
734 * whether TLB batching occurred in the past and if so then do a flush here
735 * if required. This will cost one additional flush per reclaim cycle paid
736 * by the first operation at risk such as mprotect and mumap.
738 * This must be called under the PTL so that an access to tlb_flush_batched
739 * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
742 void flush_tlb_batched_pending(struct mm_struct
*mm
)
744 int batch
= atomic_read(&mm
->tlb_flush_batched
);
745 int pending
= batch
& TLB_FLUSH_BATCH_PENDING_MASK
;
746 int flushed
= batch
>> TLB_FLUSH_BATCH_FLUSHED_SHIFT
;
748 if (pending
!= flushed
) {
749 arch_flush_tlb_batched_pending(mm
);
751 * If the new TLB flushing is pending during flushing, leave
752 * mm->tlb_flush_batched as is, to avoid losing flushing.
754 atomic_cmpxchg(&mm
->tlb_flush_batched
, batch
,
755 pending
| (pending
<< TLB_FLUSH_BATCH_FLUSHED_SHIFT
));
759 static void set_tlb_ubc_flush_pending(struct mm_struct
*mm
, pte_t pteval
,
760 unsigned long start
, unsigned long end
)
764 static bool should_defer_flush(struct mm_struct
*mm
, enum ttu_flags flags
)
768 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
771 * page_address_in_vma - The virtual address of a page in this VMA.
772 * @folio: The folio containing the page.
773 * @page: The page within the folio.
774 * @vma: The VMA we need to know the address in.
776 * Calculates the user virtual address of this page in the specified VMA.
777 * It is the caller's responsibility to check the page is actually
778 * within the VMA. There may not currently be a PTE pointing at this
779 * page, but if a page fault occurs at this address, this is the page
780 * which will be accessed.
782 * Context: Caller should hold a reference to the folio. Caller should
783 * hold a lock (eg the i_mmap_lock or the mmap_lock) which keeps the
784 * VMA from being altered.
786 * Return: The virtual address corresponding to this page in the VMA.
788 unsigned long page_address_in_vma(const struct folio
*folio
,
789 const struct page
*page
, const struct vm_area_struct
*vma
)
791 if (folio_test_anon(folio
)) {
792 struct anon_vma
*anon_vma
= folio_anon_vma(folio
);
794 * Note: swapoff's unuse_vma() is more efficient with this
795 * check, and needs it to match anon_vma when KSM is active.
797 if (!vma
->anon_vma
|| !anon_vma
||
798 vma
->anon_vma
->root
!= anon_vma
->root
)
800 } else if (!vma
->vm_file
) {
802 } else if (vma
->vm_file
->f_mapping
!= folio
->mapping
) {
806 /* KSM folios don't reach here because of the !anon_vma check */
807 return vma_address(vma
, page_pgoff(folio
, page
), 1);
811 * Returns the actual pmd_t* where we expect 'address' to be mapped from, or
812 * NULL if it doesn't exist. No guarantees / checks on what the pmd_t*
815 pmd_t
*mm_find_pmd(struct mm_struct
*mm
, unsigned long address
)
822 pgd
= pgd_offset(mm
, address
);
823 if (!pgd_present(*pgd
))
826 p4d
= p4d_offset(pgd
, address
);
827 if (!p4d_present(*p4d
))
830 pud
= pud_offset(p4d
, address
);
831 if (!pud_present(*pud
))
834 pmd
= pmd_offset(pud
, address
);
839 struct folio_referenced_arg
{
842 unsigned long vm_flags
;
843 struct mem_cgroup
*memcg
;
847 * arg: folio_referenced_arg will be passed
849 static bool folio_referenced_one(struct folio
*folio
,
850 struct vm_area_struct
*vma
, unsigned long address
, void *arg
)
852 struct folio_referenced_arg
*pra
= arg
;
853 DEFINE_FOLIO_VMA_WALK(pvmw
, folio
, vma
, address
, 0);
855 unsigned long start
= address
, ptes
= 0;
857 while (page_vma_mapped_walk(&pvmw
)) {
858 address
= pvmw
.address
;
860 if (vma
->vm_flags
& VM_LOCKED
) {
861 if (!folio_test_large(folio
) || !pvmw
.pte
) {
862 /* Restore the mlock which got missed */
863 mlock_vma_folio(folio
, vma
);
864 page_vma_mapped_walk_done(&pvmw
);
865 pra
->vm_flags
|= VM_LOCKED
;
866 return false; /* To break the loop */
869 * For large folio fully mapped to VMA, will
870 * be handled after the pvmw loop.
872 * For large folio cross VMA boundaries, it's
873 * expected to be picked by page reclaim. But
874 * should skip reference of pages which are in
875 * the range of VM_LOCKED vma. As page reclaim
876 * should just count the reference of pages out
877 * the range of VM_LOCKED vma.
885 * Skip the non-shared swapbacked folio mapped solely by
886 * the exiting or OOM-reaped process. This avoids redundant
887 * swap-out followed by an immediate unmap.
889 if ((!atomic_read(&vma
->vm_mm
->mm_users
) ||
890 check_stable_address_space(vma
->vm_mm
)) &&
891 folio_test_anon(folio
) && folio_test_swapbacked(folio
) &&
892 !folio_maybe_mapped_shared(folio
)) {
893 pra
->referenced
= -1;
894 page_vma_mapped_walk_done(&pvmw
);
898 if (lru_gen_enabled() && pvmw
.pte
) {
899 if (lru_gen_look_around(&pvmw
))
901 } else if (pvmw
.pte
) {
902 if (ptep_clear_flush_young_notify(vma
, address
,
905 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE
)) {
906 if (pmdp_clear_flush_young_notify(vma
, address
,
910 /* unexpected pmd-mapped folio? */
917 if ((vma
->vm_flags
& VM_LOCKED
) &&
918 folio_test_large(folio
) &&
919 folio_within_vma(folio
, vma
)) {
920 unsigned long s_align
, e_align
;
922 s_align
= ALIGN_DOWN(start
, PMD_SIZE
);
923 e_align
= ALIGN_DOWN(start
+ folio_size(folio
) - 1, PMD_SIZE
);
925 /* folio doesn't cross page table boundary and fully mapped */
926 if ((s_align
== e_align
) && (ptes
== folio_nr_pages(folio
))) {
927 /* Restore the mlock which got missed */
928 mlock_vma_folio(folio
, vma
);
929 pra
->vm_flags
|= VM_LOCKED
;
930 return false; /* To break the loop */
935 folio_clear_idle(folio
);
936 if (folio_test_clear_young(folio
))
941 pra
->vm_flags
|= vma
->vm_flags
& ~VM_LOCKED
;
945 return false; /* To break the loop */
950 static bool invalid_folio_referenced_vma(struct vm_area_struct
*vma
, void *arg
)
952 struct folio_referenced_arg
*pra
= arg
;
953 struct mem_cgroup
*memcg
= pra
->memcg
;
956 * Ignore references from this mapping if it has no recency. If the
957 * folio has been used in another mapping, we will catch it; if this
958 * other mapping is already gone, the unmap path will have set the
959 * referenced flag or activated the folio in zap_pte_range().
961 if (!vma_has_recency(vma
))
965 * If we are reclaiming on behalf of a cgroup, skip counting on behalf
966 * of references from different cgroups.
968 if (memcg
&& !mm_match_cgroup(vma
->vm_mm
, memcg
))
975 * folio_referenced() - Test if the folio was referenced.
976 * @folio: The folio to test.
977 * @is_locked: Caller holds lock on the folio.
978 * @memcg: target memory cgroup
979 * @vm_flags: A combination of all the vma->vm_flags which referenced the folio.
981 * Quick test_and_clear_referenced for all mappings of a folio,
983 * Return: The number of mappings which referenced the folio. Return -1 if
984 * the function bailed out due to rmap lock contention.
986 int folio_referenced(struct folio
*folio
, int is_locked
,
987 struct mem_cgroup
*memcg
, unsigned long *vm_flags
)
989 bool we_locked
= false;
990 struct folio_referenced_arg pra
= {
991 .mapcount
= folio_mapcount(folio
),
994 struct rmap_walk_control rwc
= {
995 .rmap_one
= folio_referenced_one
,
997 .anon_lock
= folio_lock_anon_vma_read
,
999 .invalid_vma
= invalid_folio_referenced_vma
,
1006 if (!folio_raw_mapping(folio
))
1009 if (!is_locked
&& (!folio_test_anon(folio
) || folio_test_ksm(folio
))) {
1010 we_locked
= folio_trylock(folio
);
1015 rmap_walk(folio
, &rwc
);
1016 *vm_flags
= pra
.vm_flags
;
1019 folio_unlock(folio
);
1021 return rwc
.contended
? -1 : pra
.referenced
;
1024 static int page_vma_mkclean_one(struct page_vma_mapped_walk
*pvmw
)
1027 struct vm_area_struct
*vma
= pvmw
->vma
;
1028 struct mmu_notifier_range range
;
1029 unsigned long address
= pvmw
->address
;
1032 * We have to assume the worse case ie pmd for invalidation. Note that
1033 * the folio can not be freed from this function.
1035 mmu_notifier_range_init(&range
, MMU_NOTIFY_PROTECTION_PAGE
, 0,
1036 vma
->vm_mm
, address
, vma_address_end(pvmw
));
1037 mmu_notifier_invalidate_range_start(&range
);
1039 while (page_vma_mapped_walk(pvmw
)) {
1042 address
= pvmw
->address
;
1044 pte_t
*pte
= pvmw
->pte
;
1045 pte_t entry
= ptep_get(pte
);
1048 * PFN swap PTEs, such as device-exclusive ones, that
1049 * actually map pages are clean and not writable from a
1050 * CPU perspective. The MMU notifier takes care of any
1053 if (!pte_present(entry
))
1055 if (!pte_dirty(entry
) && !pte_write(entry
))
1058 flush_cache_page(vma
, address
, pte_pfn(entry
));
1059 entry
= ptep_clear_flush(vma
, address
, pte
);
1060 entry
= pte_wrprotect(entry
);
1061 entry
= pte_mkclean(entry
);
1062 set_pte_at(vma
->vm_mm
, address
, pte
, entry
);
1065 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1066 pmd_t
*pmd
= pvmw
->pmd
;
1069 if (!pmd_dirty(*pmd
) && !pmd_write(*pmd
))
1072 flush_cache_range(vma
, address
,
1073 address
+ HPAGE_PMD_SIZE
);
1074 entry
= pmdp_invalidate(vma
, address
, pmd
);
1075 entry
= pmd_wrprotect(entry
);
1076 entry
= pmd_mkclean(entry
);
1077 set_pmd_at(vma
->vm_mm
, address
, pmd
, entry
);
1080 /* unexpected pmd-mapped folio? */
1089 mmu_notifier_invalidate_range_end(&range
);
1094 static bool page_mkclean_one(struct folio
*folio
, struct vm_area_struct
*vma
,
1095 unsigned long address
, void *arg
)
1097 DEFINE_FOLIO_VMA_WALK(pvmw
, folio
, vma
, address
, PVMW_SYNC
);
1100 *cleaned
+= page_vma_mkclean_one(&pvmw
);
1105 static bool invalid_mkclean_vma(struct vm_area_struct
*vma
, void *arg
)
1107 if (vma
->vm_flags
& VM_SHARED
)
1113 int folio_mkclean(struct folio
*folio
)
1116 struct address_space
*mapping
;
1117 struct rmap_walk_control rwc
= {
1118 .arg
= (void *)&cleaned
,
1119 .rmap_one
= page_mkclean_one
,
1120 .invalid_vma
= invalid_mkclean_vma
,
1123 BUG_ON(!folio_test_locked(folio
));
1125 if (!folio_mapped(folio
))
1128 mapping
= folio_mapping(folio
);
1132 rmap_walk(folio
, &rwc
);
1136 EXPORT_SYMBOL_GPL(folio_mkclean
);
1138 struct wrprotect_file_state
{
1142 unsigned long nr_pages
;
1145 static bool mapping_wrprotect_range_one(struct folio
*folio
,
1146 struct vm_area_struct
*vma
, unsigned long address
, void *arg
)
1148 struct wrprotect_file_state
*state
= (struct wrprotect_file_state
*)arg
;
1149 struct page_vma_mapped_walk pvmw
= {
1151 .nr_pages
= state
->nr_pages
,
1152 .pgoff
= state
->pgoff
,
1158 state
->cleaned
+= page_vma_mkclean_one(&pvmw
);
1163 static void __rmap_walk_file(struct folio
*folio
, struct address_space
*mapping
,
1164 pgoff_t pgoff_start
, unsigned long nr_pages
,
1165 struct rmap_walk_control
*rwc
, bool locked
);
1168 * mapping_wrprotect_range() - Write-protect all mappings in a specified range.
1170 * @mapping: The mapping whose reverse mapping should be traversed.
1171 * @pgoff: The page offset at which @pfn is mapped within @mapping.
1172 * @pfn: The PFN of the page mapped in @mapping at @pgoff.
1173 * @nr_pages: The number of physically contiguous base pages spanned.
1175 * Traverses the reverse mapping, finding all VMAs which contain a shared
1176 * mapping of the pages in the specified range in @mapping, and write-protects
1177 * them (that is, updates the page tables to mark the mappings read-only such
1178 * that a write protection fault arises when the mappings are written to).
1180 * The @pfn value need not refer to a folio, but rather can reference a kernel
1181 * allocation which is mapped into userland. We therefore do not require that
1182 * the page maps to a folio with a valid mapping or index field, rather the
1183 * caller specifies these in @mapping and @pgoff.
1185 * Return: the number of write-protected PTEs, or an error.
1187 int mapping_wrprotect_range(struct address_space
*mapping
, pgoff_t pgoff
,
1188 unsigned long pfn
, unsigned long nr_pages
)
1190 struct wrprotect_file_state state
= {
1194 .nr_pages
= nr_pages
,
1196 struct rmap_walk_control rwc
= {
1197 .arg
= (void *)&state
,
1198 .rmap_one
= mapping_wrprotect_range_one
,
1199 .invalid_vma
= invalid_mkclean_vma
,
1205 __rmap_walk_file(/* folio = */NULL
, mapping
, pgoff
, nr_pages
, &rwc
,
1206 /* locked = */false);
1208 return state
.cleaned
;
1210 EXPORT_SYMBOL_GPL(mapping_wrprotect_range
);
1213 * pfn_mkclean_range - Cleans the PTEs (including PMDs) mapped with range of
1214 * [@pfn, @pfn + @nr_pages) at the specific offset (@pgoff)
1215 * within the @vma of shared mappings. And since clean PTEs
1216 * should also be readonly, write protects them too.
1218 * @nr_pages: number of physically contiguous pages srarting with @pfn.
1219 * @pgoff: page offset that the @pfn mapped with.
1220 * @vma: vma that @pfn mapped within.
1222 * Returns the number of cleaned PTEs (including PMDs).
1224 int pfn_mkclean_range(unsigned long pfn
, unsigned long nr_pages
, pgoff_t pgoff
,
1225 struct vm_area_struct
*vma
)
1227 struct page_vma_mapped_walk pvmw
= {
1229 .nr_pages
= nr_pages
,
1235 if (invalid_mkclean_vma(vma
, NULL
))
1238 pvmw
.address
= vma_address(vma
, pgoff
, nr_pages
);
1239 VM_BUG_ON_VMA(pvmw
.address
== -EFAULT
, vma
);
1241 return page_vma_mkclean_one(&pvmw
);
1244 static __always_inline
unsigned int __folio_add_rmap(struct folio
*folio
,
1245 struct page
*page
, int nr_pages
, struct vm_area_struct
*vma
,
1246 enum rmap_level level
, int *nr_pmdmapped
)
1248 atomic_t
*mapped
= &folio
->_nr_pages_mapped
;
1249 const int orig_nr_pages
= nr_pages
;
1250 int first
= 0, nr
= 0;
1252 __folio_rmap_sanity_checks(folio
, page
, nr_pages
, level
);
1255 case RMAP_LEVEL_PTE
:
1256 if (!folio_test_large(folio
)) {
1257 nr
= atomic_inc_and_test(&folio
->_mapcount
);
1261 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT
)) {
1262 nr
= folio_add_return_large_mapcount(folio
, orig_nr_pages
, vma
);
1263 if (nr
== orig_nr_pages
)
1264 /* Was completely unmapped. */
1265 nr
= folio_large_nr_pages(folio
);
1272 first
+= atomic_inc_and_test(&page
->_mapcount
);
1273 } while (page
++, --nr_pages
> 0);
1276 atomic_add_return_relaxed(first
, mapped
) < ENTIRELY_MAPPED
)
1279 folio_add_large_mapcount(folio
, orig_nr_pages
, vma
);
1281 case RMAP_LEVEL_PMD
:
1282 case RMAP_LEVEL_PUD
:
1283 first
= atomic_inc_and_test(&folio
->_entire_mapcount
);
1284 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT
)) {
1285 if (level
== RMAP_LEVEL_PMD
&& first
)
1286 *nr_pmdmapped
= folio_large_nr_pages(folio
);
1287 nr
= folio_inc_return_large_mapcount(folio
, vma
);
1289 /* Was completely unmapped. */
1290 nr
= folio_large_nr_pages(folio
);
1297 nr
= atomic_add_return_relaxed(ENTIRELY_MAPPED
, mapped
);
1298 if (likely(nr
< ENTIRELY_MAPPED
+ ENTIRELY_MAPPED
)) {
1299 nr_pages
= folio_large_nr_pages(folio
);
1301 * We only track PMD mappings of PMD-sized
1302 * folios separately.
1304 if (level
== RMAP_LEVEL_PMD
)
1305 *nr_pmdmapped
= nr_pages
;
1306 nr
= nr_pages
- (nr
& FOLIO_PAGES_MAPPED
);
1307 /* Raced ahead of a remove and another add? */
1308 if (unlikely(nr
< 0))
1311 /* Raced ahead of a remove of ENTIRELY_MAPPED */
1315 folio_inc_large_mapcount(folio
, vma
);
1322 * folio_move_anon_rmap - move a folio to our anon_vma
1323 * @folio: The folio to move to our anon_vma
1324 * @vma: The vma the folio belongs to
1326 * When a folio belongs exclusively to one process after a COW event,
1327 * that folio can be moved into the anon_vma that belongs to just that
1328 * process, so the rmap code will not search the parent or sibling processes.
1330 void folio_move_anon_rmap(struct folio
*folio
, struct vm_area_struct
*vma
)
1332 void *anon_vma
= vma
->anon_vma
;
1334 VM_BUG_ON_FOLIO(!folio_test_locked(folio
), folio
);
1335 VM_BUG_ON_VMA(!anon_vma
, vma
);
1337 anon_vma
+= PAGE_MAPPING_ANON
;
1339 * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
1340 * simultaneously, so a concurrent reader (eg folio_referenced()'s
1341 * folio_test_anon()) will not see one without the other.
1343 WRITE_ONCE(folio
->mapping
, anon_vma
);
1347 * __folio_set_anon - set up a new anonymous rmap for a folio
1348 * @folio: The folio to set up the new anonymous rmap for.
1349 * @vma: VM area to add the folio to.
1350 * @address: User virtual address of the mapping
1351 * @exclusive: Whether the folio is exclusive to the process.
1353 static void __folio_set_anon(struct folio
*folio
, struct vm_area_struct
*vma
,
1354 unsigned long address
, bool exclusive
)
1356 struct anon_vma
*anon_vma
= vma
->anon_vma
;
1361 * If the folio isn't exclusive to this vma, we must use the _oldest_
1362 * possible anon_vma for the folio mapping!
1365 anon_vma
= anon_vma
->root
;
1368 * page_idle does a lockless/optimistic rmap scan on folio->mapping.
1369 * Make sure the compiler doesn't split the stores of anon_vma and
1370 * the PAGE_MAPPING_ANON type identifier, otherwise the rmap code
1371 * could mistake the mapping for a struct address_space and crash.
1373 anon_vma
= (void *) anon_vma
+ PAGE_MAPPING_ANON
;
1374 WRITE_ONCE(folio
->mapping
, (struct address_space
*) anon_vma
);
1375 folio
->index
= linear_page_index(vma
, address
);
1379 * __page_check_anon_rmap - sanity check anonymous rmap addition
1380 * @folio: The folio containing @page.
1381 * @page: the page to check the mapping of
1382 * @vma: the vm area in which the mapping is added
1383 * @address: the user virtual address mapped
1385 static void __page_check_anon_rmap(const struct folio
*folio
,
1386 const struct page
*page
, struct vm_area_struct
*vma
,
1387 unsigned long address
)
1390 * The page's anon-rmap details (mapping and index) are guaranteed to
1391 * be set up correctly at this point.
1393 * We have exclusion against folio_add_anon_rmap_*() because the caller
1394 * always holds the page locked.
1396 * We have exclusion against folio_add_new_anon_rmap because those pages
1397 * are initially only visible via the pagetables, and the pte is locked
1398 * over the call to folio_add_new_anon_rmap.
1400 VM_BUG_ON_FOLIO(folio_anon_vma(folio
)->root
!= vma
->anon_vma
->root
,
1402 VM_BUG_ON_PAGE(page_pgoff(folio
, page
) != linear_page_index(vma
, address
),
1406 static void __folio_mod_stat(struct folio
*folio
, int nr
, int nr_pmdmapped
)
1411 idx
= folio_test_anon(folio
) ? NR_ANON_MAPPED
: NR_FILE_MAPPED
;
1412 __lruvec_stat_mod_folio(folio
, idx
, nr
);
1415 if (folio_test_anon(folio
)) {
1417 __lruvec_stat_mod_folio(folio
, idx
, nr_pmdmapped
);
1419 /* NR_*_PMDMAPPED are not maintained per-memcg */
1420 idx
= folio_test_swapbacked(folio
) ?
1421 NR_SHMEM_PMDMAPPED
: NR_FILE_PMDMAPPED
;
1422 __mod_node_page_state(folio_pgdat(folio
), idx
,
1428 static __always_inline
void __folio_add_anon_rmap(struct folio
*folio
,
1429 struct page
*page
, int nr_pages
, struct vm_area_struct
*vma
,
1430 unsigned long address
, rmap_t flags
, enum rmap_level level
)
1432 int i
, nr
, nr_pmdmapped
= 0;
1434 VM_WARN_ON_FOLIO(!folio_test_anon(folio
), folio
);
1436 nr
= __folio_add_rmap(folio
, page
, nr_pages
, vma
, level
, &nr_pmdmapped
);
1438 if (likely(!folio_test_ksm(folio
)))
1439 __page_check_anon_rmap(folio
, page
, vma
, address
);
1441 __folio_mod_stat(folio
, nr
, nr_pmdmapped
);
1443 if (flags
& RMAP_EXCLUSIVE
) {
1445 case RMAP_LEVEL_PTE
:
1446 for (i
= 0; i
< nr_pages
; i
++)
1447 SetPageAnonExclusive(page
+ i
);
1449 case RMAP_LEVEL_PMD
:
1450 SetPageAnonExclusive(page
);
1452 case RMAP_LEVEL_PUD
:
1454 * Keep the compiler happy, we don't support anonymous
1462 VM_WARN_ON_FOLIO(!folio_test_large(folio
) && PageAnonExclusive(page
) &&
1463 atomic_read(&folio
->_mapcount
) > 0, folio
);
1464 for (i
= 0; i
< nr_pages
; i
++) {
1465 struct page
*cur_page
= page
+ i
;
1467 VM_WARN_ON_FOLIO(folio_test_large(folio
) &&
1468 folio_entire_mapcount(folio
) > 1 &&
1469 PageAnonExclusive(cur_page
), folio
);
1470 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT
))
1474 * While PTE-mapping a THP we have a PMD and a PTE
1477 VM_WARN_ON_FOLIO(atomic_read(&cur_page
->_mapcount
) > 0 &&
1478 PageAnonExclusive(cur_page
), folio
);
1482 * For large folio, only mlock it if it's fully mapped to VMA. It's
1483 * not easy to check whether the large folio is fully mapped to VMA
1484 * here. Only mlock normal 4K folio and leave page reclaim to handle
1487 if (!folio_test_large(folio
))
1488 mlock_vma_folio(folio
, vma
);
1492 * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
1493 * @folio: The folio to add the mappings to
1494 * @page: The first page to add
1495 * @nr_pages: The number of pages which will be mapped
1496 * @vma: The vm area in which the mappings are added
1497 * @address: The user virtual address of the first page to map
1498 * @flags: The rmap flags
1500 * The page range of folio is defined by [first_page, first_page + nr_pages)
1502 * The caller needs to hold the page table lock, and the page must be locked in
1503 * the anon_vma case: to serialize mapping,index checking after setting,
1504 * and to ensure that an anon folio is not being upgraded racily to a KSM folio
1505 * (but KSM folios are never downgraded).
1507 void folio_add_anon_rmap_ptes(struct folio
*folio
, struct page
*page
,
1508 int nr_pages
, struct vm_area_struct
*vma
, unsigned long address
,
1511 __folio_add_anon_rmap(folio
, page
, nr_pages
, vma
, address
, flags
,
1516 * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
1517 * @folio: The folio to add the mapping to
1518 * @page: The first page to add
1519 * @vma: The vm area in which the mapping is added
1520 * @address: The user virtual address of the first page to map
1521 * @flags: The rmap flags
1523 * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
1525 * The caller needs to hold the page table lock, and the page must be locked in
1526 * the anon_vma case: to serialize mapping,index checking after setting.
1528 void folio_add_anon_rmap_pmd(struct folio
*folio
, struct page
*page
,
1529 struct vm_area_struct
*vma
, unsigned long address
, rmap_t flags
)
1531 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1532 __folio_add_anon_rmap(folio
, page
, HPAGE_PMD_NR
, vma
, address
, flags
,
1540 * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
1541 * @folio: The folio to add the mapping to.
1542 * @vma: the vm area in which the mapping is added
1543 * @address: the user virtual address mapped
1544 * @flags: The rmap flags
1546 * Like folio_add_anon_rmap_*() but must only be called on *new* folios.
1547 * This means the inc-and-test can be bypassed.
1548 * The folio doesn't necessarily need to be locked while it's exclusive
1549 * unless two threads map it concurrently. However, the folio must be
1550 * locked if it's shared.
1552 * If the folio is pmd-mappable, it is accounted as a THP.
1554 void folio_add_new_anon_rmap(struct folio
*folio
, struct vm_area_struct
*vma
,
1555 unsigned long address
, rmap_t flags
)
1557 const bool exclusive
= flags
& RMAP_EXCLUSIVE
;
1558 int nr
= 1, nr_pmdmapped
= 0;
1560 VM_WARN_ON_FOLIO(folio_test_hugetlb(folio
), folio
);
1561 VM_WARN_ON_FOLIO(!exclusive
&& !folio_test_locked(folio
), folio
);
1564 * VM_DROPPABLE mappings don't swap; instead they're just dropped when
1565 * under memory pressure.
1567 if (!folio_test_swapbacked(folio
) && !(vma
->vm_flags
& VM_DROPPABLE
))
1568 __folio_set_swapbacked(folio
);
1569 __folio_set_anon(folio
, vma
, address
, exclusive
);
1571 if (likely(!folio_test_large(folio
))) {
1572 /* increment count (starts at -1) */
1573 atomic_set(&folio
->_mapcount
, 0);
1575 SetPageAnonExclusive(&folio
->page
);
1576 } else if (!folio_test_pmd_mappable(folio
)) {
1579 nr
= folio_large_nr_pages(folio
);
1580 for (i
= 0; i
< nr
; i
++) {
1581 struct page
*page
= folio_page(folio
, i
);
1583 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT
))
1584 /* increment count (starts at -1) */
1585 atomic_set(&page
->_mapcount
, 0);
1587 SetPageAnonExclusive(page
);
1590 folio_set_large_mapcount(folio
, nr
, vma
);
1591 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT
))
1592 atomic_set(&folio
->_nr_pages_mapped
, nr
);
1594 nr
= folio_large_nr_pages(folio
);
1595 /* increment count (starts at -1) */
1596 atomic_set(&folio
->_entire_mapcount
, 0);
1597 folio_set_large_mapcount(folio
, 1, vma
);
1598 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT
))
1599 atomic_set(&folio
->_nr_pages_mapped
, ENTIRELY_MAPPED
);
1601 SetPageAnonExclusive(&folio
->page
);
1605 VM_WARN_ON_ONCE(address
< vma
->vm_start
||
1606 address
+ (nr
<< PAGE_SHIFT
) > vma
->vm_end
);
1608 __folio_mod_stat(folio
, nr
, nr_pmdmapped
);
1609 mod_mthp_stat(folio_order(folio
), MTHP_STAT_NR_ANON
, 1);
1612 static __always_inline
void __folio_add_file_rmap(struct folio
*folio
,
1613 struct page
*page
, int nr_pages
, struct vm_area_struct
*vma
,
1614 enum rmap_level level
)
1616 int nr
, nr_pmdmapped
= 0;
1618 VM_WARN_ON_FOLIO(folio_test_anon(folio
), folio
);
1620 nr
= __folio_add_rmap(folio
, page
, nr_pages
, vma
, level
, &nr_pmdmapped
);
1621 __folio_mod_stat(folio
, nr
, nr_pmdmapped
);
1623 /* See comments in folio_add_anon_rmap_*() */
1624 if (!folio_test_large(folio
))
1625 mlock_vma_folio(folio
, vma
);
1629 * folio_add_file_rmap_ptes - add PTE mappings to a page range of a folio
1630 * @folio: The folio to add the mappings to
1631 * @page: The first page to add
1632 * @nr_pages: The number of pages that will be mapped using PTEs
1633 * @vma: The vm area in which the mappings are added
1635 * The page range of the folio is defined by [page, page + nr_pages)
1637 * The caller needs to hold the page table lock.
1639 void folio_add_file_rmap_ptes(struct folio
*folio
, struct page
*page
,
1640 int nr_pages
, struct vm_area_struct
*vma
)
1642 __folio_add_file_rmap(folio
, page
, nr_pages
, vma
, RMAP_LEVEL_PTE
);
1646 * folio_add_file_rmap_pmd - add a PMD mapping to a page range of a folio
1647 * @folio: The folio to add the mapping to
1648 * @page: The first page to add
1649 * @vma: The vm area in which the mapping is added
1651 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
1653 * The caller needs to hold the page table lock.
1655 void folio_add_file_rmap_pmd(struct folio
*folio
, struct page
*page
,
1656 struct vm_area_struct
*vma
)
1658 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1659 __folio_add_file_rmap(folio
, page
, HPAGE_PMD_NR
, vma
, RMAP_LEVEL_PMD
);
1666 * folio_add_file_rmap_pud - add a PUD mapping to a page range of a folio
1667 * @folio: The folio to add the mapping to
1668 * @page: The first page to add
1669 * @vma: The vm area in which the mapping is added
1671 * The page range of the folio is defined by [page, page + HPAGE_PUD_NR)
1673 * The caller needs to hold the page table lock.
1675 void folio_add_file_rmap_pud(struct folio
*folio
, struct page
*page
,
1676 struct vm_area_struct
*vma
)
1678 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1679 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1680 __folio_add_file_rmap(folio
, page
, HPAGE_PUD_NR
, vma
, RMAP_LEVEL_PUD
);
1686 static __always_inline
void __folio_remove_rmap(struct folio
*folio
,
1687 struct page
*page
, int nr_pages
, struct vm_area_struct
*vma
,
1688 enum rmap_level level
)
1690 atomic_t
*mapped
= &folio
->_nr_pages_mapped
;
1691 int last
= 0, nr
= 0, nr_pmdmapped
= 0;
1692 bool partially_mapped
= false;
1694 __folio_rmap_sanity_checks(folio
, page
, nr_pages
, level
);
1697 case RMAP_LEVEL_PTE
:
1698 if (!folio_test_large(folio
)) {
1699 nr
= atomic_add_negative(-1, &folio
->_mapcount
);
1703 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT
)) {
1704 nr
= folio_sub_return_large_mapcount(folio
, nr_pages
, vma
);
1706 /* Now completely unmapped. */
1707 nr
= folio_nr_pages(folio
);
1709 partially_mapped
= nr
< folio_large_nr_pages(folio
) &&
1710 !folio_entire_mapcount(folio
);
1716 folio_sub_large_mapcount(folio
, nr_pages
, vma
);
1718 last
+= atomic_add_negative(-1, &page
->_mapcount
);
1719 } while (page
++, --nr_pages
> 0);
1722 atomic_sub_return_relaxed(last
, mapped
) < ENTIRELY_MAPPED
)
1725 partially_mapped
= nr
&& atomic_read(mapped
);
1727 case RMAP_LEVEL_PMD
:
1728 case RMAP_LEVEL_PUD
:
1729 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT
)) {
1730 last
= atomic_add_negative(-1, &folio
->_entire_mapcount
);
1731 if (level
== RMAP_LEVEL_PMD
&& last
)
1732 nr_pmdmapped
= folio_large_nr_pages(folio
);
1733 nr
= folio_dec_return_large_mapcount(folio
, vma
);
1735 /* Now completely unmapped. */
1736 nr
= folio_large_nr_pages(folio
);
1738 partially_mapped
= last
&&
1739 nr
< folio_large_nr_pages(folio
);
1745 folio_dec_large_mapcount(folio
, vma
);
1746 last
= atomic_add_negative(-1, &folio
->_entire_mapcount
);
1748 nr
= atomic_sub_return_relaxed(ENTIRELY_MAPPED
, mapped
);
1749 if (likely(nr
< ENTIRELY_MAPPED
)) {
1750 nr_pages
= folio_large_nr_pages(folio
);
1751 if (level
== RMAP_LEVEL_PMD
)
1752 nr_pmdmapped
= nr_pages
;
1753 nr
= nr_pages
- (nr
& FOLIO_PAGES_MAPPED
);
1754 /* Raced ahead of another remove and an add? */
1755 if (unlikely(nr
< 0))
1758 /* An add of ENTIRELY_MAPPED raced ahead */
1763 partially_mapped
= nr
&& nr
< nr_pmdmapped
;
1768 * Queue anon large folio for deferred split if at least one page of
1769 * the folio is unmapped and at least one page is still mapped.
1771 * Check partially_mapped first to ensure it is a large folio.
1773 if (partially_mapped
&& folio_test_anon(folio
) &&
1774 !folio_test_partially_mapped(folio
))
1775 deferred_split_folio(folio
, true);
1777 __folio_mod_stat(folio
, -nr
, -nr_pmdmapped
);
1780 * It would be tidy to reset folio_test_anon mapping when fully
1781 * unmapped, but that might overwrite a racing folio_add_anon_rmap_*()
1782 * which increments mapcount after us but sets mapping before us:
1783 * so leave the reset to free_pages_prepare, and remember that
1784 * it's only reliable while mapped.
1787 munlock_vma_folio(folio
, vma
);
1791 * folio_remove_rmap_ptes - remove PTE mappings from a page range of a folio
1792 * @folio: The folio to remove the mappings from
1793 * @page: The first page to remove
1794 * @nr_pages: The number of pages that will be removed from the mapping
1795 * @vma: The vm area from which the mappings are removed
1797 * The page range of the folio is defined by [page, page + nr_pages)
1799 * The caller needs to hold the page table lock.
1801 void folio_remove_rmap_ptes(struct folio
*folio
, struct page
*page
,
1802 int nr_pages
, struct vm_area_struct
*vma
)
1804 __folio_remove_rmap(folio
, page
, nr_pages
, vma
, RMAP_LEVEL_PTE
);
1808 * folio_remove_rmap_pmd - remove a PMD mapping from a page range of a folio
1809 * @folio: The folio to remove the mapping from
1810 * @page: The first page to remove
1811 * @vma: The vm area from which the mapping is removed
1813 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
1815 * The caller needs to hold the page table lock.
1817 void folio_remove_rmap_pmd(struct folio
*folio
, struct page
*page
,
1818 struct vm_area_struct
*vma
)
1820 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1821 __folio_remove_rmap(folio
, page
, HPAGE_PMD_NR
, vma
, RMAP_LEVEL_PMD
);
1828 * folio_remove_rmap_pud - remove a PUD mapping from a page range of a folio
1829 * @folio: The folio to remove the mapping from
1830 * @page: The first page to remove
1831 * @vma: The vm area from which the mapping is removed
1833 * The page range of the folio is defined by [page, page + HPAGE_PUD_NR)
1835 * The caller needs to hold the page table lock.
1837 void folio_remove_rmap_pud(struct folio
*folio
, struct page
*page
,
1838 struct vm_area_struct
*vma
)
1840 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1841 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1842 __folio_remove_rmap(folio
, page
, HPAGE_PUD_NR
, vma
, RMAP_LEVEL_PUD
);
1848 /* We support batch unmapping of PTEs for lazyfree large folios */
1849 static inline bool can_batch_unmap_folio_ptes(unsigned long addr
,
1850 struct folio
*folio
, pte_t
*ptep
)
1852 const fpb_t fpb_flags
= FPB_IGNORE_DIRTY
| FPB_IGNORE_SOFT_DIRTY
;
1853 int max_nr
= folio_nr_pages(folio
);
1854 pte_t pte
= ptep_get(ptep
);
1856 if (!folio_test_anon(folio
) || folio_test_swapbacked(folio
))
1858 if (pte_unused(pte
))
1860 if (pte_pfn(pte
) != folio_pfn(folio
))
1863 return folio_pte_batch(folio
, addr
, ptep
, pte
, max_nr
, fpb_flags
, NULL
,
1864 NULL
, NULL
) == max_nr
;
1868 * @arg: enum ttu_flags will be passed to this argument
1870 static bool try_to_unmap_one(struct folio
*folio
, struct vm_area_struct
*vma
,
1871 unsigned long address
, void *arg
)
1873 struct mm_struct
*mm
= vma
->vm_mm
;
1874 DEFINE_FOLIO_VMA_WALK(pvmw
, folio
, vma
, address
, 0);
1875 bool anon_exclusive
, ret
= true;
1877 struct page
*subpage
;
1878 struct mmu_notifier_range range
;
1879 enum ttu_flags flags
= (enum ttu_flags
)(long)arg
;
1880 unsigned long nr_pages
= 1, end_addr
;
1882 unsigned long hsz
= 0;
1885 * When racing against e.g. zap_pte_range() on another cpu,
1886 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
1887 * try_to_unmap() may return before page_mapped() has become false,
1888 * if page table locking is skipped: use TTU_SYNC to wait for that.
1890 if (flags
& TTU_SYNC
)
1891 pvmw
.flags
= PVMW_SYNC
;
1894 * For THP, we have to assume the worse case ie pmd for invalidation.
1895 * For hugetlb, it could be much worse if we need to do pud
1896 * invalidation in the case of pmd sharing.
1898 * Note that the folio can not be freed in this function as call of
1899 * try_to_unmap() must hold a reference on the folio.
1901 range
.end
= vma_address_end(&pvmw
);
1902 mmu_notifier_range_init(&range
, MMU_NOTIFY_CLEAR
, 0, vma
->vm_mm
,
1903 address
, range
.end
);
1904 if (folio_test_hugetlb(folio
)) {
1906 * If sharing is possible, start and end will be adjusted
1909 adjust_range_if_pmd_sharing_possible(vma
, &range
.start
,
1912 /* We need the huge page size for set_huge_pte_at() */
1913 hsz
= huge_page_size(hstate_vma(vma
));
1915 mmu_notifier_invalidate_range_start(&range
);
1917 while (page_vma_mapped_walk(&pvmw
)) {
1919 * If the folio is in an mlock()d vma, we must not swap it out.
1921 if (!(flags
& TTU_IGNORE_MLOCK
) &&
1922 (vma
->vm_flags
& VM_LOCKED
)) {
1923 /* Restore the mlock which got missed */
1924 if (!folio_test_large(folio
))
1925 mlock_vma_folio(folio
, vma
);
1930 if (folio_test_anon(folio
) && !folio_test_swapbacked(folio
)) {
1931 if (unmap_huge_pmd_locked(vma
, pvmw
.address
, pvmw
.pmd
, folio
))
1934 * unmap_huge_pmd_locked has either already marked
1935 * the folio as swap-backed or decided to retain it
1936 * due to GUP or speculative references.
1941 if (flags
& TTU_SPLIT_HUGE_PMD
) {
1943 * We temporarily have to drop the PTL and
1944 * restart so we can process the PTE-mapped THP.
1946 split_huge_pmd_locked(vma
, pvmw
.address
,
1948 flags
&= ~TTU_SPLIT_HUGE_PMD
;
1949 page_vma_mapped_walk_restart(&pvmw
);
1954 /* Unexpected PMD-mapped THP? */
1955 VM_BUG_ON_FOLIO(!pvmw
.pte
, folio
);
1958 * Handle PFN swap PTEs, such as device-exclusive ones, that
1959 * actually map pages.
1961 pteval
= ptep_get(pvmw
.pte
);
1962 if (likely(pte_present(pteval
))) {
1963 pfn
= pte_pfn(pteval
);
1965 pfn
= swp_offset_pfn(pte_to_swp_entry(pteval
));
1966 VM_WARN_ON_FOLIO(folio_test_hugetlb(folio
), folio
);
1969 subpage
= folio_page(folio
, pfn
- folio_pfn(folio
));
1970 address
= pvmw
.address
;
1971 anon_exclusive
= folio_test_anon(folio
) &&
1972 PageAnonExclusive(subpage
);
1974 if (folio_test_hugetlb(folio
)) {
1975 bool anon
= folio_test_anon(folio
);
1978 * The try_to_unmap() is only passed a hugetlb page
1979 * in the case where the hugetlb page is poisoned.
1981 VM_BUG_ON_PAGE(!PageHWPoison(subpage
), subpage
);
1983 * huge_pmd_unshare may unmap an entire PMD page.
1984 * There is no way of knowing exactly which PMDs may
1985 * be cached for this mm, so we must flush them all.
1986 * start/end were already adjusted above to cover this
1989 flush_cache_range(vma
, range
.start
, range
.end
);
1992 * To call huge_pmd_unshare, i_mmap_rwsem must be
1993 * held in write mode. Caller needs to explicitly
1994 * do this outside rmap routines.
1996 * We also must hold hugetlb vma_lock in write mode.
1997 * Lock order dictates acquiring vma_lock BEFORE
1998 * i_mmap_rwsem. We can only try lock here and fail
2002 VM_BUG_ON(!(flags
& TTU_RMAP_LOCKED
));
2003 if (!hugetlb_vma_trylock_write(vma
))
2005 if (huge_pmd_unshare(mm
, vma
, address
, pvmw
.pte
)) {
2006 hugetlb_vma_unlock_write(vma
);
2007 flush_tlb_range(vma
,
2008 range
.start
, range
.end
);
2010 * The ref count of the PMD page was
2011 * dropped which is part of the way map
2012 * counting is done for shared PMDs.
2013 * Return 'true' here. When there is
2014 * no other sharing, huge_pmd_unshare
2015 * returns false and we will unmap the
2016 * actual page and drop map count
2021 hugetlb_vma_unlock_write(vma
);
2023 pteval
= huge_ptep_clear_flush(vma
, address
, pvmw
.pte
);
2024 if (pte_dirty(pteval
))
2025 folio_mark_dirty(folio
);
2026 } else if (likely(pte_present(pteval
))) {
2027 if (folio_test_large(folio
) && !(flags
& TTU_HWPOISON
) &&
2028 can_batch_unmap_folio_ptes(address
, folio
, pvmw
.pte
))
2029 nr_pages
= folio_nr_pages(folio
);
2030 end_addr
= address
+ nr_pages
* PAGE_SIZE
;
2031 flush_cache_range(vma
, address
, end_addr
);
2033 /* Nuke the page table entry. */
2034 pteval
= get_and_clear_full_ptes(mm
, address
, pvmw
.pte
, nr_pages
, 0);
2036 * We clear the PTE but do not flush so potentially
2037 * a remote CPU could still be writing to the folio.
2038 * If the entry was previously clean then the
2039 * architecture must guarantee that a clear->dirty
2040 * transition on a cached TLB entry is written through
2041 * and traps if the PTE is unmapped.
2043 if (should_defer_flush(mm
, flags
))
2044 set_tlb_ubc_flush_pending(mm
, pteval
, address
, end_addr
);
2046 flush_tlb_range(vma
, address
, end_addr
);
2047 if (pte_dirty(pteval
))
2048 folio_mark_dirty(folio
);
2050 pte_clear(mm
, address
, pvmw
.pte
);
2054 * Now the pte is cleared. If this pte was uffd-wp armed,
2055 * we may want to replace a none pte with a marker pte if
2056 * it's file-backed, so we don't lose the tracking info.
2058 pte_install_uffd_wp_if_needed(vma
, address
, pvmw
.pte
, pteval
);
2060 /* Update high watermark before we lower rss */
2061 update_hiwater_rss(mm
);
2063 if (PageHWPoison(subpage
) && (flags
& TTU_HWPOISON
)) {
2064 pteval
= swp_entry_to_pte(make_hwpoison_entry(subpage
));
2065 if (folio_test_hugetlb(folio
)) {
2066 hugetlb_count_sub(folio_nr_pages(folio
), mm
);
2067 set_huge_pte_at(mm
, address
, pvmw
.pte
, pteval
,
2070 dec_mm_counter(mm
, mm_counter(folio
));
2071 set_pte_at(mm
, address
, pvmw
.pte
, pteval
);
2073 } else if (likely(pte_present(pteval
)) && pte_unused(pteval
) &&
2074 !userfaultfd_armed(vma
)) {
2076 * The guest indicated that the page content is of no
2077 * interest anymore. Simply discard the pte, vmscan
2078 * will take care of the rest.
2079 * A future reference will then fault in a new zero
2080 * page. When userfaultfd is active, we must not drop
2081 * this page though, as its main user (postcopy
2082 * migration) will not expect userfaults on already
2085 dec_mm_counter(mm
, mm_counter(folio
));
2086 } else if (folio_test_anon(folio
)) {
2087 swp_entry_t entry
= page_swap_entry(subpage
);
2090 * Store the swap location in the pte.
2091 * See handle_pte_fault() ...
2093 if (unlikely(folio_test_swapbacked(folio
) !=
2094 folio_test_swapcache(folio
))) {
2099 /* MADV_FREE page check */
2100 if (!folio_test_swapbacked(folio
)) {
2101 int ref_count
, map_count
;
2104 * Synchronize with gup_pte_range():
2105 * - clear PTE; barrier; read refcount
2106 * - inc refcount; barrier; read PTE
2110 ref_count
= folio_ref_count(folio
);
2111 map_count
= folio_mapcount(folio
);
2114 * Order reads for page refcount and dirty flag
2115 * (see comments in __remove_mapping()).
2119 if (folio_test_dirty(folio
) && !(vma
->vm_flags
& VM_DROPPABLE
)) {
2121 * redirtied either using the page table or a previously
2122 * obtained GUP reference.
2124 set_ptes(mm
, address
, pvmw
.pte
, pteval
, nr_pages
);
2125 folio_set_swapbacked(folio
);
2127 } else if (ref_count
!= 1 + map_count
) {
2129 * Additional reference. Could be a GUP reference or any
2130 * speculative reference. GUP users must mark the folio
2131 * dirty if there was a modification. This folio cannot be
2132 * reclaimed right now either way, so act just like nothing
2134 * We'll come back here later and detect if the folio was
2135 * dirtied when the additional reference is gone.
2137 set_ptes(mm
, address
, pvmw
.pte
, pteval
, nr_pages
);
2140 add_mm_counter(mm
, MM_ANONPAGES
, -nr_pages
);
2144 if (swap_duplicate(entry
) < 0) {
2145 set_pte_at(mm
, address
, pvmw
.pte
, pteval
);
2150 * arch_unmap_one() is expected to be a NOP on
2151 * architectures where we could have PFN swap PTEs,
2152 * so we'll not check/care.
2154 if (arch_unmap_one(mm
, vma
, address
, pteval
) < 0) {
2156 set_pte_at(mm
, address
, pvmw
.pte
, pteval
);
2160 /* See folio_try_share_anon_rmap(): clear PTE first. */
2161 if (anon_exclusive
&&
2162 folio_try_share_anon_rmap_pte(folio
, subpage
)) {
2164 set_pte_at(mm
, address
, pvmw
.pte
, pteval
);
2167 if (list_empty(&mm
->mmlist
)) {
2168 spin_lock(&mmlist_lock
);
2169 if (list_empty(&mm
->mmlist
))
2170 list_add(&mm
->mmlist
, &init_mm
.mmlist
);
2171 spin_unlock(&mmlist_lock
);
2173 dec_mm_counter(mm
, MM_ANONPAGES
);
2174 inc_mm_counter(mm
, MM_SWAPENTS
);
2175 swp_pte
= swp_entry_to_pte(entry
);
2177 swp_pte
= pte_swp_mkexclusive(swp_pte
);
2178 if (likely(pte_present(pteval
))) {
2179 if (pte_soft_dirty(pteval
))
2180 swp_pte
= pte_swp_mksoft_dirty(swp_pte
);
2181 if (pte_uffd_wp(pteval
))
2182 swp_pte
= pte_swp_mkuffd_wp(swp_pte
);
2184 if (pte_swp_soft_dirty(pteval
))
2185 swp_pte
= pte_swp_mksoft_dirty(swp_pte
);
2186 if (pte_swp_uffd_wp(pteval
))
2187 swp_pte
= pte_swp_mkuffd_wp(swp_pte
);
2189 set_pte_at(mm
, address
, pvmw
.pte
, swp_pte
);
2192 * This is a locked file-backed folio,
2193 * so it cannot be removed from the page
2194 * cache and replaced by a new folio before
2195 * mmu_notifier_invalidate_range_end, so no
2196 * concurrent thread might update its page table
2197 * to point at a new folio while a device is
2198 * still using this folio.
2200 * See Documentation/mm/mmu_notifier.rst
2202 dec_mm_counter(mm
, mm_counter_file(folio
));
2205 if (unlikely(folio_test_hugetlb(folio
))) {
2206 hugetlb_remove_rmap(folio
);
2208 folio_remove_rmap_ptes(folio
, subpage
, nr_pages
, vma
);
2209 folio_ref_sub(folio
, nr_pages
- 1);
2211 if (vma
->vm_flags
& VM_LOCKED
)
2212 mlock_drain_local();
2214 /* We have already batched the entire folio */
2221 page_vma_mapped_walk_done(&pvmw
);
2225 mmu_notifier_invalidate_range_end(&range
);
2230 static bool invalid_migration_vma(struct vm_area_struct
*vma
, void *arg
)
2232 return vma_is_temporary_stack(vma
);
2235 static int folio_not_mapped(struct folio
*folio
)
2237 return !folio_mapped(folio
);
2241 * try_to_unmap - Try to remove all page table mappings to a folio.
2242 * @folio: The folio to unmap.
2243 * @flags: action and flags
2245 * Tries to remove all the page table entries which are mapping this
2246 * folio. It is the caller's responsibility to check if the folio is
2247 * still mapped if needed (use TTU_SYNC to prevent accounting races).
2249 * Context: Caller must hold the folio lock.
2251 void try_to_unmap(struct folio
*folio
, enum ttu_flags flags
)
2253 struct rmap_walk_control rwc
= {
2254 .rmap_one
= try_to_unmap_one
,
2255 .arg
= (void *)flags
,
2256 .done
= folio_not_mapped
,
2257 .anon_lock
= folio_lock_anon_vma_read
,
2260 if (flags
& TTU_RMAP_LOCKED
)
2261 rmap_walk_locked(folio
, &rwc
);
2263 rmap_walk(folio
, &rwc
);
2267 * @arg: enum ttu_flags will be passed to this argument.
2269 * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
2270 * containing migration entries.
2272 static bool try_to_migrate_one(struct folio
*folio
, struct vm_area_struct
*vma
,
2273 unsigned long address
, void *arg
)
2275 struct mm_struct
*mm
= vma
->vm_mm
;
2276 DEFINE_FOLIO_VMA_WALK(pvmw
, folio
, vma
, address
, 0);
2277 bool anon_exclusive
, writable
, ret
= true;
2279 struct page
*subpage
;
2280 struct mmu_notifier_range range
;
2281 enum ttu_flags flags
= (enum ttu_flags
)(long)arg
;
2283 unsigned long hsz
= 0;
2286 * When racing against e.g. zap_pte_range() on another cpu,
2287 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
2288 * try_to_migrate() may return before page_mapped() has become false,
2289 * if page table locking is skipped: use TTU_SYNC to wait for that.
2291 if (flags
& TTU_SYNC
)
2292 pvmw
.flags
= PVMW_SYNC
;
2295 * For THP, we have to assume the worse case ie pmd for invalidation.
2296 * For hugetlb, it could be much worse if we need to do pud
2297 * invalidation in the case of pmd sharing.
2299 * Note that the page can not be free in this function as call of
2300 * try_to_unmap() must hold a reference on the page.
2302 range
.end
= vma_address_end(&pvmw
);
2303 mmu_notifier_range_init(&range
, MMU_NOTIFY_CLEAR
, 0, vma
->vm_mm
,
2304 address
, range
.end
);
2305 if (folio_test_hugetlb(folio
)) {
2307 * If sharing is possible, start and end will be adjusted
2310 adjust_range_if_pmd_sharing_possible(vma
, &range
.start
,
2313 /* We need the huge page size for set_huge_pte_at() */
2314 hsz
= huge_page_size(hstate_vma(vma
));
2316 mmu_notifier_invalidate_range_start(&range
);
2318 while (page_vma_mapped_walk(&pvmw
)) {
2319 /* PMD-mapped THP migration entry */
2321 if (flags
& TTU_SPLIT_HUGE_PMD
) {
2322 split_huge_pmd_locked(vma
, pvmw
.address
,
2325 page_vma_mapped_walk_done(&pvmw
);
2328 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
2329 subpage
= folio_page(folio
,
2330 pmd_pfn(*pvmw
.pmd
) - folio_pfn(folio
));
2331 VM_BUG_ON_FOLIO(folio_test_hugetlb(folio
) ||
2332 !folio_test_pmd_mappable(folio
), folio
);
2334 if (set_pmd_migration_entry(&pvmw
, subpage
)) {
2336 page_vma_mapped_walk_done(&pvmw
);
2343 /* Unexpected PMD-mapped THP? */
2344 VM_BUG_ON_FOLIO(!pvmw
.pte
, folio
);
2347 * Handle PFN swap PTEs, such as device-exclusive ones, that
2348 * actually map pages.
2350 pteval
= ptep_get(pvmw
.pte
);
2351 if (likely(pte_present(pteval
))) {
2352 pfn
= pte_pfn(pteval
);
2354 pfn
= swp_offset_pfn(pte_to_swp_entry(pteval
));
2355 VM_WARN_ON_FOLIO(folio_test_hugetlb(folio
), folio
);
2358 subpage
= folio_page(folio
, pfn
- folio_pfn(folio
));
2359 address
= pvmw
.address
;
2360 anon_exclusive
= folio_test_anon(folio
) &&
2361 PageAnonExclusive(subpage
);
2363 if (folio_test_hugetlb(folio
)) {
2364 bool anon
= folio_test_anon(folio
);
2367 * huge_pmd_unshare may unmap an entire PMD page.
2368 * There is no way of knowing exactly which PMDs may
2369 * be cached for this mm, so we must flush them all.
2370 * start/end were already adjusted above to cover this
2373 flush_cache_range(vma
, range
.start
, range
.end
);
2376 * To call huge_pmd_unshare, i_mmap_rwsem must be
2377 * held in write mode. Caller needs to explicitly
2378 * do this outside rmap routines.
2380 * We also must hold hugetlb vma_lock in write mode.
2381 * Lock order dictates acquiring vma_lock BEFORE
2382 * i_mmap_rwsem. We can only try lock here and
2383 * fail if unsuccessful.
2386 VM_BUG_ON(!(flags
& TTU_RMAP_LOCKED
));
2387 if (!hugetlb_vma_trylock_write(vma
)) {
2388 page_vma_mapped_walk_done(&pvmw
);
2392 if (huge_pmd_unshare(mm
, vma
, address
, pvmw
.pte
)) {
2393 hugetlb_vma_unlock_write(vma
);
2394 flush_tlb_range(vma
,
2395 range
.start
, range
.end
);
2398 * The ref count of the PMD page was
2399 * dropped which is part of the way map
2400 * counting is done for shared PMDs.
2401 * Return 'true' here. When there is
2402 * no other sharing, huge_pmd_unshare
2403 * returns false and we will unmap the
2404 * actual page and drop map count
2407 page_vma_mapped_walk_done(&pvmw
);
2410 hugetlb_vma_unlock_write(vma
);
2412 /* Nuke the hugetlb page table entry */
2413 pteval
= huge_ptep_clear_flush(vma
, address
, pvmw
.pte
);
2414 if (pte_dirty(pteval
))
2415 folio_mark_dirty(folio
);
2416 writable
= pte_write(pteval
);
2417 } else if (likely(pte_present(pteval
))) {
2418 flush_cache_page(vma
, address
, pfn
);
2419 /* Nuke the page table entry. */
2420 if (should_defer_flush(mm
, flags
)) {
2422 * We clear the PTE but do not flush so potentially
2423 * a remote CPU could still be writing to the folio.
2424 * If the entry was previously clean then the
2425 * architecture must guarantee that a clear->dirty
2426 * transition on a cached TLB entry is written through
2427 * and traps if the PTE is unmapped.
2429 pteval
= ptep_get_and_clear(mm
, address
, pvmw
.pte
);
2431 set_tlb_ubc_flush_pending(mm
, pteval
, address
, address
+ PAGE_SIZE
);
2433 pteval
= ptep_clear_flush(vma
, address
, pvmw
.pte
);
2435 if (pte_dirty(pteval
))
2436 folio_mark_dirty(folio
);
2437 writable
= pte_write(pteval
);
2439 pte_clear(mm
, address
, pvmw
.pte
);
2440 writable
= is_writable_device_private_entry(pte_to_swp_entry(pteval
));
2443 VM_WARN_ON_FOLIO(writable
&& folio_test_anon(folio
) &&
2444 !anon_exclusive
, folio
);
2446 /* Update high watermark before we lower rss */
2447 update_hiwater_rss(mm
);
2449 if (PageHWPoison(subpage
)) {
2450 VM_WARN_ON_FOLIO(folio_is_device_private(folio
), folio
);
2452 pteval
= swp_entry_to_pte(make_hwpoison_entry(subpage
));
2453 if (folio_test_hugetlb(folio
)) {
2454 hugetlb_count_sub(folio_nr_pages(folio
), mm
);
2455 set_huge_pte_at(mm
, address
, pvmw
.pte
, pteval
,
2458 dec_mm_counter(mm
, mm_counter(folio
));
2459 set_pte_at(mm
, address
, pvmw
.pte
, pteval
);
2461 } else if (likely(pte_present(pteval
)) && pte_unused(pteval
) &&
2462 !userfaultfd_armed(vma
)) {
2464 * The guest indicated that the page content is of no
2465 * interest anymore. Simply discard the pte, vmscan
2466 * will take care of the rest.
2467 * A future reference will then fault in a new zero
2468 * page. When userfaultfd is active, we must not drop
2469 * this page though, as its main user (postcopy
2470 * migration) will not expect userfaults on already
2473 dec_mm_counter(mm
, mm_counter(folio
));
2479 * arch_unmap_one() is expected to be a NOP on
2480 * architectures where we could have PFN swap PTEs,
2481 * so we'll not check/care.
2483 if (arch_unmap_one(mm
, vma
, address
, pteval
) < 0) {
2484 if (folio_test_hugetlb(folio
))
2485 set_huge_pte_at(mm
, address
, pvmw
.pte
,
2488 set_pte_at(mm
, address
, pvmw
.pte
, pteval
);
2490 page_vma_mapped_walk_done(&pvmw
);
2494 /* See folio_try_share_anon_rmap_pte(): clear PTE first. */
2495 if (folio_test_hugetlb(folio
)) {
2496 if (anon_exclusive
&&
2497 hugetlb_try_share_anon_rmap(folio
)) {
2498 set_huge_pte_at(mm
, address
, pvmw
.pte
,
2501 page_vma_mapped_walk_done(&pvmw
);
2504 } else if (anon_exclusive
&&
2505 folio_try_share_anon_rmap_pte(folio
, subpage
)) {
2506 set_pte_at(mm
, address
, pvmw
.pte
, pteval
);
2508 page_vma_mapped_walk_done(&pvmw
);
2513 * Store the pfn of the page in a special migration
2514 * pte. do_swap_page() will wait until the migration
2515 * pte is removed and then restart fault handling.
2518 entry
= make_writable_migration_entry(
2519 page_to_pfn(subpage
));
2520 else if (anon_exclusive
)
2521 entry
= make_readable_exclusive_migration_entry(
2522 page_to_pfn(subpage
));
2524 entry
= make_readable_migration_entry(
2525 page_to_pfn(subpage
));
2526 if (likely(pte_present(pteval
))) {
2527 if (pte_young(pteval
))
2528 entry
= make_migration_entry_young(entry
);
2529 if (pte_dirty(pteval
))
2530 entry
= make_migration_entry_dirty(entry
);
2531 swp_pte
= swp_entry_to_pte(entry
);
2532 if (pte_soft_dirty(pteval
))
2533 swp_pte
= pte_swp_mksoft_dirty(swp_pte
);
2534 if (pte_uffd_wp(pteval
))
2535 swp_pte
= pte_swp_mkuffd_wp(swp_pte
);
2537 swp_pte
= swp_entry_to_pte(entry
);
2538 if (pte_swp_soft_dirty(pteval
))
2539 swp_pte
= pte_swp_mksoft_dirty(swp_pte
);
2540 if (pte_swp_uffd_wp(pteval
))
2541 swp_pte
= pte_swp_mkuffd_wp(swp_pte
);
2543 if (folio_test_hugetlb(folio
))
2544 set_huge_pte_at(mm
, address
, pvmw
.pte
, swp_pte
,
2547 set_pte_at(mm
, address
, pvmw
.pte
, swp_pte
);
2548 trace_set_migration_pte(address
, pte_val(swp_pte
),
2549 folio_order(folio
));
2551 * No need to invalidate here it will synchronize on
2552 * against the special swap migration pte.
2556 if (unlikely(folio_test_hugetlb(folio
)))
2557 hugetlb_remove_rmap(folio
);
2559 folio_remove_rmap_pte(folio
, subpage
, vma
);
2560 if (vma
->vm_flags
& VM_LOCKED
)
2561 mlock_drain_local();
2565 mmu_notifier_invalidate_range_end(&range
);
2571 * try_to_migrate - try to replace all page table mappings with swap entries
2572 * @folio: the folio to replace page table entries for
2573 * @flags: action and flags
2575 * Tries to remove all the page table entries which are mapping this folio and
2576 * replace them with special swap entries. Caller must hold the folio lock.
2578 void try_to_migrate(struct folio
*folio
, enum ttu_flags flags
)
2580 struct rmap_walk_control rwc
= {
2581 .rmap_one
= try_to_migrate_one
,
2582 .arg
= (void *)flags
,
2583 .done
= folio_not_mapped
,
2584 .anon_lock
= folio_lock_anon_vma_read
,
2588 * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and
2589 * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags.
2591 if (WARN_ON_ONCE(flags
& ~(TTU_RMAP_LOCKED
| TTU_SPLIT_HUGE_PMD
|
2592 TTU_SYNC
| TTU_BATCH_FLUSH
)))
2595 if (folio_is_zone_device(folio
) &&
2596 (!folio_is_device_private(folio
) && !folio_is_device_coherent(folio
)))
2600 * During exec, a temporary VMA is setup and later moved.
2601 * The VMA is moved under the anon_vma lock but not the
2602 * page tables leading to a race where migration cannot
2603 * find the migration ptes. Rather than increasing the
2604 * locking requirements of exec(), migration skips
2605 * temporary VMAs until after exec() completes.
2607 if (!folio_test_ksm(folio
) && folio_test_anon(folio
))
2608 rwc
.invalid_vma
= invalid_migration_vma
;
2610 if (flags
& TTU_RMAP_LOCKED
)
2611 rmap_walk_locked(folio
, &rwc
);
2613 rmap_walk(folio
, &rwc
);
2616 #ifdef CONFIG_DEVICE_PRIVATE
2618 * make_device_exclusive() - Mark a page for exclusive use by a device
2619 * @mm: mm_struct of associated target process
2620 * @addr: the virtual address to mark for exclusive device access
2621 * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering
2622 * @foliop: folio pointer will be stored here on success.
2624 * This function looks up the page mapped at the given address, grabs a
2625 * folio reference, locks the folio and replaces the PTE with special
2626 * device-exclusive PFN swap entry, preventing access through the process
2627 * page tables. The function will return with the folio locked and referenced.
2629 * On fault, the device-exclusive entries are replaced with the original PTE
2630 * under folio lock, after calling MMU notifiers.
2632 * Only anonymous non-hugetlb folios are supported and the VMA must have
2633 * write permissions such that we can fault in the anonymous page writable
2634 * in order to mark it exclusive. The caller must hold the mmap_lock in read
2637 * A driver using this to program access from a device must use a mmu notifier
2638 * critical section to hold a device specific lock during programming. Once
2639 * programming is complete it should drop the folio lock and reference after
2640 * which point CPU access to the page will revoke the exclusive access.
2643 * #. This function always operates on individual PTEs mapping individual
2644 * pages. PMD-sized THPs are first remapped to be mapped by PTEs before
2645 * the conversion happens on a single PTE corresponding to @addr.
2646 * #. While concurrent access through the process page tables is prevented,
2647 * concurrent access through other page references (e.g., earlier GUP
2648 * invocation) is not handled and not supported.
2649 * #. device-exclusive entries are considered "clean" and "old" by core-mm.
2650 * Device drivers must update the folio state when informed by MMU
2653 * Returns: pointer to mapped page on success, otherwise a negative error.
2655 struct page
*make_device_exclusive(struct mm_struct
*mm
, unsigned long addr
,
2656 void *owner
, struct folio
**foliop
)
2658 struct mmu_notifier_range range
;
2659 struct folio
*folio
, *fw_folio
;
2660 struct vm_area_struct
*vma
;
2661 struct folio_walk fw
;
2667 mmap_assert_locked(mm
);
2668 addr
= PAGE_ALIGN_DOWN(addr
);
2671 * Fault in the page writable and try to lock it; note that if the
2672 * address would already be marked for exclusive use by a device,
2673 * the GUP call would undo that first by triggering a fault.
2675 * If any other device would already map this page exclusively, the
2676 * fault will trigger a conversion to an ordinary
2677 * (non-device-exclusive) PTE and issue a MMU_NOTIFY_EXCLUSIVE.
2680 page
= get_user_page_vma_remote(mm
, addr
,
2681 FOLL_GET
| FOLL_WRITE
| FOLL_SPLIT_PMD
,
2685 folio
= page_folio(page
);
2687 if (!folio_test_anon(folio
) || folio_test_hugetlb(folio
)) {
2689 return ERR_PTR(-EOPNOTSUPP
);
2692 ret
= folio_lock_killable(folio
);
2695 return ERR_PTR(ret
);
2699 * Inform secondary MMUs that we are going to convert this PTE to
2700 * device-exclusive, such that they unmap it now. Note that the
2701 * caller must filter this event out to prevent livelocks.
2703 mmu_notifier_range_init_owner(&range
, MMU_NOTIFY_EXCLUSIVE
, 0,
2704 mm
, addr
, addr
+ PAGE_SIZE
, owner
);
2705 mmu_notifier_invalidate_range_start(&range
);
2708 * Let's do a second walk and make sure we still find the same page
2709 * mapped writable. Note that any page of an anonymous folio can
2710 * only be mapped writable using exactly one PTE ("exclusive"), so
2711 * there cannot be other mappings.
2713 fw_folio
= folio_walk_start(&fw
, vma
, addr
, 0);
2714 if (fw_folio
!= folio
|| fw
.page
!= page
||
2715 fw
.level
!= FW_LEVEL_PTE
|| !pte_write(fw
.pte
)) {
2717 folio_walk_end(&fw
, vma
);
2718 mmu_notifier_invalidate_range_end(&range
);
2719 folio_unlock(folio
);
2724 /* Nuke the page table entry so we get the uptodate dirty bit. */
2725 flush_cache_page(vma
, addr
, page_to_pfn(page
));
2726 fw
.pte
= ptep_clear_flush(vma
, addr
, fw
.ptep
);
2728 /* Set the dirty flag on the folio now the PTE is gone. */
2729 if (pte_dirty(fw
.pte
))
2730 folio_mark_dirty(folio
);
2733 * Store the pfn of the page in a special device-exclusive PFN swap PTE.
2734 * do_swap_page() will trigger the conversion back while holding the
2737 entry
= make_device_exclusive_entry(page_to_pfn(page
));
2738 swp_pte
= swp_entry_to_pte(entry
);
2739 if (pte_soft_dirty(fw
.pte
))
2740 swp_pte
= pte_swp_mksoft_dirty(swp_pte
);
2741 /* The pte is writable, uffd-wp does not apply. */
2742 set_pte_at(mm
, addr
, fw
.ptep
, swp_pte
);
2744 folio_walk_end(&fw
, vma
);
2745 mmu_notifier_invalidate_range_end(&range
);
2749 EXPORT_SYMBOL_GPL(make_device_exclusive
);
2752 void __put_anon_vma(struct anon_vma
*anon_vma
)
2754 struct anon_vma
*root
= anon_vma
->root
;
2756 anon_vma_free(anon_vma
);
2757 if (root
!= anon_vma
&& atomic_dec_and_test(&root
->refcount
))
2758 anon_vma_free(root
);
2761 static struct anon_vma
*rmap_walk_anon_lock(const struct folio
*folio
,
2762 struct rmap_walk_control
*rwc
)
2764 struct anon_vma
*anon_vma
;
2767 return rwc
->anon_lock(folio
, rwc
);
2770 * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
2771 * because that depends on page_mapped(); but not all its usages
2772 * are holding mmap_lock. Users without mmap_lock are required to
2773 * take a reference count to prevent the anon_vma disappearing
2775 anon_vma
= folio_anon_vma(folio
);
2779 if (anon_vma_trylock_read(anon_vma
))
2782 if (rwc
->try_lock
) {
2784 rwc
->contended
= true;
2788 anon_vma_lock_read(anon_vma
);
2794 * rmap_walk_anon - do something to anonymous page using the object-based
2796 * @folio: the folio to be handled
2797 * @rwc: control variable according to each walk type
2798 * @locked: caller holds relevant rmap lock
2800 * Find all the mappings of a folio using the mapping pointer and the vma
2801 * chains contained in the anon_vma struct it points to.
2803 static void rmap_walk_anon(struct folio
*folio
,
2804 struct rmap_walk_control
*rwc
, bool locked
)
2806 struct anon_vma
*anon_vma
;
2807 pgoff_t pgoff_start
, pgoff_end
;
2808 struct anon_vma_chain
*avc
;
2811 anon_vma
= folio_anon_vma(folio
);
2812 /* anon_vma disappear under us? */
2813 VM_BUG_ON_FOLIO(!anon_vma
, folio
);
2815 anon_vma
= rmap_walk_anon_lock(folio
, rwc
);
2820 pgoff_start
= folio_pgoff(folio
);
2821 pgoff_end
= pgoff_start
+ folio_nr_pages(folio
) - 1;
2822 anon_vma_interval_tree_foreach(avc
, &anon_vma
->rb_root
,
2823 pgoff_start
, pgoff_end
) {
2824 struct vm_area_struct
*vma
= avc
->vma
;
2825 unsigned long address
= vma_address(vma
, pgoff_start
,
2826 folio_nr_pages(folio
));
2828 VM_BUG_ON_VMA(address
== -EFAULT
, vma
);
2831 if (rwc
->invalid_vma
&& rwc
->invalid_vma(vma
, rwc
->arg
))
2834 if (!rwc
->rmap_one(folio
, vma
, address
, rwc
->arg
))
2836 if (rwc
->done
&& rwc
->done(folio
))
2841 anon_vma_unlock_read(anon_vma
);
2845 * __rmap_walk_file() - Traverse the reverse mapping for a file-backed mapping
2846 * of a page mapped within a specified page cache object at a specified offset.
2848 * @folio: Either the folio whose mappings to traverse, or if NULL,
2849 * the callbacks specified in @rwc will be configured such
2850 * as to be able to look up mappings correctly.
2851 * @mapping: The page cache object whose mapping VMAs we intend to
2852 * traverse. If @folio is non-NULL, this should be equal to
2853 * folio_mapping(folio).
2854 * @pgoff_start: The offset within @mapping of the page which we are
2855 * looking up. If @folio is non-NULL, this should be equal
2856 * to folio_pgoff(folio).
2857 * @nr_pages: The number of pages mapped by the mapping. If @folio is
2858 * non-NULL, this should be equal to folio_nr_pages(folio).
2859 * @rwc: The reverse mapping walk control object describing how
2860 * the traversal should proceed.
2861 * @locked: Is the @mapping already locked? If not, we acquire the
2864 static void __rmap_walk_file(struct folio
*folio
, struct address_space
*mapping
,
2865 pgoff_t pgoff_start
, unsigned long nr_pages
,
2866 struct rmap_walk_control
*rwc
, bool locked
)
2868 pgoff_t pgoff_end
= pgoff_start
+ nr_pages
- 1;
2869 struct vm_area_struct
*vma
;
2871 VM_WARN_ON_FOLIO(folio
&& mapping
!= folio_mapping(folio
), folio
);
2872 VM_WARN_ON_FOLIO(folio
&& pgoff_start
!= folio_pgoff(folio
), folio
);
2873 VM_WARN_ON_FOLIO(folio
&& nr_pages
!= folio_nr_pages(folio
), folio
);
2876 if (i_mmap_trylock_read(mapping
))
2879 if (rwc
->try_lock
) {
2880 rwc
->contended
= true;
2884 i_mmap_lock_read(mapping
);
2887 vma_interval_tree_foreach(vma
, &mapping
->i_mmap
,
2888 pgoff_start
, pgoff_end
) {
2889 unsigned long address
= vma_address(vma
, pgoff_start
, nr_pages
);
2891 VM_BUG_ON_VMA(address
== -EFAULT
, vma
);
2894 if (rwc
->invalid_vma
&& rwc
->invalid_vma(vma
, rwc
->arg
))
2897 if (!rwc
->rmap_one(folio
, vma
, address
, rwc
->arg
))
2899 if (rwc
->done
&& rwc
->done(folio
))
2904 i_mmap_unlock_read(mapping
);
2908 * rmap_walk_file - do something to file page using the object-based rmap method
2909 * @folio: the folio to be handled
2910 * @rwc: control variable according to each walk type
2911 * @locked: caller holds relevant rmap lock
2913 * Find all the mappings of a folio using the mapping pointer and the vma chains
2914 * contained in the address_space struct it points to.
2916 static void rmap_walk_file(struct folio
*folio
,
2917 struct rmap_walk_control
*rwc
, bool locked
)
2920 * The folio lock not only makes sure that folio->mapping cannot
2921 * suddenly be NULLified by truncation, it makes sure that the structure
2922 * at mapping cannot be freed and reused yet, so we can safely take
2923 * mapping->i_mmap_rwsem.
2925 VM_BUG_ON_FOLIO(!folio_test_locked(folio
), folio
);
2927 if (!folio
->mapping
)
2930 __rmap_walk_file(folio
, folio
->mapping
, folio
->index
,
2931 folio_nr_pages(folio
), rwc
, locked
);
2934 void rmap_walk(struct folio
*folio
, struct rmap_walk_control
*rwc
)
2936 if (unlikely(folio_test_ksm(folio
)))
2937 rmap_walk_ksm(folio
, rwc
);
2938 else if (folio_test_anon(folio
))
2939 rmap_walk_anon(folio
, rwc
, false);
2941 rmap_walk_file(folio
, rwc
, false);
2944 /* Like rmap_walk, but caller holds relevant rmap lock */
2945 void rmap_walk_locked(struct folio
*folio
, struct rmap_walk_control
*rwc
)
2947 /* no ksm support for now */
2948 VM_BUG_ON_FOLIO(folio_test_ksm(folio
), folio
);
2949 if (folio_test_anon(folio
))
2950 rmap_walk_anon(folio
, rwc
, true);
2952 rmap_walk_file(folio
, rwc
, true);
2955 #ifdef CONFIG_HUGETLB_PAGE
2957 * The following two functions are for anonymous (private mapped) hugepages.
2958 * Unlike common anonymous pages, anonymous hugepages have no accounting code
2959 * and no lru code, because we handle hugepages differently from common pages.
2961 void hugetlb_add_anon_rmap(struct folio
*folio
, struct vm_area_struct
*vma
,
2962 unsigned long address
, rmap_t flags
)
2964 VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio
), folio
);
2965 VM_WARN_ON_FOLIO(!folio_test_anon(folio
), folio
);
2967 atomic_inc(&folio
->_entire_mapcount
);
2968 atomic_inc(&folio
->_large_mapcount
);
2969 if (flags
& RMAP_EXCLUSIVE
)
2970 SetPageAnonExclusive(&folio
->page
);
2971 VM_WARN_ON_FOLIO(folio_entire_mapcount(folio
) > 1 &&
2972 PageAnonExclusive(&folio
->page
), folio
);
2975 void hugetlb_add_new_anon_rmap(struct folio
*folio
,
2976 struct vm_area_struct
*vma
, unsigned long address
)
2978 VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio
), folio
);
2980 BUG_ON(address
< vma
->vm_start
|| address
>= vma
->vm_end
);
2981 /* increment count (starts at -1) */
2982 atomic_set(&folio
->_entire_mapcount
, 0);
2983 atomic_set(&folio
->_large_mapcount
, 0);
2984 folio_clear_hugetlb_restore_reserve(folio
);
2985 __folio_set_anon(folio
, vma
, address
, true);
2986 SetPageAnonExclusive(&folio
->page
);
2988 #endif /* CONFIG_HUGETLB_PAGE */