1 // SPDX-License-Identifier: GPL-2.0
5 * (C) Copyright 1995 Linus Torvalds
6 * (C) Copyright 2002 Christoph Hellwig
9 #include <linux/capability.h>
10 #include <linux/mman.h>
12 #include <linux/sched/user.h>
13 #include <linux/swap.h>
14 #include <linux/swapops.h>
15 #include <linux/pagemap.h>
16 #include <linux/pagevec.h>
17 #include <linux/pagewalk.h>
18 #include <linux/mempolicy.h>
19 #include <linux/syscalls.h>
20 #include <linux/sched.h>
21 #include <linux/export.h>
22 #include <linux/rmap.h>
23 #include <linux/mmzone.h>
24 #include <linux/hugetlb.h>
25 #include <linux/memcontrol.h>
26 #include <linux/mm_inline.h>
27 #include <linux/secretmem.h>
33 struct folio_batch fbatch
;
36 static DEFINE_PER_CPU(struct mlock_fbatch
, mlock_fbatch
) = {
37 .lock
= INIT_LOCAL_LOCK(lock
),
40 bool can_do_mlock(void)
42 if (rlimit(RLIMIT_MEMLOCK
) != 0)
44 if (capable(CAP_IPC_LOCK
))
48 EXPORT_SYMBOL(can_do_mlock
);
51 * Mlocked folios are marked with the PG_mlocked flag for efficient testing
52 * in vmscan and, possibly, the fault path; and to support semi-accurate
55 * An mlocked folio [folio_test_mlocked(folio)] is unevictable. As such, it
56 * will be ostensibly placed on the LRU "unevictable" list (actually no such
57 * list exists), rather than the [in]active lists. PG_unevictable is set to
58 * indicate the unevictable state.
61 static struct lruvec
*__mlock_folio(struct folio
*folio
, struct lruvec
*lruvec
)
63 /* There is nothing more we can do while it's off LRU */
64 if (!folio_test_clear_lru(folio
))
67 lruvec
= folio_lruvec_relock_irq(folio
, lruvec
);
69 if (unlikely(folio_evictable(folio
))) {
71 * This is a little surprising, but quite possible: PG_mlocked
72 * must have got cleared already by another CPU. Could this
73 * folio be unevictable? I'm not sure, but move it now if so.
75 if (folio_test_unevictable(folio
)) {
76 lruvec_del_folio(lruvec
, folio
);
77 folio_clear_unevictable(folio
);
78 lruvec_add_folio(lruvec
, folio
);
80 __count_vm_events(UNEVICTABLE_PGRESCUED
,
81 folio_nr_pages(folio
));
86 if (folio_test_unevictable(folio
)) {
87 if (folio_test_mlocked(folio
))
92 lruvec_del_folio(lruvec
, folio
);
93 folio_clear_active(folio
);
94 folio_set_unevictable(folio
);
95 folio
->mlock_count
= !!folio_test_mlocked(folio
);
96 lruvec_add_folio(lruvec
, folio
);
97 __count_vm_events(UNEVICTABLE_PGCULLED
, folio_nr_pages(folio
));
103 static struct lruvec
*__mlock_new_folio(struct folio
*folio
, struct lruvec
*lruvec
)
105 VM_BUG_ON_FOLIO(folio_test_lru(folio
), folio
);
107 lruvec
= folio_lruvec_relock_irq(folio
, lruvec
);
109 /* As above, this is a little surprising, but possible */
110 if (unlikely(folio_evictable(folio
)))
113 folio_set_unevictable(folio
);
114 folio
->mlock_count
= !!folio_test_mlocked(folio
);
115 __count_vm_events(UNEVICTABLE_PGCULLED
, folio_nr_pages(folio
));
117 lruvec_add_folio(lruvec
, folio
);
118 folio_set_lru(folio
);
122 static struct lruvec
*__munlock_folio(struct folio
*folio
, struct lruvec
*lruvec
)
124 int nr_pages
= folio_nr_pages(folio
);
125 bool isolated
= false;
127 if (!folio_test_clear_lru(folio
))
131 lruvec
= folio_lruvec_relock_irq(folio
, lruvec
);
133 if (folio_test_unevictable(folio
)) {
134 /* Then mlock_count is maintained, but might undercount */
135 if (folio
->mlock_count
)
136 folio
->mlock_count
--;
137 if (folio
->mlock_count
)
140 /* else assume that was the last mlock: reclaim will fix it if not */
143 if (folio_test_clear_mlocked(folio
)) {
144 __zone_stat_mod_folio(folio
, NR_MLOCK
, -nr_pages
);
145 if (isolated
|| !folio_test_unevictable(folio
))
146 __count_vm_events(UNEVICTABLE_PGMUNLOCKED
, nr_pages
);
148 __count_vm_events(UNEVICTABLE_PGSTRANDED
, nr_pages
);
151 /* folio_evictable() has to be checked *after* clearing Mlocked */
152 if (isolated
&& folio_test_unevictable(folio
) && folio_evictable(folio
)) {
153 lruvec_del_folio(lruvec
, folio
);
154 folio_clear_unevictable(folio
);
155 lruvec_add_folio(lruvec
, folio
);
156 __count_vm_events(UNEVICTABLE_PGRESCUED
, nr_pages
);
160 folio_set_lru(folio
);
165 * Flags held in the low bits of a struct folio pointer on the mlock_fbatch.
167 #define LRU_FOLIO 0x1
168 #define NEW_FOLIO 0x2
169 static inline struct folio
*mlock_lru(struct folio
*folio
)
171 return (struct folio
*)((unsigned long)folio
+ LRU_FOLIO
);
174 static inline struct folio
*mlock_new(struct folio
*folio
)
176 return (struct folio
*)((unsigned long)folio
+ NEW_FOLIO
);
180 * mlock_folio_batch() is derived from folio_batch_move_lru(): perhaps that can
181 * make use of such folio pointer flags in future, but for now just keep it for
182 * mlock. We could use three separate folio batches instead, but one feels
183 * better (munlocking a full folio batch does not need to drain mlocking folio
186 static void mlock_folio_batch(struct folio_batch
*fbatch
)
188 struct lruvec
*lruvec
= NULL
;
193 for (i
= 0; i
< folio_batch_count(fbatch
); i
++) {
194 folio
= fbatch
->folios
[i
];
195 mlock
= (unsigned long)folio
& (LRU_FOLIO
| NEW_FOLIO
);
196 folio
= (struct folio
*)((unsigned long)folio
- mlock
);
197 fbatch
->folios
[i
] = folio
;
199 if (mlock
& LRU_FOLIO
)
200 lruvec
= __mlock_folio(folio
, lruvec
);
201 else if (mlock
& NEW_FOLIO
)
202 lruvec
= __mlock_new_folio(folio
, lruvec
);
204 lruvec
= __munlock_folio(folio
, lruvec
);
208 unlock_page_lruvec_irq(lruvec
);
209 folios_put(fbatch
->folios
, folio_batch_count(fbatch
));
210 folio_batch_reinit(fbatch
);
213 void mlock_drain_local(void)
215 struct folio_batch
*fbatch
;
217 local_lock(&mlock_fbatch
.lock
);
218 fbatch
= this_cpu_ptr(&mlock_fbatch
.fbatch
);
219 if (folio_batch_count(fbatch
))
220 mlock_folio_batch(fbatch
);
221 local_unlock(&mlock_fbatch
.lock
);
224 void mlock_drain_remote(int cpu
)
226 struct folio_batch
*fbatch
;
228 WARN_ON_ONCE(cpu_online(cpu
));
229 fbatch
= &per_cpu(mlock_fbatch
.fbatch
, cpu
);
230 if (folio_batch_count(fbatch
))
231 mlock_folio_batch(fbatch
);
234 bool need_mlock_drain(int cpu
)
236 return folio_batch_count(&per_cpu(mlock_fbatch
.fbatch
, cpu
));
240 * mlock_folio - mlock a folio already on (or temporarily off) LRU
241 * @folio: folio to be mlocked.
243 void mlock_folio(struct folio
*folio
)
245 struct folio_batch
*fbatch
;
247 local_lock(&mlock_fbatch
.lock
);
248 fbatch
= this_cpu_ptr(&mlock_fbatch
.fbatch
);
250 if (!folio_test_set_mlocked(folio
)) {
251 int nr_pages
= folio_nr_pages(folio
);
253 zone_stat_mod_folio(folio
, NR_MLOCK
, nr_pages
);
254 __count_vm_events(UNEVICTABLE_PGMLOCKED
, nr_pages
);
258 if (!folio_batch_add(fbatch
, mlock_lru(folio
)) ||
259 folio_test_large(folio
) || lru_cache_disabled())
260 mlock_folio_batch(fbatch
);
261 local_unlock(&mlock_fbatch
.lock
);
265 * mlock_new_folio - mlock a newly allocated folio not yet on LRU
266 * @folio: folio to be mlocked, either normal or a THP head.
268 void mlock_new_folio(struct folio
*folio
)
270 struct folio_batch
*fbatch
;
271 int nr_pages
= folio_nr_pages(folio
);
273 local_lock(&mlock_fbatch
.lock
);
274 fbatch
= this_cpu_ptr(&mlock_fbatch
.fbatch
);
275 folio_set_mlocked(folio
);
277 zone_stat_mod_folio(folio
, NR_MLOCK
, nr_pages
);
278 __count_vm_events(UNEVICTABLE_PGMLOCKED
, nr_pages
);
281 if (!folio_batch_add(fbatch
, mlock_new(folio
)) ||
282 folio_test_large(folio
) || lru_cache_disabled())
283 mlock_folio_batch(fbatch
);
284 local_unlock(&mlock_fbatch
.lock
);
288 * munlock_folio - munlock a folio
289 * @folio: folio to be munlocked, either normal or a THP head.
291 void munlock_folio(struct folio
*folio
)
293 struct folio_batch
*fbatch
;
295 local_lock(&mlock_fbatch
.lock
);
296 fbatch
= this_cpu_ptr(&mlock_fbatch
.fbatch
);
298 * folio_test_clear_mlocked(folio) must be left to __munlock_folio(),
299 * which will check whether the folio is multiply mlocked.
302 if (!folio_batch_add(fbatch
, folio
) ||
303 folio_test_large(folio
) || lru_cache_disabled())
304 mlock_folio_batch(fbatch
);
305 local_unlock(&mlock_fbatch
.lock
);
308 static int mlock_pte_range(pmd_t
*pmd
, unsigned long addr
,
309 unsigned long end
, struct mm_walk
*walk
)
312 struct vm_area_struct
*vma
= walk
->vma
;
314 pte_t
*start_pte
, *pte
;
318 ptl
= pmd_trans_huge_lock(pmd
, vma
);
320 if (!pmd_present(*pmd
))
322 if (is_huge_zero_pmd(*pmd
))
324 folio
= page_folio(pmd_page(*pmd
));
325 if (vma
->vm_flags
& VM_LOCKED
)
328 munlock_folio(folio
);
332 start_pte
= pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, &ptl
);
334 walk
->action
= ACTION_AGAIN
;
337 for (pte
= start_pte
; addr
!= end
; pte
++, addr
+= PAGE_SIZE
) {
338 ptent
= ptep_get(pte
);
339 if (!pte_present(ptent
))
341 folio
= vm_normal_folio(vma
, addr
, ptent
);
342 if (!folio
|| folio_is_zone_device(folio
))
344 if (folio_test_large(folio
))
346 if (vma
->vm_flags
& VM_LOCKED
)
349 munlock_folio(folio
);
351 pte_unmap(start_pte
);
359 * mlock_vma_pages_range() - mlock any pages already in the range,
360 * or munlock all pages in the range.
361 * @vma - vma containing range to be mlock()ed or munlock()ed
362 * @start - start address in @vma of the range
363 * @end - end of range in @vma
364 * @newflags - the new set of flags for @vma.
366 * Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED;
367 * called for munlock() and munlockall(), to clear VM_LOCKED from @vma.
369 static void mlock_vma_pages_range(struct vm_area_struct
*vma
,
370 unsigned long start
, unsigned long end
, vm_flags_t newflags
)
372 static const struct mm_walk_ops mlock_walk_ops
= {
373 .pmd_entry
= mlock_pte_range
,
374 .walk_lock
= PGWALK_WRLOCK_VERIFY
,
378 * There is a slight chance that concurrent page migration,
379 * or page reclaim finding a page of this now-VM_LOCKED vma,
380 * will call mlock_vma_folio() and raise page's mlock_count:
381 * double counting, leaving the page unevictable indefinitely.
382 * Communicate this danger to mlock_vma_folio() with VM_IO,
383 * which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas.
384 * mmap_lock is held in write mode here, so this weird
385 * combination should not be visible to other mmap_lock users;
386 * but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED.
388 if (newflags
& VM_LOCKED
)
390 vma_start_write(vma
);
391 vm_flags_reset_once(vma
, newflags
);
394 walk_page_range(vma
->vm_mm
, start
, end
, &mlock_walk_ops
, NULL
);
397 if (newflags
& VM_IO
) {
399 vm_flags_reset_once(vma
, newflags
);
404 * mlock_fixup - handle mlock[all]/munlock[all] requests.
406 * Filters out "special" vmas -- VM_LOCKED never gets set for these, and
407 * munlock is a no-op. However, for some special vmas, we go ahead and
410 * For vmas that pass the filters, merge/split as appropriate.
412 static int mlock_fixup(struct vma_iterator
*vmi
, struct vm_area_struct
*vma
,
413 struct vm_area_struct
**prev
, unsigned long start
,
414 unsigned long end
, vm_flags_t newflags
)
416 struct mm_struct
*mm
= vma
->vm_mm
;
420 vm_flags_t oldflags
= vma
->vm_flags
;
422 if (newflags
== oldflags
|| (oldflags
& VM_SPECIAL
) ||
423 is_vm_hugetlb_page(vma
) || vma
== get_gate_vma(current
->mm
) ||
424 vma_is_dax(vma
) || vma_is_secretmem(vma
))
425 /* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */
428 pgoff
= vma
->vm_pgoff
+ ((start
- vma
->vm_start
) >> PAGE_SHIFT
);
429 *prev
= vma_merge(vmi
, mm
, *prev
, start
, end
, newflags
,
430 vma
->anon_vma
, vma
->vm_file
, pgoff
, vma_policy(vma
),
431 vma
->vm_userfaultfd_ctx
, anon_vma_name(vma
));
437 if (start
!= vma
->vm_start
) {
438 ret
= split_vma(vmi
, vma
, start
, 1);
443 if (end
!= vma
->vm_end
) {
444 ret
= split_vma(vmi
, vma
, end
, 0);
451 * Keep track of amount of locked VM.
453 nr_pages
= (end
- start
) >> PAGE_SHIFT
;
454 if (!(newflags
& VM_LOCKED
))
455 nr_pages
= -nr_pages
;
456 else if (oldflags
& VM_LOCKED
)
458 mm
->locked_vm
+= nr_pages
;
461 * vm_flags is protected by the mmap_lock held in write mode.
462 * It's okay if try_to_unmap_one unmaps a page just after we
463 * set VM_LOCKED, populate_vma_page_range will bring it back.
465 if ((newflags
& VM_LOCKED
) && (oldflags
& VM_LOCKED
)) {
466 /* No work to do, and mlocking twice would be wrong */
467 vma_start_write(vma
);
468 vm_flags_reset(vma
, newflags
);
470 mlock_vma_pages_range(vma
, start
, end
, newflags
);
477 static int apply_vma_lock_flags(unsigned long start
, size_t len
,
480 unsigned long nstart
, end
, tmp
;
481 struct vm_area_struct
*vma
, *prev
;
482 VMA_ITERATOR(vmi
, current
->mm
, start
);
484 VM_BUG_ON(offset_in_page(start
));
485 VM_BUG_ON(len
!= PAGE_ALIGN(len
));
491 vma
= vma_iter_load(&vmi
);
495 prev
= vma_prev(&vmi
);
496 if (start
> vma
->vm_start
)
501 for_each_vma_range(vmi
, vma
, end
) {
505 if (vma
->vm_start
!= tmp
)
508 newflags
= vma
->vm_flags
& ~VM_LOCKED_MASK
;
510 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
514 error
= mlock_fixup(&vmi
, vma
, &prev
, nstart
, tmp
, newflags
);
517 tmp
= vma_iter_end(&vmi
);
528 * Go through vma areas and sum size of mlocked
529 * vma pages, as return value.
530 * Note deferred memory locking case(mlock2(,,MLOCK_ONFAULT)
532 * Return value: previously mlocked page counts
534 static unsigned long count_mm_mlocked_page_nr(struct mm_struct
*mm
,
535 unsigned long start
, size_t len
)
537 struct vm_area_struct
*vma
;
538 unsigned long count
= 0;
540 VMA_ITERATOR(vmi
, mm
, start
);
542 /* Don't overflow past ULONG_MAX */
543 if (unlikely(ULONG_MAX
- len
< start
))
548 for_each_vma_range(vmi
, vma
, end
) {
549 if (vma
->vm_flags
& VM_LOCKED
) {
550 if (start
> vma
->vm_start
)
551 count
-= (start
- vma
->vm_start
);
552 if (end
< vma
->vm_end
) {
553 count
+= end
- vma
->vm_start
;
556 count
+= vma
->vm_end
- vma
->vm_start
;
560 return count
>> PAGE_SHIFT
;
564 * convert get_user_pages() return value to posix mlock() error
566 static int __mlock_posix_error_return(long retval
)
568 if (retval
== -EFAULT
)
570 else if (retval
== -ENOMEM
)
575 static __must_check
int do_mlock(unsigned long start
, size_t len
, vm_flags_t flags
)
577 unsigned long locked
;
578 unsigned long lock_limit
;
581 start
= untagged_addr(start
);
586 len
= PAGE_ALIGN(len
+ (offset_in_page(start
)));
589 lock_limit
= rlimit(RLIMIT_MEMLOCK
);
590 lock_limit
>>= PAGE_SHIFT
;
591 locked
= len
>> PAGE_SHIFT
;
593 if (mmap_write_lock_killable(current
->mm
))
596 locked
+= current
->mm
->locked_vm
;
597 if ((locked
> lock_limit
) && (!capable(CAP_IPC_LOCK
))) {
599 * It is possible that the regions requested intersect with
600 * previously mlocked areas, that part area in "mm->locked_vm"
601 * should not be counted to new mlock increment count. So check
602 * and adjust locked count if necessary.
604 locked
-= count_mm_mlocked_page_nr(current
->mm
,
608 /* check against resource limits */
609 if ((locked
<= lock_limit
) || capable(CAP_IPC_LOCK
))
610 error
= apply_vma_lock_flags(start
, len
, flags
);
612 mmap_write_unlock(current
->mm
);
616 error
= __mm_populate(start
, len
, 0);
618 return __mlock_posix_error_return(error
);
622 SYSCALL_DEFINE2(mlock
, unsigned long, start
, size_t, len
)
624 return do_mlock(start
, len
, VM_LOCKED
);
627 SYSCALL_DEFINE3(mlock2
, unsigned long, start
, size_t, len
, int, flags
)
629 vm_flags_t vm_flags
= VM_LOCKED
;
631 if (flags
& ~MLOCK_ONFAULT
)
634 if (flags
& MLOCK_ONFAULT
)
635 vm_flags
|= VM_LOCKONFAULT
;
637 return do_mlock(start
, len
, vm_flags
);
640 SYSCALL_DEFINE2(munlock
, unsigned long, start
, size_t, len
)
644 start
= untagged_addr(start
);
646 len
= PAGE_ALIGN(len
+ (offset_in_page(start
)));
649 if (mmap_write_lock_killable(current
->mm
))
651 ret
= apply_vma_lock_flags(start
, len
, 0);
652 mmap_write_unlock(current
->mm
);
658 * Take the MCL_* flags passed into mlockall (or 0 if called from munlockall)
659 * and translate into the appropriate modifications to mm->def_flags and/or the
660 * flags for all current VMAs.
662 * There are a couple of subtleties with this. If mlockall() is called multiple
663 * times with different flags, the values do not necessarily stack. If mlockall
664 * is called once including the MCL_FUTURE flag and then a second time without
665 * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags.
667 static int apply_mlockall_flags(int flags
)
669 VMA_ITERATOR(vmi
, current
->mm
, 0);
670 struct vm_area_struct
*vma
, *prev
= NULL
;
671 vm_flags_t to_add
= 0;
673 current
->mm
->def_flags
&= ~VM_LOCKED_MASK
;
674 if (flags
& MCL_FUTURE
) {
675 current
->mm
->def_flags
|= VM_LOCKED
;
677 if (flags
& MCL_ONFAULT
)
678 current
->mm
->def_flags
|= VM_LOCKONFAULT
;
680 if (!(flags
& MCL_CURRENT
))
684 if (flags
& MCL_CURRENT
) {
686 if (flags
& MCL_ONFAULT
)
687 to_add
|= VM_LOCKONFAULT
;
690 for_each_vma(vmi
, vma
) {
693 newflags
= vma
->vm_flags
& ~VM_LOCKED_MASK
;
697 mlock_fixup(&vmi
, vma
, &prev
, vma
->vm_start
, vma
->vm_end
,
705 SYSCALL_DEFINE1(mlockall
, int, flags
)
707 unsigned long lock_limit
;
710 if (!flags
|| (flags
& ~(MCL_CURRENT
| MCL_FUTURE
| MCL_ONFAULT
)) ||
711 flags
== MCL_ONFAULT
)
717 lock_limit
= rlimit(RLIMIT_MEMLOCK
);
718 lock_limit
>>= PAGE_SHIFT
;
720 if (mmap_write_lock_killable(current
->mm
))
724 if (!(flags
& MCL_CURRENT
) || (current
->mm
->total_vm
<= lock_limit
) ||
725 capable(CAP_IPC_LOCK
))
726 ret
= apply_mlockall_flags(flags
);
727 mmap_write_unlock(current
->mm
);
728 if (!ret
&& (flags
& MCL_CURRENT
))
729 mm_populate(0, TASK_SIZE
);
734 SYSCALL_DEFINE0(munlockall
)
738 if (mmap_write_lock_killable(current
->mm
))
740 ret
= apply_mlockall_flags(0);
741 mmap_write_unlock(current
->mm
);
746 * Objects with different lifetime than processes (SHM_LOCK and SHM_HUGETLB
747 * shm segments) get accounted against the user_struct instead.
749 static DEFINE_SPINLOCK(shmlock_user_lock
);
751 int user_shm_lock(size_t size
, struct ucounts
*ucounts
)
753 unsigned long lock_limit
, locked
;
757 locked
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
758 lock_limit
= rlimit(RLIMIT_MEMLOCK
);
759 if (lock_limit
!= RLIM_INFINITY
)
760 lock_limit
>>= PAGE_SHIFT
;
761 spin_lock(&shmlock_user_lock
);
762 memlock
= inc_rlimit_ucounts(ucounts
, UCOUNT_RLIMIT_MEMLOCK
, locked
);
764 if ((memlock
== LONG_MAX
|| memlock
> lock_limit
) && !capable(CAP_IPC_LOCK
)) {
765 dec_rlimit_ucounts(ucounts
, UCOUNT_RLIMIT_MEMLOCK
, locked
);
768 if (!get_ucounts(ucounts
)) {
769 dec_rlimit_ucounts(ucounts
, UCOUNT_RLIMIT_MEMLOCK
, locked
);
775 spin_unlock(&shmlock_user_lock
);
779 void user_shm_unlock(size_t size
, struct ucounts
*ucounts
)
781 spin_lock(&shmlock_user_lock
);
782 dec_rlimit_ucounts(ucounts
, UCOUNT_RLIMIT_MEMLOCK
, (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
);
783 spin_unlock(&shmlock_user_lock
);
784 put_ucounts(ucounts
);