]>
Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1da177e4 LT |
2 | /* |
3 | * mm/mmap.c | |
4 | * | |
5 | * Written by obz. | |
6 | * | |
046c6884 | 7 | * Address space accounting code <alan@lxorguk.ukuu.org.uk> |
1da177e4 LT |
8 | */ |
9 | ||
b1de0d13 MH |
10 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
11 | ||
e8420a8e | 12 | #include <linux/kernel.h> |
1da177e4 | 13 | #include <linux/slab.h> |
4af3c9cc | 14 | #include <linux/backing-dev.h> |
1da177e4 | 15 | #include <linux/mm.h> |
17fca131 | 16 | #include <linux/mm_inline.h> |
615d6e87 | 17 | #include <linux/vmacache.h> |
1da177e4 LT |
18 | #include <linux/shm.h> |
19 | #include <linux/mman.h> | |
20 | #include <linux/pagemap.h> | |
21 | #include <linux/swap.h> | |
22 | #include <linux/syscalls.h> | |
c59ede7b | 23 | #include <linux/capability.h> |
1da177e4 LT |
24 | #include <linux/init.h> |
25 | #include <linux/file.h> | |
26 | #include <linux/fs.h> | |
27 | #include <linux/personality.h> | |
28 | #include <linux/security.h> | |
29 | #include <linux/hugetlb.h> | |
c01d5b30 | 30 | #include <linux/shmem_fs.h> |
1da177e4 | 31 | #include <linux/profile.h> |
b95f1b31 | 32 | #include <linux/export.h> |
1da177e4 LT |
33 | #include <linux/mount.h> |
34 | #include <linux/mempolicy.h> | |
35 | #include <linux/rmap.h> | |
cddb8a5c | 36 | #include <linux/mmu_notifier.h> |
82f71ae4 | 37 | #include <linux/mmdebug.h> |
cdd6c482 | 38 | #include <linux/perf_event.h> |
120a795d | 39 | #include <linux/audit.h> |
b15d00b6 | 40 | #include <linux/khugepaged.h> |
2b144498 | 41 | #include <linux/uprobes.h> |
1640879a AS |
42 | #include <linux/notifier.h> |
43 | #include <linux/memory.h> | |
b1de0d13 | 44 | #include <linux/printk.h> |
19a809af | 45 | #include <linux/userfaultfd_k.h> |
d977d56c | 46 | #include <linux/moduleparam.h> |
62b5f7d0 | 47 | #include <linux/pkeys.h> |
21292580 | 48 | #include <linux/oom.h> |
04f5866e | 49 | #include <linux/sched/mm.h> |
1da177e4 | 50 | |
7c0f6ba6 | 51 | #include <linux/uaccess.h> |
1da177e4 LT |
52 | #include <asm/cacheflush.h> |
53 | #include <asm/tlb.h> | |
d6dd61c8 | 54 | #include <asm/mmu_context.h> |
1da177e4 | 55 | |
df529cab JK |
56 | #define CREATE_TRACE_POINTS |
57 | #include <trace/events/mmap.h> | |
58 | ||
42b77728 JB |
59 | #include "internal.h" |
60 | ||
3a459756 KK |
61 | #ifndef arch_mmap_check |
62 | #define arch_mmap_check(addr, len, flags) (0) | |
63 | #endif | |
64 | ||
d07e2259 DC |
65 | #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS |
66 | const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN; | |
67 | const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX; | |
68 | int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS; | |
69 | #endif | |
70 | #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS | |
71 | const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN; | |
72 | const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX; | |
73 | int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; | |
74 | #endif | |
75 | ||
f4fcd558 | 76 | static bool ignore_rlimit_data; |
d977d56c | 77 | core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); |
d07e2259 | 78 | |
e0da382c HD |
79 | static void unmap_region(struct mm_struct *mm, |
80 | struct vm_area_struct *vma, struct vm_area_struct *prev, | |
81 | unsigned long start, unsigned long end); | |
82 | ||
64e45507 PF |
83 | static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) |
84 | { | |
85 | return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); | |
86 | } | |
87 | ||
88 | /* Update vma->vm_page_prot to reflect vma->vm_flags. */ | |
89 | void vma_set_page_prot(struct vm_area_struct *vma) | |
90 | { | |
91 | unsigned long vm_flags = vma->vm_flags; | |
6d2329f8 | 92 | pgprot_t vm_page_prot; |
64e45507 | 93 | |
6d2329f8 AA |
94 | vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); |
95 | if (vma_wants_writenotify(vma, vm_page_prot)) { | |
64e45507 | 96 | vm_flags &= ~VM_SHARED; |
6d2329f8 | 97 | vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); |
64e45507 | 98 | } |
c1e8d7c6 | 99 | /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ |
6d2329f8 | 100 | WRITE_ONCE(vma->vm_page_prot, vm_page_prot); |
64e45507 PF |
101 | } |
102 | ||
1da177e4 | 103 | /* |
c8c06efa | 104 | * Requires inode->i_mapping->i_mmap_rwsem |
1da177e4 LT |
105 | */ |
106 | static void __remove_shared_vm_struct(struct vm_area_struct *vma, | |
107 | struct file *file, struct address_space *mapping) | |
108 | { | |
1da177e4 | 109 | if (vma->vm_flags & VM_SHARED) |
4bb5f5d9 | 110 | mapping_unmap_writable(mapping); |
1da177e4 LT |
111 | |
112 | flush_dcache_mmap_lock(mapping); | |
27ba0644 | 113 | vma_interval_tree_remove(vma, &mapping->i_mmap); |
1da177e4 LT |
114 | flush_dcache_mmap_unlock(mapping); |
115 | } | |
116 | ||
117 | /* | |
6b2dbba8 | 118 | * Unlink a file-based vm structure from its interval tree, to hide |
a8fb5618 | 119 | * vma from rmap and vmtruncate before freeing its page tables. |
1da177e4 | 120 | */ |
a8fb5618 | 121 | void unlink_file_vma(struct vm_area_struct *vma) |
1da177e4 LT |
122 | { |
123 | struct file *file = vma->vm_file; | |
124 | ||
1da177e4 LT |
125 | if (file) { |
126 | struct address_space *mapping = file->f_mapping; | |
83cde9e8 | 127 | i_mmap_lock_write(mapping); |
1da177e4 | 128 | __remove_shared_vm_struct(vma, file, mapping); |
83cde9e8 | 129 | i_mmap_unlock_write(mapping); |
1da177e4 | 130 | } |
a8fb5618 HD |
131 | } |
132 | ||
133 | /* | |
134 | * Close a vm structure and free it, returning the next. | |
135 | */ | |
136 | static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) | |
137 | { | |
138 | struct vm_area_struct *next = vma->vm_next; | |
139 | ||
a8fb5618 | 140 | might_sleep(); |
1da177e4 LT |
141 | if (vma->vm_ops && vma->vm_ops->close) |
142 | vma->vm_ops->close(vma); | |
e9714acf | 143 | if (vma->vm_file) |
a8fb5618 | 144 | fput(vma->vm_file); |
f0be3d32 | 145 | mpol_put(vma_policy(vma)); |
3928d4f5 | 146 | vm_area_free(vma); |
a8fb5618 | 147 | return next; |
1da177e4 LT |
148 | } |
149 | ||
2e7ce7d3 LH |
150 | /* |
151 | * check_brk_limits() - Use platform specific check of range & verify mlock | |
152 | * limits. | |
153 | * @addr: The address to check | |
154 | * @len: The size of increase. | |
155 | * | |
156 | * Return: 0 on success. | |
157 | */ | |
158 | static int check_brk_limits(unsigned long addr, unsigned long len) | |
159 | { | |
160 | unsigned long mapped_addr; | |
161 | ||
162 | mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); | |
163 | if (IS_ERR_VALUE(mapped_addr)) | |
164 | return mapped_addr; | |
165 | ||
166 | return mlock_future_check(current->mm, current->mm->def_flags, len); | |
167 | } | |
168 | static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, | |
169 | unsigned long newbrk, unsigned long oldbrk, | |
170 | struct list_head *uf); | |
171 | static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *brkvma, | |
172 | unsigned long addr, unsigned long request, | |
173 | unsigned long flags); | |
6a6160a7 | 174 | SYSCALL_DEFINE1(brk, unsigned long, brk) |
1da177e4 | 175 | { |
9bc8039e | 176 | unsigned long newbrk, oldbrk, origbrk; |
1da177e4 | 177 | struct mm_struct *mm = current->mm; |
2e7ce7d3 | 178 | struct vm_area_struct *brkvma, *next = NULL; |
a5b4592c | 179 | unsigned long min_brk; |
128557ff | 180 | bool populate; |
9bc8039e | 181 | bool downgraded = false; |
897ab3e0 | 182 | LIST_HEAD(uf); |
2e7ce7d3 | 183 | MA_STATE(mas, &mm->mm_mt, 0, 0); |
1da177e4 | 184 | |
d8ed45c5 | 185 | if (mmap_write_lock_killable(mm)) |
dc0ef0df | 186 | return -EINTR; |
1da177e4 | 187 | |
9bc8039e YS |
188 | origbrk = mm->brk; |
189 | ||
a5b4592c | 190 | #ifdef CONFIG_COMPAT_BRK |
5520e894 JK |
191 | /* |
192 | * CONFIG_COMPAT_BRK can still be overridden by setting | |
193 | * randomize_va_space to 2, which will still cause mm->start_brk | |
194 | * to be arbitrarily shifted | |
195 | */ | |
4471a675 | 196 | if (current->brk_randomized) |
5520e894 JK |
197 | min_brk = mm->start_brk; |
198 | else | |
199 | min_brk = mm->end_data; | |
a5b4592c JK |
200 | #else |
201 | min_brk = mm->start_brk; | |
202 | #endif | |
203 | if (brk < min_brk) | |
1da177e4 | 204 | goto out; |
1e624196 RG |
205 | |
206 | /* | |
207 | * Check against rlimit here. If this check is done later after the test | |
208 | * of oldbrk with newbrk then it can escape the test and let the data | |
209 | * segment grow beyond its set limit the in case where the limit is | |
210 | * not page aligned -Ram Gupta | |
211 | */ | |
8764b338 CG |
212 | if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk, |
213 | mm->end_data, mm->start_data)) | |
1e624196 RG |
214 | goto out; |
215 | ||
1da177e4 LT |
216 | newbrk = PAGE_ALIGN(brk); |
217 | oldbrk = PAGE_ALIGN(mm->brk); | |
9bc8039e YS |
218 | if (oldbrk == newbrk) { |
219 | mm->brk = brk; | |
220 | goto success; | |
221 | } | |
1da177e4 | 222 | |
9bc8039e YS |
223 | /* |
224 | * Always allow shrinking brk. | |
2e7ce7d3 | 225 | * do_brk_munmap() may downgrade mmap_lock to read. |
9bc8039e | 226 | */ |
1da177e4 | 227 | if (brk <= mm->brk) { |
9bc8039e YS |
228 | int ret; |
229 | ||
2e7ce7d3 LH |
230 | /* Search one past newbrk */ |
231 | mas_set(&mas, newbrk); | |
232 | brkvma = mas_find(&mas, oldbrk); | |
233 | BUG_ON(brkvma == NULL); | |
234 | if (brkvma->vm_start >= oldbrk) | |
235 | goto out; /* mapping intersects with an existing non-brk vma. */ | |
9bc8039e | 236 | /* |
2e7ce7d3 LH |
237 | * mm->brk must be protected by write mmap_lock. |
238 | * do_brk_munmap() may downgrade the lock, so update it | |
239 | * before calling do_brk_munmap(). | |
9bc8039e YS |
240 | */ |
241 | mm->brk = brk; | |
2e7ce7d3 LH |
242 | mas.last = oldbrk - 1; |
243 | ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf); | |
244 | if (ret == 1) { | |
9bc8039e | 245 | downgraded = true; |
2e7ce7d3 LH |
246 | goto success; |
247 | } else if (!ret) | |
248 | goto success; | |
249 | ||
250 | mm->brk = origbrk; | |
251 | goto out; | |
1da177e4 LT |
252 | } |
253 | ||
2e7ce7d3 LH |
254 | if (check_brk_limits(oldbrk, newbrk - oldbrk)) |
255 | goto out; | |
256 | ||
257 | /* | |
258 | * Only check if the next VMA is within the stack_guard_gap of the | |
259 | * expansion area | |
260 | */ | |
261 | mas_set(&mas, oldbrk); | |
262 | next = mas_find(&mas, newbrk - 1 + PAGE_SIZE + stack_guard_gap); | |
1be7107f | 263 | if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) |
1da177e4 LT |
264 | goto out; |
265 | ||
2e7ce7d3 | 266 | brkvma = mas_prev(&mas, mm->start_brk); |
1da177e4 | 267 | /* Ok, looks good - let it rip. */ |
2e7ce7d3 | 268 | if (do_brk_flags(&mas, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) |
1da177e4 | 269 | goto out; |
2e7ce7d3 | 270 | |
1da177e4 | 271 | mm->brk = brk; |
9bc8039e YS |
272 | |
273 | success: | |
128557ff | 274 | populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0; |
9bc8039e | 275 | if (downgraded) |
d8ed45c5 | 276 | mmap_read_unlock(mm); |
9bc8039e | 277 | else |
d8ed45c5 | 278 | mmap_write_unlock(mm); |
897ab3e0 | 279 | userfaultfd_unmap_complete(mm, &uf); |
128557ff ML |
280 | if (populate) |
281 | mm_populate(oldbrk, newbrk - oldbrk); | |
282 | return brk; | |
283 | ||
1da177e4 | 284 | out: |
d8ed45c5 | 285 | mmap_write_unlock(mm); |
b7204006 | 286 | return origbrk; |
1da177e4 LT |
287 | } |
288 | ||
d4af56c5 LH |
289 | #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) |
290 | extern void mt_validate(struct maple_tree *mt); | |
291 | extern void mt_dump(const struct maple_tree *mt); | |
1da177e4 | 292 | |
d4af56c5 LH |
293 | /* Validate the maple tree */ |
294 | static void validate_mm_mt(struct mm_struct *mm) | |
295 | { | |
296 | struct maple_tree *mt = &mm->mm_mt; | |
297 | struct vm_area_struct *vma_mt, *vma = mm->mmap; | |
298 | ||
299 | MA_STATE(mas, mt, 0, 0); | |
300 | ||
301 | mt_validate(&mm->mm_mt); | |
302 | mas_for_each(&mas, vma_mt, ULONG_MAX) { | |
303 | if (xa_is_zero(vma_mt)) | |
304 | continue; | |
305 | ||
306 | if (!vma) | |
307 | break; | |
308 | ||
309 | if ((vma != vma_mt) || | |
310 | (vma->vm_start != vma_mt->vm_start) || | |
311 | (vma->vm_end != vma_mt->vm_end) || | |
312 | (vma->vm_start != mas.index) || | |
313 | (vma->vm_end - 1 != mas.last)) { | |
314 | pr_emerg("issue in %s\n", current->comm); | |
315 | dump_stack(); | |
d4af56c5 | 316 | dump_vma(vma_mt); |
524e00b3 | 317 | pr_emerg("and vm_next\n"); |
d4af56c5 | 318 | dump_vma(vma->vm_next); |
d4af56c5 LH |
319 | pr_emerg("mt piv: %p %lu - %lu\n", vma_mt, |
320 | mas.index, mas.last); | |
321 | pr_emerg("mt vma: %p %lu - %lu\n", vma_mt, | |
322 | vma_mt->vm_start, vma_mt->vm_end); | |
524e00b3 LH |
323 | if (vma->vm_prev) { |
324 | pr_emerg("ll prev: %p %lu - %lu\n", | |
325 | vma->vm_prev, vma->vm_prev->vm_start, | |
326 | vma->vm_prev->vm_end); | |
327 | } | |
328 | pr_emerg("ll vma: %p %lu - %lu\n", vma, | |
d4af56c5 | 329 | vma->vm_start, vma->vm_end); |
524e00b3 LH |
330 | if (vma->vm_next) { |
331 | pr_emerg("ll next: %p %lu - %lu\n", | |
332 | vma->vm_next, vma->vm_next->vm_start, | |
333 | vma->vm_next->vm_end); | |
334 | } | |
d4af56c5 LH |
335 | |
336 | mt_dump(mas.tree); | |
337 | if (vma_mt->vm_end != mas.last + 1) { | |
338 | pr_err("vma: %p vma_mt %lu-%lu\tmt %lu-%lu\n", | |
339 | mm, vma_mt->vm_start, vma_mt->vm_end, | |
340 | mas.index, mas.last); | |
341 | mt_dump(mas.tree); | |
342 | } | |
343 | VM_BUG_ON_MM(vma_mt->vm_end != mas.last + 1, mm); | |
344 | if (vma_mt->vm_start != mas.index) { | |
345 | pr_err("vma: %p vma_mt %p %lu - %lu doesn't match\n", | |
346 | mm, vma_mt, vma_mt->vm_start, vma_mt->vm_end); | |
347 | mt_dump(mas.tree); | |
348 | } | |
349 | VM_BUG_ON_MM(vma_mt->vm_start != mas.index, mm); | |
350 | } | |
351 | VM_BUG_ON(vma != vma_mt); | |
352 | vma = vma->vm_next; | |
353 | ||
354 | } | |
355 | VM_BUG_ON(vma); | |
356 | } | |
1da177e4 | 357 | |
eafd4dc4 | 358 | static void validate_mm(struct mm_struct *mm) |
1da177e4 LT |
359 | { |
360 | int bug = 0; | |
361 | int i = 0; | |
5a0768f6 | 362 | unsigned long highest_address = 0; |
ed8ea815 | 363 | struct vm_area_struct *vma = mm->mmap; |
ff26f70f | 364 | |
524e00b3 LH |
365 | validate_mm_mt(mm); |
366 | ||
ed8ea815 | 367 | while (vma) { |
524e00b3 | 368 | #ifdef CONFIG_DEBUG_VM_RB |
12352d3c | 369 | struct anon_vma *anon_vma = vma->anon_vma; |
ed8ea815 | 370 | struct anon_vma_chain *avc; |
ff26f70f | 371 | |
12352d3c KK |
372 | if (anon_vma) { |
373 | anon_vma_lock_read(anon_vma); | |
374 | list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) | |
375 | anon_vma_interval_tree_verify(avc); | |
376 | anon_vma_unlock_read(anon_vma); | |
377 | } | |
524e00b3 | 378 | #endif |
12352d3c | 379 | |
1be7107f | 380 | highest_address = vm_end_gap(vma); |
ed8ea815 | 381 | vma = vma->vm_next; |
1da177e4 LT |
382 | i++; |
383 | } | |
5a0768f6 | 384 | if (i != mm->map_count) { |
8542bdfc | 385 | pr_emerg("map_count %d vm_next %d\n", mm->map_count, i); |
5a0768f6 ML |
386 | bug = 1; |
387 | } | |
388 | if (highest_address != mm->highest_vm_end) { | |
8542bdfc | 389 | pr_emerg("mm->highest_vm_end %lx, found %lx\n", |
ff26f70f | 390 | mm->highest_vm_end, highest_address); |
5a0768f6 ML |
391 | bug = 1; |
392 | } | |
96dad67f | 393 | VM_BUG_ON_MM(bug, mm); |
1da177e4 | 394 | } |
524e00b3 LH |
395 | |
396 | #else /* !CONFIG_DEBUG_VM_MAPLE_TREE */ | |
d4af56c5 | 397 | #define validate_mm_mt(root) do { } while (0) |
1da177e4 | 398 | #define validate_mm(mm) do { } while (0) |
524e00b3 | 399 | #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ |
8f26e0b1 | 400 | |
bf181b9f ML |
401 | /* |
402 | * vma has some anon_vma assigned, and is already inserted on that | |
403 | * anon_vma's interval trees. | |
404 | * | |
405 | * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the | |
406 | * vma must be removed from the anon_vma's interval trees using | |
407 | * anon_vma_interval_tree_pre_update_vma(). | |
408 | * | |
409 | * After the update, the vma will be reinserted using | |
410 | * anon_vma_interval_tree_post_update_vma(). | |
411 | * | |
c1e8d7c6 | 412 | * The entire update must be protected by exclusive mmap_lock and by |
bf181b9f ML |
413 | * the root anon_vma's mutex. |
414 | */ | |
415 | static inline void | |
416 | anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) | |
417 | { | |
418 | struct anon_vma_chain *avc; | |
419 | ||
420 | list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) | |
421 | anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); | |
422 | } | |
423 | ||
424 | static inline void | |
425 | anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) | |
426 | { | |
427 | struct anon_vma_chain *avc; | |
428 | ||
429 | list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) | |
430 | anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); | |
431 | } | |
432 | ||
524e00b3 LH |
433 | /* |
434 | * range_has_overlap() - Check the @start - @end range for overlapping VMAs and | |
435 | * sets up a pointer to the previous VMA | |
436 | * @mm: the mm struct | |
437 | * @start: the start address of the range | |
438 | * @end: the end address of the range | |
439 | * @pprev: the pointer to the pointer of the previous VMA | |
440 | * | |
441 | * Returns: True if there is an overlapping VMA, false otherwise | |
442 | */ | |
443 | static inline | |
444 | bool range_has_overlap(struct mm_struct *mm, unsigned long start, | |
445 | unsigned long end, struct vm_area_struct **pprev) | |
1da177e4 | 446 | { |
524e00b3 | 447 | struct vm_area_struct *existing; |
1da177e4 | 448 | |
524e00b3 LH |
449 | MA_STATE(mas, &mm->mm_mt, start, start); |
450 | existing = mas_find(&mas, end - 1); | |
451 | *pprev = mas_prev(&mas, 0); | |
452 | return existing ? true : false; | |
1da177e4 LT |
453 | } |
454 | ||
3903b55a | 455 | /* |
f39af059 | 456 | * __vma_next() - Get the next VMA. |
3903b55a LH |
457 | * @mm: The mm_struct. |
458 | * @vma: The current vma. | |
459 | * | |
460 | * If @vma is NULL, return the first vma in the mm. | |
461 | * | |
462 | * Returns: The next VMA after @vma. | |
463 | */ | |
f39af059 | 464 | static inline struct vm_area_struct *__vma_next(struct mm_struct *mm, |
3903b55a LH |
465 | struct vm_area_struct *vma) |
466 | { | |
467 | if (!vma) | |
468 | return mm->mmap; | |
469 | ||
470 | return vma->vm_next; | |
471 | } | |
fb8090b6 | 472 | |
e8420a8e CH |
473 | static unsigned long count_vma_pages_range(struct mm_struct *mm, |
474 | unsigned long addr, unsigned long end) | |
475 | { | |
2e3af1db | 476 | VMA_ITERATOR(vmi, mm, addr); |
e8420a8e | 477 | struct vm_area_struct *vma; |
2e3af1db | 478 | unsigned long nr_pages = 0; |
e8420a8e | 479 | |
2e3af1db MWO |
480 | for_each_vma_range(vmi, vma, end) { |
481 | unsigned long vm_start = max(addr, vma->vm_start); | |
482 | unsigned long vm_end = min(end, vma->vm_end); | |
e8420a8e | 483 | |
2e3af1db | 484 | nr_pages += PHYS_PFN(vm_end - vm_start); |
e8420a8e CH |
485 | } |
486 | ||
487 | return nr_pages; | |
488 | } | |
489 | ||
cb8f488c | 490 | static void __vma_link_file(struct vm_area_struct *vma) |
1da177e4 | 491 | { |
48aae425 | 492 | struct file *file; |
1da177e4 LT |
493 | |
494 | file = vma->vm_file; | |
495 | if (file) { | |
496 | struct address_space *mapping = file->f_mapping; | |
497 | ||
1da177e4 | 498 | if (vma->vm_flags & VM_SHARED) |
cf508b58 | 499 | mapping_allow_writable(mapping); |
1da177e4 LT |
500 | |
501 | flush_dcache_mmap_lock(mapping); | |
27ba0644 | 502 | vma_interval_tree_insert(vma, &mapping->i_mmap); |
1da177e4 LT |
503 | flush_dcache_mmap_unlock(mapping); |
504 | } | |
505 | } | |
506 | ||
d4af56c5 LH |
507 | /* |
508 | * vma_mas_store() - Store a VMA in the maple tree. | |
509 | * @vma: The vm_area_struct | |
510 | * @mas: The maple state | |
511 | * | |
512 | * Efficient way to store a VMA in the maple tree when the @mas has already | |
513 | * walked to the correct location. | |
514 | * | |
515 | * Note: the end address is inclusive in the maple tree. | |
516 | */ | |
517 | void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) | |
518 | { | |
519 | trace_vma_store(mas->tree, vma); | |
520 | mas_set_range(mas, vma->vm_start, vma->vm_end - 1); | |
521 | mas_store_prealloc(mas, vma); | |
522 | } | |
523 | ||
524 | /* | |
525 | * vma_mas_remove() - Remove a VMA from the maple tree. | |
526 | * @vma: The vm_area_struct | |
527 | * @mas: The maple state | |
528 | * | |
529 | * Efficient way to remove a VMA from the maple tree when the @mas has already | |
530 | * been established and points to the correct location. | |
531 | * Note: the end address is inclusive in the maple tree. | |
532 | */ | |
533 | void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) | |
534 | { | |
535 | trace_vma_mas_szero(mas->tree, vma->vm_start, vma->vm_end - 1); | |
536 | mas->index = vma->vm_start; | |
537 | mas->last = vma->vm_end - 1; | |
538 | mas_store_prealloc(mas, NULL); | |
539 | } | |
540 | ||
541 | /* | |
542 | * vma_mas_szero() - Set a given range to zero. Used when modifying a | |
543 | * vm_area_struct start or end. | |
544 | * | |
545 | * @mm: The struct_mm | |
546 | * @start: The start address to zero | |
547 | * @end: The end address to zero. | |
548 | */ | |
549 | static inline void vma_mas_szero(struct ma_state *mas, unsigned long start, | |
550 | unsigned long end) | |
551 | { | |
552 | trace_vma_mas_szero(mas->tree, start, end - 1); | |
553 | mas_set_range(mas, start, end - 1); | |
554 | mas_store_prealloc(mas, NULL); | |
555 | } | |
556 | ||
d4af56c5 | 557 | static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma, |
524e00b3 | 558 | struct vm_area_struct *prev) |
1da177e4 | 559 | { |
d4af56c5 | 560 | MA_STATE(mas, &mm->mm_mt, 0, 0); |
1da177e4 LT |
561 | struct address_space *mapping = NULL; |
562 | ||
d4af56c5 LH |
563 | if (mas_preallocate(&mas, vma, GFP_KERNEL)) |
564 | return -ENOMEM; | |
565 | ||
64ac4940 | 566 | if (vma->vm_file) { |
1da177e4 | 567 | mapping = vma->vm_file->f_mapping; |
83cde9e8 | 568 | i_mmap_lock_write(mapping); |
64ac4940 | 569 | } |
1da177e4 | 570 | |
d4af56c5 | 571 | vma_mas_store(vma, &mas); |
524e00b3 | 572 | __vma_link_list(mm, vma, prev); |
1da177e4 LT |
573 | __vma_link_file(vma); |
574 | ||
1da177e4 | 575 | if (mapping) |
83cde9e8 | 576 | i_mmap_unlock_write(mapping); |
1da177e4 LT |
577 | |
578 | mm->map_count++; | |
579 | validate_mm(mm); | |
d4af56c5 | 580 | return 0; |
1da177e4 LT |
581 | } |
582 | ||
583 | /* | |
88f6b4c3 | 584 | * Helper for vma_adjust() in the split_vma insert case: insert a vma into the |
524e00b3 | 585 | * mm's list and the mm tree. It has already been inserted into the interval tree. |
1da177e4 | 586 | */ |
d4af56c5 | 587 | static void __insert_vm_struct(struct mm_struct *mm, struct ma_state *mas, |
3b0e81a1 | 588 | struct vm_area_struct *vma, unsigned long location) |
1da177e4 | 589 | { |
6597d783 | 590 | struct vm_area_struct *prev; |
d4af56c5 | 591 | |
3b0e81a1 | 592 | mas_set(mas, location); |
524e00b3 | 593 | prev = mas_prev(mas, 0); |
d4af56c5 LH |
594 | vma_mas_store(vma, mas); |
595 | __vma_link_list(mm, vma, prev); | |
1da177e4 LT |
596 | mm->map_count++; |
597 | } | |
598 | ||
4dd1b841 LH |
599 | /* |
600 | * vma_expand - Expand an existing VMA | |
601 | * | |
602 | * @mas: The maple state | |
603 | * @vma: The vma to expand | |
604 | * @start: The start of the vma | |
605 | * @end: The exclusive end of the vma | |
606 | * @pgoff: The page offset of vma | |
607 | * @next: The current of next vma. | |
608 | * | |
609 | * Expand @vma to @start and @end. Can expand off the start and end. Will | |
610 | * expand over @next if it's different from @vma and @end == @next->vm_end. | |
611 | * Checking if the @vma can expand and merge with @next needs to be handled by | |
612 | * the caller. | |
613 | * | |
614 | * Returns: 0 on success | |
615 | */ | |
616 | inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma, | |
617 | unsigned long start, unsigned long end, pgoff_t pgoff, | |
618 | struct vm_area_struct *next) | |
619 | { | |
620 | struct mm_struct *mm = vma->vm_mm; | |
621 | struct address_space *mapping = NULL; | |
622 | struct rb_root_cached *root = NULL; | |
623 | struct anon_vma *anon_vma = vma->anon_vma; | |
624 | struct file *file = vma->vm_file; | |
625 | bool remove_next = false; | |
626 | ||
627 | if (next && (vma != next) && (end == next->vm_end)) { | |
628 | remove_next = true; | |
629 | if (next->anon_vma && !vma->anon_vma) { | |
630 | int error; | |
631 | ||
632 | anon_vma = next->anon_vma; | |
633 | vma->anon_vma = anon_vma; | |
634 | error = anon_vma_clone(vma, next); | |
635 | if (error) | |
636 | return error; | |
637 | } | |
638 | } | |
639 | ||
640 | /* Not merging but overwriting any part of next is not handled. */ | |
641 | VM_BUG_ON(next && !remove_next && next != vma && end > next->vm_start); | |
642 | /* Only handles expanding */ | |
643 | VM_BUG_ON(vma->vm_start < start || vma->vm_end > end); | |
644 | ||
645 | if (mas_preallocate(mas, vma, GFP_KERNEL)) | |
646 | goto nomem; | |
647 | ||
648 | vma_adjust_trans_huge(vma, start, end, 0); | |
649 | ||
650 | if (file) { | |
651 | mapping = file->f_mapping; | |
652 | root = &mapping->i_mmap; | |
653 | uprobe_munmap(vma, vma->vm_start, vma->vm_end); | |
654 | i_mmap_lock_write(mapping); | |
655 | } | |
656 | ||
657 | if (anon_vma) { | |
658 | anon_vma_lock_write(anon_vma); | |
659 | anon_vma_interval_tree_pre_update_vma(vma); | |
660 | } | |
661 | ||
662 | if (file) { | |
663 | flush_dcache_mmap_lock(mapping); | |
664 | vma_interval_tree_remove(vma, root); | |
665 | } | |
666 | ||
667 | vma->vm_start = start; | |
668 | vma->vm_end = end; | |
669 | vma->vm_pgoff = pgoff; | |
670 | /* Note: mas must be pointing to the expanding VMA */ | |
671 | vma_mas_store(vma, mas); | |
672 | ||
673 | if (file) { | |
674 | vma_interval_tree_insert(vma, root); | |
675 | flush_dcache_mmap_unlock(mapping); | |
676 | } | |
677 | ||
678 | /* Expanding over the next vma */ | |
679 | if (remove_next) { | |
680 | /* Remove from mm linked list - also updates highest_vm_end */ | |
681 | __vma_unlink_list(mm, next); | |
682 | ||
683 | /* Kill the cache */ | |
684 | vmacache_invalidate(mm); | |
685 | ||
686 | if (file) | |
687 | __remove_shared_vm_struct(next, file, mapping); | |
688 | ||
689 | } else if (!next) { | |
690 | mm->highest_vm_end = vm_end_gap(vma); | |
691 | } | |
692 | ||
693 | if (anon_vma) { | |
694 | anon_vma_interval_tree_post_update_vma(vma); | |
695 | anon_vma_unlock_write(anon_vma); | |
696 | } | |
697 | ||
698 | if (file) { | |
699 | i_mmap_unlock_write(mapping); | |
700 | uprobe_mmap(vma); | |
701 | } | |
702 | ||
703 | if (remove_next) { | |
704 | if (file) { | |
705 | uprobe_munmap(next, next->vm_start, next->vm_end); | |
706 | fput(file); | |
707 | } | |
708 | if (next->anon_vma) | |
709 | anon_vma_merge(vma, next); | |
710 | mm->map_count--; | |
711 | mpol_put(vma_policy(next)); | |
712 | vm_area_free(next); | |
713 | } | |
714 | ||
715 | validate_mm(mm); | |
716 | return 0; | |
717 | ||
718 | nomem: | |
719 | return -ENOMEM; | |
720 | } | |
721 | ||
1da177e4 LT |
722 | /* |
723 | * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that | |
724 | * is already present in an i_mmap tree without adjusting the tree. | |
725 | * The following helper function should be used when such adjustments | |
726 | * are necessary. The "insert" vma (if any) is to be inserted | |
727 | * before we drop the necessary locks. | |
728 | */ | |
e86f15ee AA |
729 | int __vma_adjust(struct vm_area_struct *vma, unsigned long start, |
730 | unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert, | |
731 | struct vm_area_struct *expand) | |
1da177e4 LT |
732 | { |
733 | struct mm_struct *mm = vma->vm_mm; | |
524e00b3 LH |
734 | struct vm_area_struct *next_next, *next = find_vma(mm, vma->vm_end); |
735 | struct vm_area_struct *orig_vma = vma; | |
1da177e4 | 736 | struct address_space *mapping = NULL; |
f808c13f | 737 | struct rb_root_cached *root = NULL; |
012f1800 | 738 | struct anon_vma *anon_vma = NULL; |
1da177e4 | 739 | struct file *file = vma->vm_file; |
524e00b3 | 740 | bool vma_changed = false; |
1da177e4 LT |
741 | long adjust_next = 0; |
742 | int remove_next = 0; | |
d4af56c5 LH |
743 | MA_STATE(mas, &mm->mm_mt, 0, 0); |
744 | struct vm_area_struct *exporter = NULL, *importer = NULL; | |
3b0e81a1 | 745 | unsigned long ll_prev = vma->vm_start; /* linked list prev. */ |
1da177e4 | 746 | |
d4af56c5 | 747 | if (next && !insert) { |
1da177e4 LT |
748 | if (end >= next->vm_end) { |
749 | /* | |
750 | * vma expands, overlapping all the next, and | |
751 | * perhaps the one after too (mprotect case 6). | |
86d12e47 | 752 | * The only other cases that gets here are |
e86f15ee | 753 | * case 1, case 7 and case 8. |
1da177e4 | 754 | */ |
e86f15ee AA |
755 | if (next == expand) { |
756 | /* | |
757 | * The only case where we don't expand "vma" | |
758 | * and we expand "next" instead is case 8. | |
759 | */ | |
760 | VM_WARN_ON(end != next->vm_end); | |
761 | /* | |
762 | * remove_next == 3 means we're | |
763 | * removing "vma" and that to do so we | |
764 | * swapped "vma" and "next". | |
765 | */ | |
766 | remove_next = 3; | |
767 | VM_WARN_ON(file != next->vm_file); | |
768 | swap(vma, next); | |
769 | } else { | |
770 | VM_WARN_ON(expand != vma); | |
771 | /* | |
772 | * case 1, 6, 7, remove_next == 2 is case 6, | |
773 | * remove_next == 1 is case 1 or 7. | |
774 | */ | |
775 | remove_next = 1 + (end > next->vm_end); | |
d4af56c5 LH |
776 | if (remove_next == 2) |
777 | next_next = find_vma(mm, next->vm_end); | |
778 | ||
e86f15ee AA |
779 | VM_WARN_ON(remove_next == 2 && |
780 | end != next->vm_next->vm_end); | |
e86f15ee AA |
781 | } |
782 | ||
287d97ac | 783 | exporter = next; |
1da177e4 | 784 | importer = vma; |
734537c9 KS |
785 | |
786 | /* | |
787 | * If next doesn't have anon_vma, import from vma after | |
788 | * next, if the vma overlaps with it. | |
789 | */ | |
97a42cd4 | 790 | if (remove_next == 2 && !next->anon_vma) |
734537c9 KS |
791 | exporter = next->vm_next; |
792 | ||
1da177e4 LT |
793 | } else if (end > next->vm_start) { |
794 | /* | |
795 | * vma expands, overlapping part of the next: | |
796 | * mprotect case 5 shifting the boundary up. | |
797 | */ | |
f9d86a60 | 798 | adjust_next = (end - next->vm_start); |
287d97ac | 799 | exporter = next; |
1da177e4 | 800 | importer = vma; |
e86f15ee | 801 | VM_WARN_ON(expand != importer); |
1da177e4 LT |
802 | } else if (end < vma->vm_end) { |
803 | /* | |
804 | * vma shrinks, and !insert tells it's not | |
805 | * split_vma inserting another: so it must be | |
806 | * mprotect case 4 shifting the boundary down. | |
807 | */ | |
f9d86a60 | 808 | adjust_next = -(vma->vm_end - end); |
287d97ac | 809 | exporter = vma; |
1da177e4 | 810 | importer = next; |
e86f15ee | 811 | VM_WARN_ON(expand != importer); |
1da177e4 | 812 | } |
1da177e4 | 813 | |
5beb4930 RR |
814 | /* |
815 | * Easily overlooked: when mprotect shifts the boundary, | |
816 | * make sure the expanding vma has anon_vma set if the | |
817 | * shrinking vma had, to cover any anon pages imported. | |
818 | */ | |
287d97ac | 819 | if (exporter && exporter->anon_vma && !importer->anon_vma) { |
c4ea95d7 DF |
820 | int error; |
821 | ||
b800c91a | 822 | importer->anon_vma = exporter->anon_vma; |
c4ea95d7 | 823 | error = anon_vma_clone(importer, exporter); |
3fe89b3e | 824 | if (error) |
c4ea95d7 | 825 | return error; |
5beb4930 RR |
826 | } |
827 | } | |
37f9f559 | 828 | |
d4af56c5 LH |
829 | if (mas_preallocate(&mas, vma, GFP_KERNEL)) |
830 | return -ENOMEM; | |
831 | ||
832 | vma_adjust_trans_huge(orig_vma, start, end, adjust_next); | |
1da177e4 LT |
833 | if (file) { |
834 | mapping = file->f_mapping; | |
27ba0644 KS |
835 | root = &mapping->i_mmap; |
836 | uprobe_munmap(vma, vma->vm_start, vma->vm_end); | |
682968e0 | 837 | |
27ba0644 KS |
838 | if (adjust_next) |
839 | uprobe_munmap(next, next->vm_start, next->vm_end); | |
682968e0 | 840 | |
83cde9e8 | 841 | i_mmap_lock_write(mapping); |
1da177e4 | 842 | if (insert) { |
1da177e4 | 843 | /* |
6b2dbba8 | 844 | * Put into interval tree now, so instantiated pages |
1da177e4 LT |
845 | * are visible to arm/parisc __flush_dcache_page |
846 | * throughout; but we cannot insert into address | |
847 | * space until vma start or end is updated. | |
848 | */ | |
849 | __vma_link_file(insert); | |
850 | } | |
851 | } | |
852 | ||
bf181b9f ML |
853 | anon_vma = vma->anon_vma; |
854 | if (!anon_vma && adjust_next) | |
855 | anon_vma = next->anon_vma; | |
856 | if (anon_vma) { | |
e86f15ee AA |
857 | VM_WARN_ON(adjust_next && next->anon_vma && |
858 | anon_vma != next->anon_vma); | |
4fc3f1d6 | 859 | anon_vma_lock_write(anon_vma); |
bf181b9f ML |
860 | anon_vma_interval_tree_pre_update_vma(vma); |
861 | if (adjust_next) | |
862 | anon_vma_interval_tree_pre_update_vma(next); | |
863 | } | |
012f1800 | 864 | |
0fc48a6e | 865 | if (file) { |
1da177e4 | 866 | flush_dcache_mmap_lock(mapping); |
6b2dbba8 | 867 | vma_interval_tree_remove(vma, root); |
1da177e4 | 868 | if (adjust_next) |
6b2dbba8 | 869 | vma_interval_tree_remove(next, root); |
1da177e4 LT |
870 | } |
871 | ||
d3737187 | 872 | if (start != vma->vm_start) { |
3b0e81a1 LH |
873 | if ((vma->vm_start < start) && |
874 | (!insert || (insert->vm_end != start))) { | |
524e00b3 | 875 | vma_mas_szero(&mas, vma->vm_start, start); |
3b0e81a1 LH |
876 | VM_WARN_ON(insert && insert->vm_start > vma->vm_start); |
877 | } else { | |
878 | vma_changed = true; | |
879 | } | |
d3737187 | 880 | vma->vm_start = start; |
d3737187 ML |
881 | } |
882 | if (end != vma->vm_end) { | |
3b0e81a1 LH |
883 | if (vma->vm_end > end) { |
884 | if (!insert || (insert->vm_start != end)) { | |
885 | vma_mas_szero(&mas, end, vma->vm_end); | |
886 | VM_WARN_ON(insert && | |
887 | insert->vm_end < vma->vm_end); | |
888 | } else if (insert->vm_start == end) { | |
889 | ll_prev = vma->vm_end; | |
890 | } | |
891 | } else { | |
892 | vma_changed = true; | |
893 | } | |
d3737187 | 894 | vma->vm_end = end; |
524e00b3 LH |
895 | if (!next) |
896 | mm->highest_vm_end = vm_end_gap(vma); | |
d3737187 | 897 | } |
d4af56c5 | 898 | |
524e00b3 | 899 | if (vma_changed) |
d4af56c5 LH |
900 | vma_mas_store(vma, &mas); |
901 | ||
1da177e4 LT |
902 | vma->vm_pgoff = pgoff; |
903 | if (adjust_next) { | |
f9d86a60 WY |
904 | next->vm_start += adjust_next; |
905 | next->vm_pgoff += adjust_next >> PAGE_SHIFT; | |
d4af56c5 | 906 | vma_mas_store(next, &mas); |
1da177e4 LT |
907 | } |
908 | ||
0fc48a6e | 909 | if (file) { |
1da177e4 | 910 | if (adjust_next) |
6b2dbba8 ML |
911 | vma_interval_tree_insert(next, root); |
912 | vma_interval_tree_insert(vma, root); | |
1da177e4 LT |
913 | flush_dcache_mmap_unlock(mapping); |
914 | } | |
915 | ||
916 | if (remove_next) { | |
917 | /* | |
918 | * vma_merge has merged next into vma, and needs | |
919 | * us to remove next before dropping the locks. | |
d4af56c5 LH |
920 | * Since we have expanded over this vma, the maple tree will |
921 | * have overwritten by storing the value | |
1da177e4 | 922 | */ |
524e00b3 LH |
923 | __vma_unlink_list(mm, next); |
924 | if (remove_next == 2) | |
925 | __vma_unlink_list(mm, next_next); | |
926 | /* Kill the cache */ | |
927 | vmacache_invalidate(mm); | |
928 | ||
d4af56c5 | 929 | if (file) { |
1da177e4 | 930 | __remove_shared_vm_struct(next, file, mapping); |
d4af56c5 LH |
931 | if (remove_next == 2) |
932 | __remove_shared_vm_struct(next_next, file, mapping); | |
933 | } | |
1da177e4 LT |
934 | } else if (insert) { |
935 | /* | |
936 | * split_vma has split insert from vma, and needs | |
937 | * us to insert it before dropping the locks | |
938 | * (it may either follow vma or precede it). | |
939 | */ | |
3b0e81a1 | 940 | __insert_vm_struct(mm, &mas, insert, ll_prev); |
1da177e4 LT |
941 | } |
942 | ||
bf181b9f ML |
943 | if (anon_vma) { |
944 | anon_vma_interval_tree_post_update_vma(vma); | |
945 | if (adjust_next) | |
946 | anon_vma_interval_tree_post_update_vma(next); | |
08b52706 | 947 | anon_vma_unlock_write(anon_vma); |
bf181b9f | 948 | } |
1da177e4 | 949 | |
0fc48a6e | 950 | if (file) { |
808fbdbe | 951 | i_mmap_unlock_write(mapping); |
7b2d81d4 | 952 | uprobe_mmap(vma); |
2b144498 SD |
953 | |
954 | if (adjust_next) | |
7b2d81d4 | 955 | uprobe_mmap(next); |
2b144498 SD |
956 | } |
957 | ||
1da177e4 | 958 | if (remove_next) { |
d4af56c5 | 959 | again: |
925d1c40 | 960 | if (file) { |
cbc91f71 | 961 | uprobe_munmap(next, next->vm_start, next->vm_end); |
1da177e4 | 962 | fput(file); |
925d1c40 | 963 | } |
5beb4930 RR |
964 | if (next->anon_vma) |
965 | anon_vma_merge(vma, next); | |
1da177e4 | 966 | mm->map_count--; |
3964acd0 | 967 | mpol_put(vma_policy(next)); |
524e00b3 LH |
968 | if (remove_next != 2) |
969 | BUG_ON(vma->vm_end < next->vm_end); | |
3928d4f5 | 970 | vm_area_free(next); |
524e00b3 | 971 | |
1da177e4 LT |
972 | /* |
973 | * In mprotect's case 6 (see comments on vma_merge), | |
974 | * we must remove another next too. It would clutter | |
975 | * up the code too much to do both in one go. | |
976 | */ | |
e86f15ee AA |
977 | if (remove_next != 3) { |
978 | /* | |
979 | * If "next" was removed and vma->vm_end was | |
980 | * expanded (up) over it, in turn | |
981 | * "next->vm_prev->vm_end" changed and the | |
982 | * "vma->vm_next" gap must be updated. | |
983 | */ | |
d4af56c5 | 984 | next = next_next; |
e86f15ee AA |
985 | } else { |
986 | /* | |
987 | * For the scope of the comment "next" and | |
988 | * "vma" considered pre-swap(): if "vma" was | |
989 | * removed, next->vm_start was expanded (down) | |
990 | * over it and the "next" gap must be updated. | |
991 | * Because of the swap() the post-swap() "vma" | |
992 | * actually points to pre-swap() "next" | |
993 | * (post-swap() "next" as opposed is now a | |
994 | * dangling pointer). | |
995 | */ | |
996 | next = vma; | |
997 | } | |
734537c9 KS |
998 | if (remove_next == 2) { |
999 | remove_next = 1; | |
1da177e4 | 1000 | goto again; |
524e00b3 | 1001 | } else if (!next) { |
fb8c41e9 AA |
1002 | /* |
1003 | * If remove_next == 2 we obviously can't | |
1004 | * reach this path. | |
1005 | * | |
1006 | * If remove_next == 3 we can't reach this | |
1007 | * path because pre-swap() next is always not | |
1008 | * NULL. pre-swap() "next" is not being | |
1009 | * removed and its next->vm_end is not altered | |
1010 | * (and furthermore "end" already matches | |
1011 | * next->vm_end in remove_next == 3). | |
1012 | * | |
1013 | * We reach this only in the remove_next == 1 | |
1014 | * case if the "next" vma that was removed was | |
1015 | * the highest vma of the mm. However in such | |
1016 | * case next->vm_end == "end" and the extended | |
1017 | * "vma" has vma->vm_end == next->vm_end so | |
1018 | * mm->highest_vm_end doesn't need any update | |
1019 | * in remove_next == 1 case. | |
1020 | */ | |
1be7107f | 1021 | VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma)); |
fb8c41e9 | 1022 | } |
1da177e4 | 1023 | } |
2b144498 | 1024 | if (insert && file) |
7b2d81d4 | 1025 | uprobe_mmap(insert); |
1da177e4 | 1026 | |
3b0e81a1 | 1027 | mas_destroy(&mas); |
1da177e4 | 1028 | validate_mm(mm); |
5beb4930 | 1029 | return 0; |
1da177e4 LT |
1030 | } |
1031 | ||
1032 | /* | |
1033 | * If the vma has a ->close operation then the driver probably needs to release | |
1034 | * per-vma resources, so we don't attempt to merge those. | |
1035 | */ | |
1da177e4 | 1036 | static inline int is_mergeable_vma(struct vm_area_struct *vma, |
19a809af | 1037 | struct file *file, unsigned long vm_flags, |
9a10064f | 1038 | struct vm_userfaultfd_ctx vm_userfaultfd_ctx, |
5c26f6ac | 1039 | struct anon_vma_name *anon_name) |
1da177e4 | 1040 | { |
34228d47 CG |
1041 | /* |
1042 | * VM_SOFTDIRTY should not prevent from VMA merging, if we | |
1043 | * match the flags but dirty bit -- the caller should mark | |
1044 | * merged VMA as dirty. If dirty bit won't be excluded from | |
8bb4e7a2 | 1045 | * comparison, we increase pressure on the memory system forcing |
34228d47 CG |
1046 | * the kernel to generate new VMAs when old one could be |
1047 | * extended instead. | |
1048 | */ | |
1049 | if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) | |
1da177e4 LT |
1050 | return 0; |
1051 | if (vma->vm_file != file) | |
1052 | return 0; | |
1053 | if (vma->vm_ops && vma->vm_ops->close) | |
1054 | return 0; | |
19a809af AA |
1055 | if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) |
1056 | return 0; | |
5c26f6ac | 1057 | if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) |
9a10064f | 1058 | return 0; |
1da177e4 LT |
1059 | return 1; |
1060 | } | |
1061 | ||
1062 | static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1, | |
965f55de SL |
1063 | struct anon_vma *anon_vma2, |
1064 | struct vm_area_struct *vma) | |
1da177e4 | 1065 | { |
965f55de SL |
1066 | /* |
1067 | * The list_is_singular() test is to avoid merging VMA cloned from | |
1068 | * parents. This can improve scalability caused by anon_vma lock. | |
1069 | */ | |
1070 | if ((!anon_vma1 || !anon_vma2) && (!vma || | |
1071 | list_is_singular(&vma->anon_vma_chain))) | |
1072 | return 1; | |
1073 | return anon_vma1 == anon_vma2; | |
1da177e4 LT |
1074 | } |
1075 | ||
1076 | /* | |
1077 | * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) | |
1078 | * in front of (at a lower virtual address and file offset than) the vma. | |
1079 | * | |
1080 | * We cannot merge two vmas if they have differently assigned (non-NULL) | |
1081 | * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. | |
1082 | * | |
1083 | * We don't check here for the merged mmap wrapping around the end of pagecache | |
45e55300 | 1084 | * indices (16TB on ia32) because do_mmap() does not permit mmap's which |
1da177e4 LT |
1085 | * wrap, nor mmaps which cover the final page at index -1UL. |
1086 | */ | |
1087 | static int | |
1088 | can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, | |
19a809af AA |
1089 | struct anon_vma *anon_vma, struct file *file, |
1090 | pgoff_t vm_pgoff, | |
9a10064f | 1091 | struct vm_userfaultfd_ctx vm_userfaultfd_ctx, |
5c26f6ac | 1092 | struct anon_vma_name *anon_name) |
1da177e4 | 1093 | { |
9a10064f | 1094 | if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && |
965f55de | 1095 | is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { |
1da177e4 LT |
1096 | if (vma->vm_pgoff == vm_pgoff) |
1097 | return 1; | |
1098 | } | |
1099 | return 0; | |
1100 | } | |
1101 | ||
1102 | /* | |
1103 | * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) | |
1104 | * beyond (at a higher virtual address and file offset than) the vma. | |
1105 | * | |
1106 | * We cannot merge two vmas if they have differently assigned (non-NULL) | |
1107 | * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. | |
1108 | */ | |
1109 | static int | |
1110 | can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, | |
19a809af AA |
1111 | struct anon_vma *anon_vma, struct file *file, |
1112 | pgoff_t vm_pgoff, | |
9a10064f | 1113 | struct vm_userfaultfd_ctx vm_userfaultfd_ctx, |
5c26f6ac | 1114 | struct anon_vma_name *anon_name) |
1da177e4 | 1115 | { |
9a10064f | 1116 | if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && |
965f55de | 1117 | is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { |
1da177e4 | 1118 | pgoff_t vm_pglen; |
d6e93217 | 1119 | vm_pglen = vma_pages(vma); |
1da177e4 LT |
1120 | if (vma->vm_pgoff + vm_pglen == vm_pgoff) |
1121 | return 1; | |
1122 | } | |
1123 | return 0; | |
1124 | } | |
1125 | ||
1126 | /* | |
9a10064f CC |
1127 | * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), |
1128 | * figure out whether that can be merged with its predecessor or its | |
1129 | * successor. Or both (it neatly fills a hole). | |
1da177e4 LT |
1130 | * |
1131 | * In most cases - when called for mmap, brk or mremap - [addr,end) is | |
1132 | * certain not to be mapped by the time vma_merge is called; but when | |
1133 | * called for mprotect, it is certain to be already mapped (either at | |
1134 | * an offset within prev, or at the start of next), and the flags of | |
1135 | * this area are about to be changed to vm_flags - and the no-change | |
1136 | * case has already been eliminated. | |
1137 | * | |
1138 | * The following mprotect cases have to be considered, where AAAA is | |
1139 | * the area passed down from mprotect_fixup, never extending beyond one | |
1140 | * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after: | |
1141 | * | |
5d42ab29 WY |
1142 | * AAAA AAAA AAAA |
1143 | * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN | |
1144 | * cannot merge might become might become | |
1145 | * PPNNNNNNNNNN PPPPPPPPPPNN | |
1146 | * mmap, brk or case 4 below case 5 below | |
1147 | * mremap move: | |
1148 | * AAAA AAAA | |
1149 | * PPPP NNNN PPPPNNNNXXXX | |
1150 | * might become might become | |
1151 | * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or | |
1152 | * PPPPPPPPNNNN 2 or PPPPPPPPXXXX 7 or | |
1153 | * PPPPNNNNNNNN 3 PPPPXXXXXXXX 8 | |
1da177e4 | 1154 | * |
8bb4e7a2 | 1155 | * It is important for case 8 that the vma NNNN overlapping the |
e86f15ee AA |
1156 | * region AAAA is never going to extended over XXXX. Instead XXXX must |
1157 | * be extended in region AAAA and NNNN must be removed. This way in | |
1158 | * all cases where vma_merge succeeds, the moment vma_adjust drops the | |
1159 | * rmap_locks, the properties of the merged vma will be already | |
1160 | * correct for the whole merged range. Some of those properties like | |
1161 | * vm_page_prot/vm_flags may be accessed by rmap_walks and they must | |
1162 | * be correct for the whole merged range immediately after the | |
1163 | * rmap_locks are released. Otherwise if XXXX would be removed and | |
1164 | * NNNN would be extended over the XXXX range, remove_migration_ptes | |
1165 | * or other rmap walkers (if working on addresses beyond the "end" | |
1166 | * parameter) may establish ptes with the wrong permissions of NNNN | |
1167 | * instead of the right permissions of XXXX. | |
1da177e4 LT |
1168 | */ |
1169 | struct vm_area_struct *vma_merge(struct mm_struct *mm, | |
1170 | struct vm_area_struct *prev, unsigned long addr, | |
1171 | unsigned long end, unsigned long vm_flags, | |
cc71aba3 | 1172 | struct anon_vma *anon_vma, struct file *file, |
19a809af | 1173 | pgoff_t pgoff, struct mempolicy *policy, |
9a10064f | 1174 | struct vm_userfaultfd_ctx vm_userfaultfd_ctx, |
5c26f6ac | 1175 | struct anon_vma_name *anon_name) |
1da177e4 LT |
1176 | { |
1177 | pgoff_t pglen = (end - addr) >> PAGE_SHIFT; | |
1178 | struct vm_area_struct *area, *next; | |
5beb4930 | 1179 | int err; |
1da177e4 LT |
1180 | |
1181 | /* | |
1182 | * We later require that vma->vm_flags == vm_flags, | |
1183 | * so this tests vma->vm_flags & VM_SPECIAL, too. | |
1184 | */ | |
1185 | if (vm_flags & VM_SPECIAL) | |
1186 | return NULL; | |
1187 | ||
f39af059 | 1188 | next = __vma_next(mm, prev); |
1da177e4 | 1189 | area = next; |
e86f15ee | 1190 | if (area && area->vm_end == end) /* cases 6, 7, 8 */ |
1da177e4 LT |
1191 | next = next->vm_next; |
1192 | ||
e86f15ee AA |
1193 | /* verify some invariant that must be enforced by the caller */ |
1194 | VM_WARN_ON(prev && addr <= prev->vm_start); | |
1195 | VM_WARN_ON(area && end > area->vm_end); | |
1196 | VM_WARN_ON(addr >= end); | |
1197 | ||
1da177e4 LT |
1198 | /* |
1199 | * Can it merge with the predecessor? | |
1200 | */ | |
1201 | if (prev && prev->vm_end == addr && | |
cc71aba3 | 1202 | mpol_equal(vma_policy(prev), policy) && |
1da177e4 | 1203 | can_vma_merge_after(prev, vm_flags, |
19a809af | 1204 | anon_vma, file, pgoff, |
9a10064f | 1205 | vm_userfaultfd_ctx, anon_name)) { |
1da177e4 LT |
1206 | /* |
1207 | * OK, it can. Can we now merge in the successor as well? | |
1208 | */ | |
1209 | if (next && end == next->vm_start && | |
1210 | mpol_equal(policy, vma_policy(next)) && | |
1211 | can_vma_merge_before(next, vm_flags, | |
19a809af AA |
1212 | anon_vma, file, |
1213 | pgoff+pglen, | |
9a10064f | 1214 | vm_userfaultfd_ctx, anon_name) && |
1da177e4 | 1215 | is_mergeable_anon_vma(prev->anon_vma, |
965f55de | 1216 | next->anon_vma, NULL)) { |
1da177e4 | 1217 | /* cases 1, 6 */ |
e86f15ee AA |
1218 | err = __vma_adjust(prev, prev->vm_start, |
1219 | next->vm_end, prev->vm_pgoff, NULL, | |
1220 | prev); | |
1da177e4 | 1221 | } else /* cases 2, 5, 7 */ |
e86f15ee AA |
1222 | err = __vma_adjust(prev, prev->vm_start, |
1223 | end, prev->vm_pgoff, NULL, prev); | |
5beb4930 RR |
1224 | if (err) |
1225 | return NULL; | |
c791576c | 1226 | khugepaged_enter_vma(prev, vm_flags); |
1da177e4 LT |
1227 | return prev; |
1228 | } | |
1229 | ||
1230 | /* | |
1231 | * Can this new request be merged in front of next? | |
1232 | */ | |
1233 | if (next && end == next->vm_start && | |
cc71aba3 | 1234 | mpol_equal(policy, vma_policy(next)) && |
1da177e4 | 1235 | can_vma_merge_before(next, vm_flags, |
19a809af | 1236 | anon_vma, file, pgoff+pglen, |
9a10064f | 1237 | vm_userfaultfd_ctx, anon_name)) { |
1da177e4 | 1238 | if (prev && addr < prev->vm_end) /* case 4 */ |
e86f15ee AA |
1239 | err = __vma_adjust(prev, prev->vm_start, |
1240 | addr, prev->vm_pgoff, NULL, next); | |
1241 | else { /* cases 3, 8 */ | |
1242 | err = __vma_adjust(area, addr, next->vm_end, | |
1243 | next->vm_pgoff - pglen, NULL, next); | |
1244 | /* | |
1245 | * In case 3 area is already equal to next and | |
1246 | * this is a noop, but in case 8 "area" has | |
1247 | * been removed and next was expanded over it. | |
1248 | */ | |
1249 | area = next; | |
1250 | } | |
5beb4930 RR |
1251 | if (err) |
1252 | return NULL; | |
c791576c | 1253 | khugepaged_enter_vma(area, vm_flags); |
1da177e4 LT |
1254 | return area; |
1255 | } | |
1256 | ||
1257 | return NULL; | |
1258 | } | |
1259 | ||
d0e9fe17 | 1260 | /* |
b4f315b4 | 1261 | * Rough compatibility check to quickly see if it's even worth looking |
d0e9fe17 LT |
1262 | * at sharing an anon_vma. |
1263 | * | |
1264 | * They need to have the same vm_file, and the flags can only differ | |
1265 | * in things that mprotect may change. | |
1266 | * | |
1267 | * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that | |
1268 | * we can merge the two vma's. For example, we refuse to merge a vma if | |
1269 | * there is a vm_ops->close() function, because that indicates that the | |
1270 | * driver is doing some kind of reference counting. But that doesn't | |
1271 | * really matter for the anon_vma sharing case. | |
1272 | */ | |
1273 | static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) | |
1274 | { | |
1275 | return a->vm_end == b->vm_start && | |
1276 | mpol_equal(vma_policy(a), vma_policy(b)) && | |
1277 | a->vm_file == b->vm_file && | |
6cb4d9a2 | 1278 | !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) && |
d0e9fe17 LT |
1279 | b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); |
1280 | } | |
1281 | ||
1282 | /* | |
1283 | * Do some basic sanity checking to see if we can re-use the anon_vma | |
1284 | * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be | |
1285 | * the same as 'old', the other will be the new one that is trying | |
1286 | * to share the anon_vma. | |
1287 | * | |
5b449489 | 1288 | * NOTE! This runs with mmap_lock held for reading, so it is possible that |
d0e9fe17 LT |
1289 | * the anon_vma of 'old' is concurrently in the process of being set up |
1290 | * by another page fault trying to merge _that_. But that's ok: if it | |
1291 | * is being set up, that automatically means that it will be a singleton | |
1292 | * acceptable for merging, so we can do all of this optimistically. But | |
4db0c3c2 | 1293 | * we do that READ_ONCE() to make sure that we never re-load the pointer. |
d0e9fe17 LT |
1294 | * |
1295 | * IOW: that the "list_is_singular()" test on the anon_vma_chain only | |
1296 | * matters for the 'stable anon_vma' case (ie the thing we want to avoid | |
1297 | * is to return an anon_vma that is "complex" due to having gone through | |
1298 | * a fork). | |
1299 | * | |
1300 | * We also make sure that the two vma's are compatible (adjacent, | |
1301 | * and with the same memory policies). That's all stable, even with just | |
5b449489 | 1302 | * a read lock on the mmap_lock. |
d0e9fe17 LT |
1303 | */ |
1304 | static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b) | |
1305 | { | |
1306 | if (anon_vma_compatible(a, b)) { | |
4db0c3c2 | 1307 | struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); |
d0e9fe17 LT |
1308 | |
1309 | if (anon_vma && list_is_singular(&old->anon_vma_chain)) | |
1310 | return anon_vma; | |
1311 | } | |
1312 | return NULL; | |
1313 | } | |
1314 | ||
1da177e4 LT |
1315 | /* |
1316 | * find_mergeable_anon_vma is used by anon_vma_prepare, to check | |
1317 | * neighbouring vmas for a suitable anon_vma, before it goes off | |
1318 | * to allocate a new anon_vma. It checks because a repetitive | |
1319 | * sequence of mprotects and faults may otherwise lead to distinct | |
1320 | * anon_vmas being allocated, preventing vma merge in subsequent | |
1321 | * mprotect. | |
1322 | */ | |
1323 | struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) | |
1324 | { | |
a67c8caa ML |
1325 | struct anon_vma *anon_vma = NULL; |
1326 | ||
1327 | /* Try next first. */ | |
1328 | if (vma->vm_next) { | |
1329 | anon_vma = reusable_anon_vma(vma->vm_next, vma, vma->vm_next); | |
1330 | if (anon_vma) | |
1331 | return anon_vma; | |
1332 | } | |
1333 | ||
1334 | /* Try prev next. */ | |
1335 | if (vma->vm_prev) | |
1336 | anon_vma = reusable_anon_vma(vma->vm_prev, vma->vm_prev, vma); | |
1337 | ||
1da177e4 | 1338 | /* |
a67c8caa ML |
1339 | * We might reach here with anon_vma == NULL if we can't find |
1340 | * any reusable anon_vma. | |
1da177e4 LT |
1341 | * There's no absolute need to look only at touching neighbours: |
1342 | * we could search further afield for "compatible" anon_vmas. | |
1343 | * But it would probably just be a waste of time searching, | |
1344 | * or lead to too many vmas hanging off the same anon_vma. | |
1345 | * We're trying to allow mprotect remerging later on, | |
1346 | * not trying to minimize memory used for anon_vmas. | |
1347 | */ | |
a67c8caa | 1348 | return anon_vma; |
1da177e4 LT |
1349 | } |
1350 | ||
40401530 AV |
1351 | /* |
1352 | * If a hint addr is less than mmap_min_addr change hint to be as | |
1353 | * low as possible but still greater than mmap_min_addr | |
1354 | */ | |
1355 | static inline unsigned long round_hint_to_min(unsigned long hint) | |
1356 | { | |
1357 | hint &= PAGE_MASK; | |
1358 | if (((void *)hint != NULL) && | |
1359 | (hint < mmap_min_addr)) | |
1360 | return PAGE_ALIGN(mmap_min_addr); | |
1361 | return hint; | |
1362 | } | |
1363 | ||
6aeb2542 MR |
1364 | int mlock_future_check(struct mm_struct *mm, unsigned long flags, |
1365 | unsigned long len) | |
363ee17f DB |
1366 | { |
1367 | unsigned long locked, lock_limit; | |
1368 | ||
1369 | /* mlock MCL_FUTURE? */ | |
1370 | if (flags & VM_LOCKED) { | |
1371 | locked = len >> PAGE_SHIFT; | |
1372 | locked += mm->locked_vm; | |
1373 | lock_limit = rlimit(RLIMIT_MEMLOCK); | |
1374 | lock_limit >>= PAGE_SHIFT; | |
1375 | if (locked > lock_limit && !capable(CAP_IPC_LOCK)) | |
1376 | return -EAGAIN; | |
1377 | } | |
1378 | return 0; | |
1379 | } | |
1380 | ||
be83bbf8 LT |
1381 | static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) |
1382 | { | |
1383 | if (S_ISREG(inode->i_mode)) | |
423913ad | 1384 | return MAX_LFS_FILESIZE; |
be83bbf8 LT |
1385 | |
1386 | if (S_ISBLK(inode->i_mode)) | |
1387 | return MAX_LFS_FILESIZE; | |
1388 | ||
76f34950 IK |
1389 | if (S_ISSOCK(inode->i_mode)) |
1390 | return MAX_LFS_FILESIZE; | |
1391 | ||
be83bbf8 LT |
1392 | /* Special "we do even unsigned file positions" case */ |
1393 | if (file->f_mode & FMODE_UNSIGNED_OFFSET) | |
1394 | return 0; | |
1395 | ||
1396 | /* Yes, random drivers might want more. But I'm tired of buggy drivers */ | |
1397 | return ULONG_MAX; | |
1398 | } | |
1399 | ||
1400 | static inline bool file_mmap_ok(struct file *file, struct inode *inode, | |
1401 | unsigned long pgoff, unsigned long len) | |
1402 | { | |
1403 | u64 maxsize = file_mmap_size_max(file, inode); | |
1404 | ||
1405 | if (maxsize && len > maxsize) | |
1406 | return false; | |
1407 | maxsize -= len; | |
1408 | if (pgoff > maxsize >> PAGE_SHIFT) | |
1409 | return false; | |
1410 | return true; | |
1411 | } | |
1412 | ||
1da177e4 | 1413 | /* |
3e4e28c5 | 1414 | * The caller must write-lock current->mm->mmap_lock. |
1da177e4 | 1415 | */ |
1fcfd8db | 1416 | unsigned long do_mmap(struct file *file, unsigned long addr, |
1da177e4 | 1417 | unsigned long len, unsigned long prot, |
45e55300 PC |
1418 | unsigned long flags, unsigned long pgoff, |
1419 | unsigned long *populate, struct list_head *uf) | |
1da177e4 | 1420 | { |
cc71aba3 | 1421 | struct mm_struct *mm = current->mm; |
45e55300 | 1422 | vm_flags_t vm_flags; |
62b5f7d0 | 1423 | int pkey = 0; |
1da177e4 | 1424 | |
524e00b3 | 1425 | validate_mm(mm); |
41badc15 | 1426 | *populate = 0; |
bebeb3d6 | 1427 | |
e37609bb PK |
1428 | if (!len) |
1429 | return -EINVAL; | |
1430 | ||
1da177e4 LT |
1431 | /* |
1432 | * Does the application expect PROT_READ to imply PROT_EXEC? | |
1433 | * | |
1434 | * (the exception is when the underlying filesystem is noexec | |
1435 | * mounted, in which case we dont add PROT_EXEC.) | |
1436 | */ | |
1437 | if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) | |
90f8572b | 1438 | if (!(file && path_noexec(&file->f_path))) |
1da177e4 LT |
1439 | prot |= PROT_EXEC; |
1440 | ||
a4ff8e86 MH |
1441 | /* force arch specific MAP_FIXED handling in get_unmapped_area */ |
1442 | if (flags & MAP_FIXED_NOREPLACE) | |
1443 | flags |= MAP_FIXED; | |
1444 | ||
7cd94146 EP |
1445 | if (!(flags & MAP_FIXED)) |
1446 | addr = round_hint_to_min(addr); | |
1447 | ||
1da177e4 LT |
1448 | /* Careful about overflows.. */ |
1449 | len = PAGE_ALIGN(len); | |
9206de95 | 1450 | if (!len) |
1da177e4 LT |
1451 | return -ENOMEM; |
1452 | ||
1453 | /* offset overflow? */ | |
1454 | if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) | |
cc71aba3 | 1455 | return -EOVERFLOW; |
1da177e4 LT |
1456 | |
1457 | /* Too many mappings? */ | |
1458 | if (mm->map_count > sysctl_max_map_count) | |
1459 | return -ENOMEM; | |
1460 | ||
1461 | /* Obtain the address to map to. we verify (or select) it and ensure | |
1462 | * that it represents a valid section of the address space. | |
1463 | */ | |
1464 | addr = get_unmapped_area(file, addr, len, pgoff, flags); | |
ff68dac6 | 1465 | if (IS_ERR_VALUE(addr)) |
1da177e4 LT |
1466 | return addr; |
1467 | ||
a4ff8e86 | 1468 | if (flags & MAP_FIXED_NOREPLACE) { |
35e43c5f | 1469 | if (find_vma_intersection(mm, addr, addr + len)) |
a4ff8e86 MH |
1470 | return -EEXIST; |
1471 | } | |
1472 | ||
62b5f7d0 DH |
1473 | if (prot == PROT_EXEC) { |
1474 | pkey = execute_only_pkey(mm); | |
1475 | if (pkey < 0) | |
1476 | pkey = 0; | |
1477 | } | |
1478 | ||
1da177e4 LT |
1479 | /* Do simple checking here so the lower-level routines won't have |
1480 | * to. we assume access permissions have been handled by the open | |
1481 | * of the memory object, so we don't do any here. | |
1482 | */ | |
45e55300 | 1483 | vm_flags = calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) | |
1da177e4 LT |
1484 | mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; |
1485 | ||
cdf7b341 | 1486 | if (flags & MAP_LOCKED) |
1da177e4 LT |
1487 | if (!can_do_mlock()) |
1488 | return -EPERM; | |
ba470de4 | 1489 | |
363ee17f DB |
1490 | if (mlock_future_check(mm, vm_flags, len)) |
1491 | return -EAGAIN; | |
1da177e4 | 1492 | |
1da177e4 | 1493 | if (file) { |
077bf22b | 1494 | struct inode *inode = file_inode(file); |
1c972597 DW |
1495 | unsigned long flags_mask; |
1496 | ||
be83bbf8 LT |
1497 | if (!file_mmap_ok(file, inode, pgoff, len)) |
1498 | return -EOVERFLOW; | |
1499 | ||
1c972597 | 1500 | flags_mask = LEGACY_MAP_MASK | file->f_op->mmap_supported_flags; |
077bf22b | 1501 | |
1da177e4 LT |
1502 | switch (flags & MAP_TYPE) { |
1503 | case MAP_SHARED: | |
1c972597 DW |
1504 | /* |
1505 | * Force use of MAP_SHARED_VALIDATE with non-legacy | |
1506 | * flags. E.g. MAP_SYNC is dangerous to use with | |
1507 | * MAP_SHARED as you don't know which consistency model | |
1508 | * you will get. We silently ignore unsupported flags | |
1509 | * with MAP_SHARED to preserve backward compatibility. | |
1510 | */ | |
1511 | flags &= LEGACY_MAP_MASK; | |
e4a9bc58 | 1512 | fallthrough; |
1c972597 DW |
1513 | case MAP_SHARED_VALIDATE: |
1514 | if (flags & ~flags_mask) | |
1515 | return -EOPNOTSUPP; | |
dc617f29 DW |
1516 | if (prot & PROT_WRITE) { |
1517 | if (!(file->f_mode & FMODE_WRITE)) | |
1518 | return -EACCES; | |
1519 | if (IS_SWAPFILE(file->f_mapping->host)) | |
1520 | return -ETXTBSY; | |
1521 | } | |
1da177e4 LT |
1522 | |
1523 | /* | |
1524 | * Make sure we don't allow writing to an append-only | |
1525 | * file.. | |
1526 | */ | |
1527 | if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) | |
1528 | return -EACCES; | |
1529 | ||
1da177e4 LT |
1530 | vm_flags |= VM_SHARED | VM_MAYSHARE; |
1531 | if (!(file->f_mode & FMODE_WRITE)) | |
1532 | vm_flags &= ~(VM_MAYWRITE | VM_SHARED); | |
e4a9bc58 | 1533 | fallthrough; |
1da177e4 LT |
1534 | case MAP_PRIVATE: |
1535 | if (!(file->f_mode & FMODE_READ)) | |
1536 | return -EACCES; | |
90f8572b | 1537 | if (path_noexec(&file->f_path)) { |
80c5606c LT |
1538 | if (vm_flags & VM_EXEC) |
1539 | return -EPERM; | |
1540 | vm_flags &= ~VM_MAYEXEC; | |
1541 | } | |
80c5606c | 1542 | |
72c2d531 | 1543 | if (!file->f_op->mmap) |
80c5606c | 1544 | return -ENODEV; |
b2c56e4f ON |
1545 | if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) |
1546 | return -EINVAL; | |
1da177e4 LT |
1547 | break; |
1548 | ||
1549 | default: | |
1550 | return -EINVAL; | |
1551 | } | |
1552 | } else { | |
1553 | switch (flags & MAP_TYPE) { | |
1554 | case MAP_SHARED: | |
b2c56e4f ON |
1555 | if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) |
1556 | return -EINVAL; | |
ce363942 TH |
1557 | /* |
1558 | * Ignore pgoff. | |
1559 | */ | |
1560 | pgoff = 0; | |
1da177e4 LT |
1561 | vm_flags |= VM_SHARED | VM_MAYSHARE; |
1562 | break; | |
1563 | case MAP_PRIVATE: | |
1564 | /* | |
1565 | * Set pgoff according to addr for anon_vma. | |
1566 | */ | |
1567 | pgoff = addr >> PAGE_SHIFT; | |
1568 | break; | |
1569 | default: | |
1570 | return -EINVAL; | |
1571 | } | |
1572 | } | |
1573 | ||
c22c0d63 ML |
1574 | /* |
1575 | * Set 'VM_NORESERVE' if we should not account for the | |
1576 | * memory use of this mapping. | |
1577 | */ | |
1578 | if (flags & MAP_NORESERVE) { | |
1579 | /* We honor MAP_NORESERVE if allowed to overcommit */ | |
1580 | if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) | |
1581 | vm_flags |= VM_NORESERVE; | |
1582 | ||
1583 | /* hugetlb applies strict overcommit unless MAP_NORESERVE */ | |
1584 | if (file && is_file_hugepages(file)) | |
1585 | vm_flags |= VM_NORESERVE; | |
1586 | } | |
1587 | ||
897ab3e0 | 1588 | addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); |
09a9f1d2 ML |
1589 | if (!IS_ERR_VALUE(addr) && |
1590 | ((vm_flags & VM_LOCKED) || | |
1591 | (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) | |
41badc15 | 1592 | *populate = len; |
bebeb3d6 | 1593 | return addr; |
0165ab44 | 1594 | } |
6be5ceb0 | 1595 | |
a90f590a DB |
1596 | unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, |
1597 | unsigned long prot, unsigned long flags, | |
1598 | unsigned long fd, unsigned long pgoff) | |
66f0dc48 HD |
1599 | { |
1600 | struct file *file = NULL; | |
1e3ee14b | 1601 | unsigned long retval; |
66f0dc48 HD |
1602 | |
1603 | if (!(flags & MAP_ANONYMOUS)) { | |
120a795d | 1604 | audit_mmap_fd(fd, flags); |
66f0dc48 HD |
1605 | file = fget(fd); |
1606 | if (!file) | |
1e3ee14b | 1607 | return -EBADF; |
7bba8f0e | 1608 | if (is_file_hugepages(file)) { |
af73e4d9 | 1609 | len = ALIGN(len, huge_page_size(hstate_file(file))); |
7bba8f0e ZL |
1610 | } else if (unlikely(flags & MAP_HUGETLB)) { |
1611 | retval = -EINVAL; | |
493af578 | 1612 | goto out_fput; |
7bba8f0e | 1613 | } |
66f0dc48 | 1614 | } else if (flags & MAP_HUGETLB) { |
c103a4dc | 1615 | struct hstate *hs; |
af73e4d9 | 1616 | |
20ac2893 | 1617 | hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); |
091d0d55 LZ |
1618 | if (!hs) |
1619 | return -EINVAL; | |
1620 | ||
1621 | len = ALIGN(len, huge_page_size(hs)); | |
66f0dc48 HD |
1622 | /* |
1623 | * VM_NORESERVE is used because the reservations will be | |
1624 | * taken when vm_ops->mmap() is called | |
66f0dc48 | 1625 | */ |
af73e4d9 | 1626 | file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, |
42d7395f | 1627 | VM_NORESERVE, |
83c1fd76 | 1628 | HUGETLB_ANONHUGE_INODE, |
42d7395f | 1629 | (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); |
66f0dc48 HD |
1630 | if (IS_ERR(file)) |
1631 | return PTR_ERR(file); | |
1632 | } | |
1633 | ||
9fbeb5ab | 1634 | retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); |
493af578 | 1635 | out_fput: |
66f0dc48 HD |
1636 | if (file) |
1637 | fput(file); | |
66f0dc48 HD |
1638 | return retval; |
1639 | } | |
1640 | ||
a90f590a DB |
1641 | SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, |
1642 | unsigned long, prot, unsigned long, flags, | |
1643 | unsigned long, fd, unsigned long, pgoff) | |
1644 | { | |
1645 | return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); | |
1646 | } | |
1647 | ||
a4679373 CH |
1648 | #ifdef __ARCH_WANT_SYS_OLD_MMAP |
1649 | struct mmap_arg_struct { | |
1650 | unsigned long addr; | |
1651 | unsigned long len; | |
1652 | unsigned long prot; | |
1653 | unsigned long flags; | |
1654 | unsigned long fd; | |
1655 | unsigned long offset; | |
1656 | }; | |
1657 | ||
1658 | SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) | |
1659 | { | |
1660 | struct mmap_arg_struct a; | |
1661 | ||
1662 | if (copy_from_user(&a, arg, sizeof(a))) | |
1663 | return -EFAULT; | |
de1741a1 | 1664 | if (offset_in_page(a.offset)) |
a4679373 CH |
1665 | return -EINVAL; |
1666 | ||
a90f590a DB |
1667 | return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, |
1668 | a.offset >> PAGE_SHIFT); | |
a4679373 CH |
1669 | } |
1670 | #endif /* __ARCH_WANT_SYS_OLD_MMAP */ | |
1671 | ||
4e950f6f | 1672 | /* |
8bb4e7a2 | 1673 | * Some shared mappings will want the pages marked read-only |
4e950f6f AD |
1674 | * to track write events. If so, we'll downgrade vm_page_prot |
1675 | * to the private version (using protection_map[] without the | |
1676 | * VM_SHARED bit). | |
1677 | */ | |
6d2329f8 | 1678 | int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) |
4e950f6f | 1679 | { |
ca16d140 | 1680 | vm_flags_t vm_flags = vma->vm_flags; |
8a04446a | 1681 | const struct vm_operations_struct *vm_ops = vma->vm_ops; |
4e950f6f AD |
1682 | |
1683 | /* If it was private or non-writable, the write bit is already clear */ | |
1684 | if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED))) | |
1685 | return 0; | |
1686 | ||
1687 | /* The backer wishes to know when pages are first written to? */ | |
8a04446a | 1688 | if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite)) |
4e950f6f AD |
1689 | return 1; |
1690 | ||
64e45507 PF |
1691 | /* The open routine did something to the protections that pgprot_modify |
1692 | * won't preserve? */ | |
6d2329f8 AA |
1693 | if (pgprot_val(vm_page_prot) != |
1694 | pgprot_val(vm_pgprot_modify(vm_page_prot, vm_flags))) | |
4e950f6f AD |
1695 | return 0; |
1696 | ||
f96f7a40 DH |
1697 | /* |
1698 | * Do we need to track softdirty? hugetlb does not support softdirty | |
1699 | * tracking yet. | |
1700 | */ | |
1701 | if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) | |
64e45507 PF |
1702 | return 1; |
1703 | ||
4e950f6f | 1704 | /* Specialty mapping? */ |
4b6e1e37 | 1705 | if (vm_flags & VM_PFNMAP) |
4e950f6f AD |
1706 | return 0; |
1707 | ||
1708 | /* Can the mapping track the dirty pages? */ | |
1709 | return vma->vm_file && vma->vm_file->f_mapping && | |
f56753ac | 1710 | mapping_can_writeback(vma->vm_file->f_mapping); |
4e950f6f AD |
1711 | } |
1712 | ||
fc8744ad LT |
1713 | /* |
1714 | * We account for memory if it's a private writeable mapping, | |
5a6fe125 | 1715 | * not hugepages and VM_NORESERVE wasn't set. |
fc8744ad | 1716 | */ |
ca16d140 | 1717 | static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) |
fc8744ad | 1718 | { |
5a6fe125 MG |
1719 | /* |
1720 | * hugetlb has its own accounting separate from the core VM | |
1721 | * VM_HUGETLB may not be set yet so we cannot check for that flag. | |
1722 | */ | |
1723 | if (file && is_file_hugepages(file)) | |
1724 | return 0; | |
1725 | ||
fc8744ad LT |
1726 | return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; |
1727 | } | |
1728 | ||
0165ab44 | 1729 | unsigned long mmap_region(struct file *file, unsigned long addr, |
897ab3e0 MR |
1730 | unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, |
1731 | struct list_head *uf) | |
0165ab44 MS |
1732 | { |
1733 | struct mm_struct *mm = current->mm; | |
4dd1b841 LH |
1734 | struct vm_area_struct *vma = NULL; |
1735 | struct vm_area_struct *next, *prev, *merge; | |
1736 | pgoff_t pglen = len >> PAGE_SHIFT; | |
0165ab44 | 1737 | unsigned long charged = 0; |
4dd1b841 LH |
1738 | unsigned long end = addr + len; |
1739 | unsigned long merge_start = addr, merge_end = end; | |
1740 | pgoff_t vm_pgoff; | |
1741 | int error; | |
1742 | MA_STATE(mas, &mm->mm_mt, addr, end - 1); | |
0165ab44 | 1743 | |
e8420a8e | 1744 | /* Check against address space limit. */ |
84638335 | 1745 | if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { |
e8420a8e CH |
1746 | unsigned long nr_pages; |
1747 | ||
1748 | /* | |
1749 | * MAP_FIXED may remove pages of mappings that intersects with | |
1750 | * requested mapping. Account for the pages it would unmap. | |
1751 | */ | |
4dd1b841 | 1752 | nr_pages = count_vma_pages_range(mm, addr, end); |
e8420a8e | 1753 | |
84638335 KK |
1754 | if (!may_expand_vm(mm, vm_flags, |
1755 | (len >> PAGE_SHIFT) - nr_pages)) | |
e8420a8e CH |
1756 | return -ENOMEM; |
1757 | } | |
1758 | ||
4dd1b841 LH |
1759 | /* Unmap any existing mapping in the area */ |
1760 | if (do_munmap(mm, addr, len, uf)) | |
fb8090b6 | 1761 | return -ENOMEM; |
4dd1b841 | 1762 | |
fc8744ad LT |
1763 | /* |
1764 | * Private writable mapping: check memory availability | |
1765 | */ | |
5a6fe125 | 1766 | if (accountable_mapping(file, vm_flags)) { |
fc8744ad | 1767 | charged = len >> PAGE_SHIFT; |
191c5424 | 1768 | if (security_vm_enough_memory_mm(mm, charged)) |
fc8744ad LT |
1769 | return -ENOMEM; |
1770 | vm_flags |= VM_ACCOUNT; | |
1da177e4 LT |
1771 | } |
1772 | ||
4dd1b841 LH |
1773 | next = mas_next(&mas, ULONG_MAX); |
1774 | prev = mas_prev(&mas, 0); | |
1775 | if (vm_flags & VM_SPECIAL) | |
1776 | goto cannot_expand; | |
1777 | ||
1778 | /* Attempt to expand an old mapping */ | |
1779 | /* Check next */ | |
1780 | if (next && next->vm_start == end && !vma_policy(next) && | |
1781 | can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, | |
1782 | NULL_VM_UFFD_CTX, NULL)) { | |
1783 | merge_end = next->vm_end; | |
1784 | vma = next; | |
1785 | vm_pgoff = next->vm_pgoff - pglen; | |
1786 | } | |
1787 | ||
1788 | /* Check prev */ | |
1789 | if (prev && prev->vm_end == addr && !vma_policy(prev) && | |
1790 | (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, | |
1791 | pgoff, vma->vm_userfaultfd_ctx, NULL) : | |
1792 | can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, | |
1793 | NULL_VM_UFFD_CTX, NULL))) { | |
1794 | merge_start = prev->vm_start; | |
1795 | vma = prev; | |
1796 | vm_pgoff = prev->vm_pgoff; | |
1797 | } | |
1798 | ||
1da177e4 | 1799 | |
4dd1b841 LH |
1800 | /* Actually expand, if possible */ |
1801 | if (vma && | |
1802 | !vma_expand(&mas, vma, merge_start, merge_end, vm_pgoff, next)) { | |
1803 | khugepaged_enter_vma(vma, vm_flags); | |
1804 | goto expanded; | |
1805 | } | |
1806 | ||
1807 | mas.index = addr; | |
1808 | mas.last = end - 1; | |
1809 | cannot_expand: | |
1da177e4 LT |
1810 | /* |
1811 | * Determine the object being mapped and call the appropriate | |
1812 | * specific mapper. the address has already been validated, but | |
1813 | * not unmapped, but the maps are removed from the list. | |
1814 | */ | |
490fc053 | 1815 | vma = vm_area_alloc(mm); |
1da177e4 LT |
1816 | if (!vma) { |
1817 | error = -ENOMEM; | |
1818 | goto unacct_error; | |
1819 | } | |
1da177e4 | 1820 | |
1da177e4 | 1821 | vma->vm_start = addr; |
4dd1b841 | 1822 | vma->vm_end = end; |
1da177e4 | 1823 | vma->vm_flags = vm_flags; |
3ed75eb8 | 1824 | vma->vm_page_prot = vm_get_page_prot(vm_flags); |
1da177e4 LT |
1825 | vma->vm_pgoff = pgoff; |
1826 | ||
1827 | if (file) { | |
4bb5f5d9 DR |
1828 | if (vm_flags & VM_SHARED) { |
1829 | error = mapping_map_writable(file->f_mapping); | |
1830 | if (error) | |
8d0920bd | 1831 | goto free_vma; |
4bb5f5d9 DR |
1832 | } |
1833 | ||
cb0942b8 | 1834 | vma->vm_file = get_file(file); |
f74ac015 | 1835 | error = call_mmap(file, vma); |
1da177e4 LT |
1836 | if (error) |
1837 | goto unmap_and_free_vma; | |
f8dbf0a7 | 1838 | |
309d08d9 LZ |
1839 | /* Can addr have changed?? |
1840 | * | |
1841 | * Answer: Yes, several device drivers can do it in their | |
1842 | * f_op->mmap method. -DaveM | |
309d08d9 LZ |
1843 | */ |
1844 | WARN_ON_ONCE(addr != vma->vm_start); | |
1845 | ||
1846 | addr = vma->vm_start; | |
4dd1b841 | 1847 | mas_reset(&mas); |
309d08d9 | 1848 | |
4dd1b841 LH |
1849 | /* |
1850 | * If vm_flags changed after call_mmap(), we should try merge | |
1851 | * vma again as we may succeed this time. | |
d70cec89 ML |
1852 | */ |
1853 | if (unlikely(vm_flags != vma->vm_flags && prev)) { | |
1854 | merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, | |
9a10064f | 1855 | NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); |
d70cec89 | 1856 | if (merge) { |
4dd1b841 LH |
1857 | /* |
1858 | * ->mmap() can change vma->vm_file and fput | |
1859 | * the original file. So fput the vma->vm_file | |
1860 | * here or we would add an extra fput for file | |
1861 | * and cause general protection fault | |
1862 | * ultimately. | |
bc4fe4cd ML |
1863 | */ |
1864 | fput(vma->vm_file); | |
d70cec89 ML |
1865 | vm_area_free(vma); |
1866 | vma = merge; | |
309d08d9 | 1867 | /* Update vm_flags to pick up the change. */ |
4dd1b841 | 1868 | addr = vma->vm_start; |
d70cec89 ML |
1869 | vm_flags = vma->vm_flags; |
1870 | goto unmap_writable; | |
1871 | } | |
1872 | } | |
1873 | ||
f8dbf0a7 | 1874 | vm_flags = vma->vm_flags; |
1da177e4 LT |
1875 | } else if (vm_flags & VM_SHARED) { |
1876 | error = shmem_zero_setup(vma); | |
1877 | if (error) | |
1878 | goto free_vma; | |
bfd40eaf KS |
1879 | } else { |
1880 | vma_set_anonymous(vma); | |
1da177e4 LT |
1881 | } |
1882 | ||
c462ac28 CM |
1883 | /* Allow architectures to sanity-check the vm_flags */ |
1884 | if (!arch_validate_flags(vma->vm_flags)) { | |
1885 | error = -EINVAL; | |
1886 | if (file) | |
1887 | goto unmap_and_free_vma; | |
1888 | else | |
1889 | goto free_vma; | |
1890 | } | |
1891 | ||
4dd1b841 | 1892 | if (mas_preallocate(&mas, vma, GFP_KERNEL)) { |
d4af56c5 LH |
1893 | error = -ENOMEM; |
1894 | if (file) | |
1895 | goto unmap_and_free_vma; | |
1896 | else | |
1897 | goto free_vma; | |
1898 | } | |
613bec09 | 1899 | |
4dd1b841 LH |
1900 | if (vma->vm_file) |
1901 | i_mmap_lock_write(vma->vm_file->f_mapping); | |
1902 | ||
1903 | vma_mas_store(vma, &mas); | |
1904 | __vma_link_list(mm, vma, prev); | |
1905 | mm->map_count++; | |
1906 | if (vma->vm_file) { | |
1907 | if (vma->vm_flags & VM_SHARED) | |
1908 | mapping_allow_writable(vma->vm_file->f_mapping); | |
1909 | ||
1910 | flush_dcache_mmap_lock(vma->vm_file->f_mapping); | |
1911 | vma_interval_tree_insert(vma, &vma->vm_file->f_mapping->i_mmap); | |
1912 | flush_dcache_mmap_unlock(vma->vm_file->f_mapping); | |
1913 | i_mmap_unlock_write(vma->vm_file->f_mapping); | |
1914 | } | |
1915 | ||
613bec09 YS |
1916 | /* |
1917 | * vma_merge() calls khugepaged_enter_vma() either, the below | |
1918 | * call covers the non-merge case. | |
1919 | */ | |
1920 | khugepaged_enter_vma(vma, vma->vm_flags); | |
1921 | ||
4d3d5b41 | 1922 | /* Once vma denies write, undo our temporary denial count */ |
d70cec89 | 1923 | unmap_writable: |
8d0920bd DH |
1924 | if (file && vm_flags & VM_SHARED) |
1925 | mapping_unmap_writable(file->f_mapping); | |
e8686772 | 1926 | file = vma->vm_file; |
4dd1b841 | 1927 | expanded: |
cdd6c482 | 1928 | perf_event_mmap(vma); |
0a4a9391 | 1929 | |
84638335 | 1930 | vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); |
1da177e4 | 1931 | if (vm_flags & VM_LOCKED) { |
e1fb4a08 DJ |
1932 | if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || |
1933 | is_vm_hugetlb_page(vma) || | |
1934 | vma == get_gate_vma(current->mm)) | |
de60f5f1 | 1935 | vma->vm_flags &= VM_LOCKED_CLEAR_MASK; |
e1fb4a08 DJ |
1936 | else |
1937 | mm->locked_vm += (len >> PAGE_SHIFT); | |
bebeb3d6 | 1938 | } |
2b144498 | 1939 | |
c7a3a88c ON |
1940 | if (file) |
1941 | uprobe_mmap(vma); | |
2b144498 | 1942 | |
d9104d1c CG |
1943 | /* |
1944 | * New (or expanded) vma always get soft dirty status. | |
1945 | * Otherwise user-space soft-dirty page tracker won't | |
1946 | * be able to distinguish situation when vma area unmapped, | |
1947 | * then new mapped in-place (which must be aimed as | |
1948 | * a completely new data area). | |
1949 | */ | |
1950 | vma->vm_flags |= VM_SOFTDIRTY; | |
1951 | ||
64e45507 PF |
1952 | vma_set_page_prot(vma); |
1953 | ||
4dd1b841 | 1954 | validate_mm(mm); |
1da177e4 LT |
1955 | return addr; |
1956 | ||
1957 | unmap_and_free_vma: | |
1527f926 | 1958 | fput(vma->vm_file); |
1da177e4 | 1959 | vma->vm_file = NULL; |
1da177e4 LT |
1960 | |
1961 | /* Undo any partial mapping done by a device driver. */ | |
e0da382c | 1962 | unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); |
4bb5f5d9 DR |
1963 | if (vm_flags & VM_SHARED) |
1964 | mapping_unmap_writable(file->f_mapping); | |
1da177e4 | 1965 | free_vma: |
3928d4f5 | 1966 | vm_area_free(vma); |
1da177e4 LT |
1967 | unacct_error: |
1968 | if (charged) | |
1969 | vm_unacct_memory(charged); | |
4dd1b841 | 1970 | validate_mm(mm); |
1da177e4 LT |
1971 | return error; |
1972 | } | |
1973 | ||
3499a131 LH |
1974 | /** |
1975 | * unmapped_area() - Find an area between the low_limit and the high_limit with | |
1976 | * the correct alignment and offset, all from @info. Note: current->mm is used | |
1977 | * for the search. | |
1978 | * | |
1979 | * @info: The unmapped area information including the range (low_limit - | |
1980 | * hight_limit), the alignment offset and mask. | |
1981 | * | |
1982 | * Return: A memory address or -ENOMEM. | |
1983 | */ | |
baceaf1c | 1984 | static unsigned long unmapped_area(struct vm_unmapped_area_info *info) |
db4fbfb9 | 1985 | { |
3499a131 | 1986 | unsigned long length, gap; |
db4fbfb9 | 1987 | |
3499a131 | 1988 | MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); |
db4fbfb9 ML |
1989 | |
1990 | /* Adjust search length to account for worst case alignment overhead */ | |
1991 | length = info->length + info->align_mask; | |
1992 | if (length < info->length) | |
1993 | return -ENOMEM; | |
1994 | ||
3499a131 LH |
1995 | if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1, |
1996 | length)) | |
db4fbfb9 ML |
1997 | return -ENOMEM; |
1998 | ||
3499a131 LH |
1999 | gap = mas.index; |
2000 | gap += (info->align_offset - gap) & info->align_mask; | |
2001 | return gap; | |
db4fbfb9 ML |
2002 | } |
2003 | ||
3499a131 LH |
2004 | /** |
2005 | * unmapped_area_topdown() - Find an area between the low_limit and the | |
2006 | * high_limit with * the correct alignment and offset at the highest available | |
2007 | * address, all from @info. Note: current->mm is used for the search. | |
2008 | * | |
2009 | * @info: The unmapped area information including the range (low_limit - | |
2010 | * hight_limit), the alignment offset and mask. | |
2011 | * | |
2012 | * Return: A memory address or -ENOMEM. | |
2013 | */ | |
baceaf1c | 2014 | static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) |
db4fbfb9 | 2015 | { |
3499a131 | 2016 | unsigned long length, gap; |
db4fbfb9 | 2017 | |
3499a131 | 2018 | MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); |
db4fbfb9 ML |
2019 | /* Adjust search length to account for worst case alignment overhead */ |
2020 | length = info->length + info->align_mask; | |
2021 | if (length < info->length) | |
2022 | return -ENOMEM; | |
2023 | ||
3499a131 LH |
2024 | if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, |
2025 | length)) | |
db4fbfb9 | 2026 | return -ENOMEM; |
db4fbfb9 | 2027 | |
3499a131 LH |
2028 | gap = mas.last + 1 - info->length; |
2029 | gap -= (gap - info->align_offset) & info->align_mask; | |
2030 | return gap; | |
db4fbfb9 ML |
2031 | } |
2032 | ||
baceaf1c JK |
2033 | /* |
2034 | * Search for an unmapped address range. | |
2035 | * | |
2036 | * We are looking for a range that: | |
2037 | * - does not intersect with any VMA; | |
2038 | * - is contained within the [low_limit, high_limit) interval; | |
2039 | * - is at least the desired size. | |
2040 | * - satisfies (begin_addr & align_mask) == (align_offset & align_mask) | |
2041 | */ | |
2042 | unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) | |
2043 | { | |
df529cab JK |
2044 | unsigned long addr; |
2045 | ||
baceaf1c | 2046 | if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) |
df529cab | 2047 | addr = unmapped_area_topdown(info); |
baceaf1c | 2048 | else |
df529cab JK |
2049 | addr = unmapped_area(info); |
2050 | ||
2051 | trace_vm_unmapped_area(addr, info); | |
2052 | return addr; | |
baceaf1c | 2053 | } |
f6795053 | 2054 | |
1da177e4 LT |
2055 | /* Get an address range which is currently unmapped. |
2056 | * For shmat() with addr=0. | |
2057 | * | |
2058 | * Ugly calling convention alert: | |
2059 | * Return value with the low bits set means error value, | |
2060 | * ie | |
2061 | * if (ret & ~PAGE_MASK) | |
2062 | * error = ret; | |
2063 | * | |
2064 | * This function "knows" that -ENOMEM has the bits set. | |
2065 | */ | |
1da177e4 | 2066 | unsigned long |
4b439e25 CL |
2067 | generic_get_unmapped_area(struct file *filp, unsigned long addr, |
2068 | unsigned long len, unsigned long pgoff, | |
2069 | unsigned long flags) | |
1da177e4 LT |
2070 | { |
2071 | struct mm_struct *mm = current->mm; | |
1be7107f | 2072 | struct vm_area_struct *vma, *prev; |
db4fbfb9 | 2073 | struct vm_unmapped_area_info info; |
2cb4de08 | 2074 | const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); |
1da177e4 | 2075 | |
f6795053 | 2076 | if (len > mmap_end - mmap_min_addr) |
1da177e4 LT |
2077 | return -ENOMEM; |
2078 | ||
06abdfb4 BH |
2079 | if (flags & MAP_FIXED) |
2080 | return addr; | |
2081 | ||
1da177e4 LT |
2082 | if (addr) { |
2083 | addr = PAGE_ALIGN(addr); | |
1be7107f | 2084 | vma = find_vma_prev(mm, addr, &prev); |
f6795053 | 2085 | if (mmap_end - len >= addr && addr >= mmap_min_addr && |
1be7107f HD |
2086 | (!vma || addr + len <= vm_start_gap(vma)) && |
2087 | (!prev || addr >= vm_end_gap(prev))) | |
1da177e4 LT |
2088 | return addr; |
2089 | } | |
1da177e4 | 2090 | |
db4fbfb9 ML |
2091 | info.flags = 0; |
2092 | info.length = len; | |
4e99b021 | 2093 | info.low_limit = mm->mmap_base; |
f6795053 | 2094 | info.high_limit = mmap_end; |
db4fbfb9 | 2095 | info.align_mask = 0; |
09ef5283 | 2096 | info.align_offset = 0; |
db4fbfb9 | 2097 | return vm_unmapped_area(&info); |
1da177e4 | 2098 | } |
4b439e25 CL |
2099 | |
2100 | #ifndef HAVE_ARCH_UNMAPPED_AREA | |
2101 | unsigned long | |
2102 | arch_get_unmapped_area(struct file *filp, unsigned long addr, | |
2103 | unsigned long len, unsigned long pgoff, | |
2104 | unsigned long flags) | |
2105 | { | |
2106 | return generic_get_unmapped_area(filp, addr, len, pgoff, flags); | |
2107 | } | |
cc71aba3 | 2108 | #endif |
1da177e4 | 2109 | |
1da177e4 LT |
2110 | /* |
2111 | * This mmap-allocator allocates new areas top-down from below the | |
2112 | * stack's low limit (the base): | |
2113 | */ | |
1da177e4 | 2114 | unsigned long |
4b439e25 CL |
2115 | generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, |
2116 | unsigned long len, unsigned long pgoff, | |
2117 | unsigned long flags) | |
1da177e4 | 2118 | { |
1be7107f | 2119 | struct vm_area_struct *vma, *prev; |
1da177e4 | 2120 | struct mm_struct *mm = current->mm; |
db4fbfb9 | 2121 | struct vm_unmapped_area_info info; |
2cb4de08 | 2122 | const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); |
1da177e4 LT |
2123 | |
2124 | /* requested length too big for entire address space */ | |
f6795053 | 2125 | if (len > mmap_end - mmap_min_addr) |
1da177e4 LT |
2126 | return -ENOMEM; |
2127 | ||
06abdfb4 BH |
2128 | if (flags & MAP_FIXED) |
2129 | return addr; | |
2130 | ||
1da177e4 LT |
2131 | /* requesting a specific address */ |
2132 | if (addr) { | |
2133 | addr = PAGE_ALIGN(addr); | |
1be7107f | 2134 | vma = find_vma_prev(mm, addr, &prev); |
f6795053 | 2135 | if (mmap_end - len >= addr && addr >= mmap_min_addr && |
1be7107f HD |
2136 | (!vma || addr + len <= vm_start_gap(vma)) && |
2137 | (!prev || addr >= vm_end_gap(prev))) | |
1da177e4 LT |
2138 | return addr; |
2139 | } | |
2140 | ||
db4fbfb9 ML |
2141 | info.flags = VM_UNMAPPED_AREA_TOPDOWN; |
2142 | info.length = len; | |
2afc745f | 2143 | info.low_limit = max(PAGE_SIZE, mmap_min_addr); |
f6795053 | 2144 | info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); |
db4fbfb9 | 2145 | info.align_mask = 0; |
09ef5283 | 2146 | info.align_offset = 0; |
db4fbfb9 | 2147 | addr = vm_unmapped_area(&info); |
b716ad95 | 2148 | |
1da177e4 LT |
2149 | /* |
2150 | * A failed mmap() very likely causes application failure, | |
2151 | * so fall back to the bottom-up function here. This scenario | |
2152 | * can happen with large stack limits and large mmap() | |
2153 | * allocations. | |
2154 | */ | |
de1741a1 | 2155 | if (offset_in_page(addr)) { |
db4fbfb9 ML |
2156 | VM_BUG_ON(addr != -ENOMEM); |
2157 | info.flags = 0; | |
2158 | info.low_limit = TASK_UNMAPPED_BASE; | |
f6795053 | 2159 | info.high_limit = mmap_end; |
db4fbfb9 ML |
2160 | addr = vm_unmapped_area(&info); |
2161 | } | |
1da177e4 LT |
2162 | |
2163 | return addr; | |
2164 | } | |
4b439e25 CL |
2165 | |
2166 | #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN | |
2167 | unsigned long | |
2168 | arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, | |
2169 | unsigned long len, unsigned long pgoff, | |
2170 | unsigned long flags) | |
2171 | { | |
2172 | return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags); | |
2173 | } | |
1da177e4 LT |
2174 | #endif |
2175 | ||
1da177e4 LT |
2176 | unsigned long |
2177 | get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, | |
2178 | unsigned long pgoff, unsigned long flags) | |
2179 | { | |
06abdfb4 BH |
2180 | unsigned long (*get_area)(struct file *, unsigned long, |
2181 | unsigned long, unsigned long, unsigned long); | |
2182 | ||
9206de95 AV |
2183 | unsigned long error = arch_mmap_check(addr, len, flags); |
2184 | if (error) | |
2185 | return error; | |
2186 | ||
2187 | /* Careful about overflows.. */ | |
2188 | if (len > TASK_SIZE) | |
2189 | return -ENOMEM; | |
2190 | ||
06abdfb4 | 2191 | get_area = current->mm->get_unmapped_area; |
c01d5b30 HD |
2192 | if (file) { |
2193 | if (file->f_op->get_unmapped_area) | |
2194 | get_area = file->f_op->get_unmapped_area; | |
2195 | } else if (flags & MAP_SHARED) { | |
2196 | /* | |
2197 | * mmap_region() will call shmem_zero_setup() to create a file, | |
2198 | * so use shmem's get_unmapped_area in case it can be huge. | |
45e55300 | 2199 | * do_mmap() will clear pgoff, so match alignment. |
c01d5b30 HD |
2200 | */ |
2201 | pgoff = 0; | |
2202 | get_area = shmem_get_unmapped_area; | |
f35b5d7d RR |
2203 | } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { |
2204 | /* Ensures that larger anonymous mappings are THP aligned. */ | |
2205 | get_area = thp_get_unmapped_area; | |
c01d5b30 HD |
2206 | } |
2207 | ||
06abdfb4 BH |
2208 | addr = get_area(file, addr, len, pgoff, flags); |
2209 | if (IS_ERR_VALUE(addr)) | |
2210 | return addr; | |
1da177e4 | 2211 | |
07ab67c8 LT |
2212 | if (addr > TASK_SIZE - len) |
2213 | return -ENOMEM; | |
de1741a1 | 2214 | if (offset_in_page(addr)) |
07ab67c8 | 2215 | return -EINVAL; |
06abdfb4 | 2216 | |
9ac4ed4b AV |
2217 | error = security_mmap_addr(addr); |
2218 | return error ? error : addr; | |
1da177e4 LT |
2219 | } |
2220 | ||
2221 | EXPORT_SYMBOL(get_unmapped_area); | |
2222 | ||
abdba2dd LH |
2223 | /** |
2224 | * find_vma_intersection() - Look up the first VMA which intersects the interval | |
2225 | * @mm: The process address space. | |
2226 | * @start_addr: The inclusive start user address. | |
2227 | * @end_addr: The exclusive end user address. | |
2228 | * | |
2229 | * Returns: The first VMA within the provided range, %NULL otherwise. Assumes | |
2230 | * start_addr < end_addr. | |
2231 | */ | |
2232 | struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, | |
2233 | unsigned long start_addr, | |
2234 | unsigned long end_addr) | |
2235 | { | |
2236 | struct vm_area_struct *vma; | |
2237 | unsigned long index = start_addr; | |
2238 | ||
2239 | mmap_assert_locked(mm); | |
2240 | /* Check the cache first. */ | |
2241 | vma = vmacache_find(mm, start_addr); | |
2242 | if (likely(vma)) | |
2243 | return vma; | |
2244 | ||
2245 | vma = mt_find(&mm->mm_mt, &index, end_addr - 1); | |
2246 | if (vma) | |
2247 | vmacache_update(start_addr, vma); | |
2248 | return vma; | |
2249 | } | |
2250 | EXPORT_SYMBOL(find_vma_intersection); | |
2251 | ||
be8432e7 LH |
2252 | /** |
2253 | * find_vma() - Find the VMA for a given address, or the next VMA. | |
2254 | * @mm: The mm_struct to check | |
2255 | * @addr: The address | |
2256 | * | |
2257 | * Returns: The VMA associated with addr, or the next VMA. | |
2258 | * May return %NULL in the case of no VMA at addr or above. | |
2259 | */ | |
48aae425 | 2260 | struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) |
1da177e4 | 2261 | { |
615d6e87 | 2262 | struct vm_area_struct *vma; |
be8432e7 | 2263 | unsigned long index = addr; |
1da177e4 | 2264 | |
5b78ed24 | 2265 | mmap_assert_locked(mm); |
841e31e5 | 2266 | /* Check the cache first. */ |
615d6e87 DB |
2267 | vma = vmacache_find(mm, addr); |
2268 | if (likely(vma)) | |
2269 | return vma; | |
841e31e5 | 2270 | |
be8432e7 | 2271 | vma = mt_find(&mm->mm_mt, &index, ULONG_MAX); |
615d6e87 DB |
2272 | if (vma) |
2273 | vmacache_update(addr, vma); | |
1da177e4 LT |
2274 | return vma; |
2275 | } | |
1da177e4 LT |
2276 | EXPORT_SYMBOL(find_vma); |
2277 | ||
7fdbd37d LH |
2278 | /** |
2279 | * find_vma_prev() - Find the VMA for a given address, or the next vma and | |
2280 | * set %pprev to the previous VMA, if any. | |
2281 | * @mm: The mm_struct to check | |
2282 | * @addr: The address | |
2283 | * @pprev: The pointer to set to the previous VMA | |
2284 | * | |
2285 | * Note that RCU lock is missing here since the external mmap_lock() is used | |
2286 | * instead. | |
2287 | * | |
2288 | * Returns: The VMA associated with @addr, or the next vma. | |
2289 | * May return %NULL in the case of no vma at addr or above. | |
6bd4837d | 2290 | */ |
1da177e4 LT |
2291 | struct vm_area_struct * |
2292 | find_vma_prev(struct mm_struct *mm, unsigned long addr, | |
2293 | struct vm_area_struct **pprev) | |
2294 | { | |
6bd4837d | 2295 | struct vm_area_struct *vma; |
7fdbd37d | 2296 | MA_STATE(mas, &mm->mm_mt, addr, addr); |
1da177e4 | 2297 | |
7fdbd37d LH |
2298 | vma = mas_walk(&mas); |
2299 | *pprev = mas_prev(&mas, 0); | |
2300 | if (!vma) | |
2301 | vma = mas_next(&mas, ULONG_MAX); | |
6bd4837d | 2302 | return vma; |
1da177e4 LT |
2303 | } |
2304 | ||
2305 | /* | |
2306 | * Verify that the stack growth is acceptable and | |
2307 | * update accounting. This is shared with both the | |
2308 | * grow-up and grow-down cases. | |
2309 | */ | |
1be7107f HD |
2310 | static int acct_stack_growth(struct vm_area_struct *vma, |
2311 | unsigned long size, unsigned long grow) | |
1da177e4 LT |
2312 | { |
2313 | struct mm_struct *mm = vma->vm_mm; | |
1be7107f | 2314 | unsigned long new_start; |
1da177e4 LT |
2315 | |
2316 | /* address space limit tests */ | |
84638335 | 2317 | if (!may_expand_vm(mm, vma->vm_flags, grow)) |
1da177e4 LT |
2318 | return -ENOMEM; |
2319 | ||
2320 | /* Stack limit test */ | |
24c79d8e | 2321 | if (size > rlimit(RLIMIT_STACK)) |
1da177e4 LT |
2322 | return -ENOMEM; |
2323 | ||
2324 | /* mlock limit tests */ | |
c5d8a364 ML |
2325 | if (mlock_future_check(mm, vma->vm_flags, grow << PAGE_SHIFT)) |
2326 | return -ENOMEM; | |
1da177e4 | 2327 | |
0d59a01b AL |
2328 | /* Check to ensure the stack will not grow into a hugetlb-only region */ |
2329 | new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start : | |
2330 | vma->vm_end - size; | |
2331 | if (is_hugepage_only_range(vma->vm_mm, new_start, size)) | |
2332 | return -EFAULT; | |
2333 | ||
1da177e4 LT |
2334 | /* |
2335 | * Overcommit.. This must be the final test, as it will | |
2336 | * update security statistics. | |
2337 | */ | |
05fa199d | 2338 | if (security_vm_enough_memory_mm(mm, grow)) |
1da177e4 LT |
2339 | return -ENOMEM; |
2340 | ||
1da177e4 LT |
2341 | return 0; |
2342 | } | |
2343 | ||
46dea3d0 | 2344 | #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64) |
1da177e4 | 2345 | /* |
46dea3d0 HD |
2346 | * PA-RISC uses this for its stack; IA64 for its Register Backing Store. |
2347 | * vma is the last one with address > vma->vm_end. Have to extend vma. | |
1da177e4 | 2348 | */ |
46dea3d0 | 2349 | int expand_upwards(struct vm_area_struct *vma, unsigned long address) |
1da177e4 | 2350 | { |
09357814 | 2351 | struct mm_struct *mm = vma->vm_mm; |
1be7107f HD |
2352 | struct vm_area_struct *next; |
2353 | unsigned long gap_addr; | |
12352d3c | 2354 | int error = 0; |
d4af56c5 | 2355 | MA_STATE(mas, &mm->mm_mt, 0, 0); |
1da177e4 LT |
2356 | |
2357 | if (!(vma->vm_flags & VM_GROWSUP)) | |
2358 | return -EFAULT; | |
2359 | ||
bd726c90 | 2360 | /* Guard against exceeding limits of the address space. */ |
1be7107f | 2361 | address &= PAGE_MASK; |
37511fb5 | 2362 | if (address >= (TASK_SIZE & PAGE_MASK)) |
12352d3c | 2363 | return -ENOMEM; |
bd726c90 | 2364 | address += PAGE_SIZE; |
12352d3c | 2365 | |
1be7107f HD |
2366 | /* Enforce stack_guard_gap */ |
2367 | gap_addr = address + stack_guard_gap; | |
bd726c90 HD |
2368 | |
2369 | /* Guard against overflow */ | |
2370 | if (gap_addr < address || gap_addr > TASK_SIZE) | |
2371 | gap_addr = TASK_SIZE; | |
2372 | ||
1be7107f | 2373 | next = vma->vm_next; |
3122e80e | 2374 | if (next && next->vm_start < gap_addr && vma_is_accessible(next)) { |
1be7107f HD |
2375 | if (!(next->vm_flags & VM_GROWSUP)) |
2376 | return -ENOMEM; | |
2377 | /* Check that both stack segments have the same anon_vma? */ | |
2378 | } | |
2379 | ||
d4af56c5 LH |
2380 | if (mas_preallocate(&mas, vma, GFP_KERNEL)) |
2381 | return -ENOMEM; | |
2382 | ||
12352d3c | 2383 | /* We must make sure the anon_vma is allocated. */ |
d4af56c5 LH |
2384 | if (unlikely(anon_vma_prepare(vma))) { |
2385 | mas_destroy(&mas); | |
1da177e4 | 2386 | return -ENOMEM; |
d4af56c5 | 2387 | } |
1da177e4 LT |
2388 | |
2389 | /* | |
2390 | * vma->vm_start/vm_end cannot change under us because the caller | |
c1e8d7c6 | 2391 | * is required to hold the mmap_lock in read mode. We need the |
1da177e4 LT |
2392 | * anon_vma lock to serialize against concurrent expand_stacks. |
2393 | */ | |
12352d3c | 2394 | anon_vma_lock_write(vma->anon_vma); |
1da177e4 LT |
2395 | |
2396 | /* Somebody else might have raced and expanded it already */ | |
2397 | if (address > vma->vm_end) { | |
2398 | unsigned long size, grow; | |
2399 | ||
2400 | size = address - vma->vm_start; | |
2401 | grow = (address - vma->vm_end) >> PAGE_SHIFT; | |
2402 | ||
42c36f63 HD |
2403 | error = -ENOMEM; |
2404 | if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { | |
2405 | error = acct_stack_growth(vma, size, grow); | |
2406 | if (!error) { | |
4128997b | 2407 | /* |
524e00b3 LH |
2408 | * We only hold a shared mmap_lock lock here, so |
2409 | * we need to protect against concurrent vma | |
2410 | * expansions. anon_vma_lock_write() doesn't | |
2411 | * help here, as we don't guarantee that all | |
2412 | * growable vmas in a mm share the same root | |
2413 | * anon vma. So, we reuse mm->page_table_lock | |
2414 | * to guard against concurrent vma expansions. | |
4128997b | 2415 | */ |
09357814 | 2416 | spin_lock(&mm->page_table_lock); |
87e8827b | 2417 | if (vma->vm_flags & VM_LOCKED) |
09357814 | 2418 | mm->locked_vm += grow; |
84638335 | 2419 | vm_stat_account(mm, vma->vm_flags, grow); |
bf181b9f | 2420 | anon_vma_interval_tree_pre_update_vma(vma); |
42c36f63 | 2421 | vma->vm_end = address; |
d4af56c5 LH |
2422 | /* Overwrite old entry in mtree. */ |
2423 | vma_mas_store(vma, &mas); | |
bf181b9f | 2424 | anon_vma_interval_tree_post_update_vma(vma); |
524e00b3 | 2425 | if (!vma->vm_next) |
1be7107f | 2426 | mm->highest_vm_end = vm_end_gap(vma); |
09357814 | 2427 | spin_unlock(&mm->page_table_lock); |
4128997b | 2428 | |
42c36f63 HD |
2429 | perf_event_mmap(vma); |
2430 | } | |
3af9e859 | 2431 | } |
1da177e4 | 2432 | } |
12352d3c | 2433 | anon_vma_unlock_write(vma->anon_vma); |
c791576c | 2434 | khugepaged_enter_vma(vma, vma->vm_flags); |
d4af56c5 | 2435 | mas_destroy(&mas); |
1da177e4 LT |
2436 | return error; |
2437 | } | |
46dea3d0 HD |
2438 | #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ |
2439 | ||
1da177e4 LT |
2440 | /* |
2441 | * vma is the first one with address < vma->vm_start. Have to extend vma. | |
2442 | */ | |
524e00b3 | 2443 | int expand_downwards(struct vm_area_struct *vma, unsigned long address) |
1da177e4 | 2444 | { |
09357814 | 2445 | struct mm_struct *mm = vma->vm_mm; |
1be7107f | 2446 | struct vm_area_struct *prev; |
0a1d5299 | 2447 | int error = 0; |
d4af56c5 | 2448 | MA_STATE(mas, &mm->mm_mt, 0, 0); |
1da177e4 | 2449 | |
8869477a | 2450 | address &= PAGE_MASK; |
0a1d5299 JH |
2451 | if (address < mmap_min_addr) |
2452 | return -EPERM; | |
8869477a | 2453 | |
1be7107f | 2454 | /* Enforce stack_guard_gap */ |
1be7107f | 2455 | prev = vma->vm_prev; |
32e4e6d5 ON |
2456 | /* Check that both stack segments have the same anon_vma? */ |
2457 | if (prev && !(prev->vm_flags & VM_GROWSDOWN) && | |
3122e80e | 2458 | vma_is_accessible(prev)) { |
32e4e6d5 | 2459 | if (address - prev->vm_end < stack_guard_gap) |
1be7107f | 2460 | return -ENOMEM; |
1be7107f HD |
2461 | } |
2462 | ||
d4af56c5 LH |
2463 | if (mas_preallocate(&mas, vma, GFP_KERNEL)) |
2464 | return -ENOMEM; | |
2465 | ||
12352d3c | 2466 | /* We must make sure the anon_vma is allocated. */ |
d4af56c5 LH |
2467 | if (unlikely(anon_vma_prepare(vma))) { |
2468 | mas_destroy(&mas); | |
12352d3c | 2469 | return -ENOMEM; |
d4af56c5 | 2470 | } |
1da177e4 LT |
2471 | |
2472 | /* | |
2473 | * vma->vm_start/vm_end cannot change under us because the caller | |
c1e8d7c6 | 2474 | * is required to hold the mmap_lock in read mode. We need the |
1da177e4 LT |
2475 | * anon_vma lock to serialize against concurrent expand_stacks. |
2476 | */ | |
12352d3c | 2477 | anon_vma_lock_write(vma->anon_vma); |
1da177e4 LT |
2478 | |
2479 | /* Somebody else might have raced and expanded it already */ | |
2480 | if (address < vma->vm_start) { | |
2481 | unsigned long size, grow; | |
2482 | ||
2483 | size = vma->vm_end - address; | |
2484 | grow = (vma->vm_start - address) >> PAGE_SHIFT; | |
2485 | ||
a626ca6a LT |
2486 | error = -ENOMEM; |
2487 | if (grow <= vma->vm_pgoff) { | |
2488 | error = acct_stack_growth(vma, size, grow); | |
2489 | if (!error) { | |
4128997b | 2490 | /* |
524e00b3 LH |
2491 | * We only hold a shared mmap_lock lock here, so |
2492 | * we need to protect against concurrent vma | |
2493 | * expansions. anon_vma_lock_write() doesn't | |
2494 | * help here, as we don't guarantee that all | |
2495 | * growable vmas in a mm share the same root | |
2496 | * anon vma. So, we reuse mm->page_table_lock | |
2497 | * to guard against concurrent vma expansions. | |
4128997b | 2498 | */ |
09357814 | 2499 | spin_lock(&mm->page_table_lock); |
87e8827b | 2500 | if (vma->vm_flags & VM_LOCKED) |
09357814 | 2501 | mm->locked_vm += grow; |
84638335 | 2502 | vm_stat_account(mm, vma->vm_flags, grow); |
bf181b9f | 2503 | anon_vma_interval_tree_pre_update_vma(vma); |
a626ca6a LT |
2504 | vma->vm_start = address; |
2505 | vma->vm_pgoff -= grow; | |
d4af56c5 LH |
2506 | /* Overwrite old entry in mtree. */ |
2507 | vma_mas_store(vma, &mas); | |
bf181b9f | 2508 | anon_vma_interval_tree_post_update_vma(vma); |
09357814 | 2509 | spin_unlock(&mm->page_table_lock); |
4128997b | 2510 | |
a626ca6a LT |
2511 | perf_event_mmap(vma); |
2512 | } | |
1da177e4 LT |
2513 | } |
2514 | } | |
12352d3c | 2515 | anon_vma_unlock_write(vma->anon_vma); |
c791576c | 2516 | khugepaged_enter_vma(vma, vma->vm_flags); |
d4af56c5 | 2517 | mas_destroy(&mas); |
1da177e4 LT |
2518 | return error; |
2519 | } | |
2520 | ||
1be7107f HD |
2521 | /* enforced gap between the expanding stack and other mappings. */ |
2522 | unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT; | |
2523 | ||
2524 | static int __init cmdline_parse_stack_guard_gap(char *p) | |
2525 | { | |
2526 | unsigned long val; | |
2527 | char *endptr; | |
2528 | ||
2529 | val = simple_strtoul(p, &endptr, 10); | |
2530 | if (!*endptr) | |
2531 | stack_guard_gap = val << PAGE_SHIFT; | |
2532 | ||
e6d09493 | 2533 | return 1; |
1be7107f HD |
2534 | } |
2535 | __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap); | |
2536 | ||
b6a2fea3 OW |
2537 | #ifdef CONFIG_STACK_GROWSUP |
2538 | int expand_stack(struct vm_area_struct *vma, unsigned long address) | |
2539 | { | |
2540 | return expand_upwards(vma, address); | |
2541 | } | |
2542 | ||
2543 | struct vm_area_struct * | |
2544 | find_extend_vma(struct mm_struct *mm, unsigned long addr) | |
2545 | { | |
2546 | struct vm_area_struct *vma, *prev; | |
2547 | ||
2548 | addr &= PAGE_MASK; | |
2549 | vma = find_vma_prev(mm, addr, &prev); | |
2550 | if (vma && (vma->vm_start <= addr)) | |
2551 | return vma; | |
4d45e75a | 2552 | if (!prev || expand_stack(prev, addr)) |
b6a2fea3 | 2553 | return NULL; |
cea10a19 | 2554 | if (prev->vm_flags & VM_LOCKED) |
fc05f566 | 2555 | populate_vma_page_range(prev, addr, prev->vm_end, NULL); |
b6a2fea3 OW |
2556 | return prev; |
2557 | } | |
2558 | #else | |
2559 | int expand_stack(struct vm_area_struct *vma, unsigned long address) | |
2560 | { | |
2561 | return expand_downwards(vma, address); | |
2562 | } | |
2563 | ||
1da177e4 | 2564 | struct vm_area_struct * |
cc71aba3 | 2565 | find_extend_vma(struct mm_struct *mm, unsigned long addr) |
1da177e4 | 2566 | { |
cc71aba3 | 2567 | struct vm_area_struct *vma; |
1da177e4 LT |
2568 | unsigned long start; |
2569 | ||
2570 | addr &= PAGE_MASK; | |
cc71aba3 | 2571 | vma = find_vma(mm, addr); |
1da177e4 LT |
2572 | if (!vma) |
2573 | return NULL; | |
2574 | if (vma->vm_start <= addr) | |
2575 | return vma; | |
2576 | if (!(vma->vm_flags & VM_GROWSDOWN)) | |
2577 | return NULL; | |
2578 | start = vma->vm_start; | |
2579 | if (expand_stack(vma, addr)) | |
2580 | return NULL; | |
cea10a19 | 2581 | if (vma->vm_flags & VM_LOCKED) |
fc05f566 | 2582 | populate_vma_page_range(vma, addr, start, NULL); |
1da177e4 LT |
2583 | return vma; |
2584 | } | |
2585 | #endif | |
2586 | ||
e1d6d01a JB |
2587 | EXPORT_SYMBOL_GPL(find_extend_vma); |
2588 | ||
1da177e4 | 2589 | /* |
2c0b3814 | 2590 | * Ok - we have the memory areas we should free on the vma list, |
1da177e4 | 2591 | * so release them, and do the vma updates. |
2c0b3814 HD |
2592 | * |
2593 | * Called with the mm semaphore held. | |
1da177e4 | 2594 | */ |
2c0b3814 | 2595 | static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) |
1da177e4 | 2596 | { |
4f74d2c8 LT |
2597 | unsigned long nr_accounted = 0; |
2598 | ||
365e9c87 HD |
2599 | /* Update high watermark before we lower total_vm */ |
2600 | update_hiwater_vm(mm); | |
1da177e4 | 2601 | do { |
2c0b3814 HD |
2602 | long nrpages = vma_pages(vma); |
2603 | ||
4f74d2c8 LT |
2604 | if (vma->vm_flags & VM_ACCOUNT) |
2605 | nr_accounted += nrpages; | |
84638335 | 2606 | vm_stat_account(mm, vma->vm_flags, -nrpages); |
a8fb5618 | 2607 | vma = remove_vma(vma); |
146425a3 | 2608 | } while (vma); |
4f74d2c8 | 2609 | vm_unacct_memory(nr_accounted); |
1da177e4 LT |
2610 | validate_mm(mm); |
2611 | } | |
2612 | ||
2613 | /* | |
2614 | * Get rid of page table information in the indicated region. | |
2615 | * | |
f10df686 | 2616 | * Called with the mm semaphore held. |
1da177e4 LT |
2617 | */ |
2618 | static void unmap_region(struct mm_struct *mm, | |
e0da382c HD |
2619 | struct vm_area_struct *vma, struct vm_area_struct *prev, |
2620 | unsigned long start, unsigned long end) | |
1da177e4 | 2621 | { |
f39af059 | 2622 | struct vm_area_struct *next = __vma_next(mm, prev); |
d16dfc55 | 2623 | struct mmu_gather tlb; |
1da177e4 LT |
2624 | |
2625 | lru_add_drain(); | |
a72afd87 | 2626 | tlb_gather_mmu(&tlb, mm); |
365e9c87 | 2627 | update_hiwater_rss(mm); |
4f74d2c8 | 2628 | unmap_vmas(&tlb, vma, start, end); |
d16dfc55 | 2629 | free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, |
6ee8630e | 2630 | next ? next->vm_start : USER_PGTABLES_CEILING); |
ae8eba8b | 2631 | tlb_finish_mmu(&tlb); |
1da177e4 LT |
2632 | } |
2633 | ||
2634 | /* | |
2635 | * Create a list of vma's touched by the unmap, removing them from the mm's | |
2636 | * vma list as we go.. | |
2637 | */ | |
246c320a | 2638 | static bool |
d4af56c5 LH |
2639 | detach_vmas_to_be_unmapped(struct mm_struct *mm, struct ma_state *mas, |
2640 | struct vm_area_struct *vma, struct vm_area_struct *prev, | |
2641 | unsigned long end) | |
1da177e4 LT |
2642 | { |
2643 | struct vm_area_struct **insertion_point; | |
2644 | struct vm_area_struct *tail_vma = NULL; | |
2645 | ||
2646 | insertion_point = (prev ? &prev->vm_next : &mm->mmap); | |
297c5eee | 2647 | vma->vm_prev = NULL; |
524e00b3 | 2648 | vma_mas_szero(mas, vma->vm_start, end); |
1da177e4 | 2649 | do { |
a213e5cf HD |
2650 | if (vma->vm_flags & VM_LOCKED) |
2651 | mm->locked_vm -= vma_pages(vma); | |
1da177e4 LT |
2652 | mm->map_count--; |
2653 | tail_vma = vma; | |
2654 | vma = vma->vm_next; | |
2655 | } while (vma && vma->vm_start < end); | |
2656 | *insertion_point = vma; | |
524e00b3 | 2657 | if (vma) |
297c5eee | 2658 | vma->vm_prev = prev; |
524e00b3 | 2659 | else |
1be7107f | 2660 | mm->highest_vm_end = prev ? vm_end_gap(prev) : 0; |
1da177e4 | 2661 | tail_vma->vm_next = NULL; |
615d6e87 DB |
2662 | |
2663 | /* Kill the cache */ | |
2664 | vmacache_invalidate(mm); | |
246c320a KS |
2665 | |
2666 | /* | |
2667 | * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or | |
2668 | * VM_GROWSUP VMA. Such VMAs can change their size under | |
2669 | * down_read(mmap_lock) and collide with the VMA we are about to unmap. | |
2670 | */ | |
2671 | if (vma && (vma->vm_flags & VM_GROWSDOWN)) | |
2672 | return false; | |
2673 | if (prev && (prev->vm_flags & VM_GROWSUP)) | |
2674 | return false; | |
2675 | return true; | |
1da177e4 LT |
2676 | } |
2677 | ||
2678 | /* | |
def5efe0 DR |
2679 | * __split_vma() bypasses sysctl_max_map_count checking. We use this where it |
2680 | * has already been checked or doesn't make sense to fail. | |
1da177e4 | 2681 | */ |
def5efe0 DR |
2682 | int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, |
2683 | unsigned long addr, int new_below) | |
1da177e4 | 2684 | { |
1da177e4 | 2685 | struct vm_area_struct *new; |
e3975891 | 2686 | int err; |
d4af56c5 | 2687 | validate_mm_mt(mm); |
1da177e4 | 2688 | |
dd3b614f DS |
2689 | if (vma->vm_ops && vma->vm_ops->may_split) { |
2690 | err = vma->vm_ops->may_split(vma, addr); | |
31383c68 DW |
2691 | if (err) |
2692 | return err; | |
2693 | } | |
1da177e4 | 2694 | |
3928d4f5 | 2695 | new = vm_area_dup(vma); |
1da177e4 | 2696 | if (!new) |
e3975891 | 2697 | return -ENOMEM; |
1da177e4 | 2698 | |
1da177e4 LT |
2699 | if (new_below) |
2700 | new->vm_end = addr; | |
2701 | else { | |
2702 | new->vm_start = addr; | |
2703 | new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); | |
2704 | } | |
2705 | ||
ef0855d3 ON |
2706 | err = vma_dup_policy(vma, new); |
2707 | if (err) | |
5beb4930 | 2708 | goto out_free_vma; |
1da177e4 | 2709 | |
c4ea95d7 DF |
2710 | err = anon_vma_clone(new, vma); |
2711 | if (err) | |
5beb4930 RR |
2712 | goto out_free_mpol; |
2713 | ||
e9714acf | 2714 | if (new->vm_file) |
1da177e4 LT |
2715 | get_file(new->vm_file); |
2716 | ||
2717 | if (new->vm_ops && new->vm_ops->open) | |
2718 | new->vm_ops->open(new); | |
2719 | ||
2720 | if (new_below) | |
5beb4930 | 2721 | err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff + |
1da177e4 LT |
2722 | ((addr - new->vm_start) >> PAGE_SHIFT), new); |
2723 | else | |
5beb4930 | 2724 | err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new); |
1da177e4 | 2725 | |
5beb4930 RR |
2726 | /* Success. */ |
2727 | if (!err) | |
2728 | return 0; | |
2729 | ||
d4af56c5 LH |
2730 | /* Avoid vm accounting in close() operation */ |
2731 | new->vm_start = new->vm_end; | |
2732 | new->vm_pgoff = 0; | |
5beb4930 | 2733 | /* Clean everything up if vma_adjust failed. */ |
58927533 RR |
2734 | if (new->vm_ops && new->vm_ops->close) |
2735 | new->vm_ops->close(new); | |
e9714acf | 2736 | if (new->vm_file) |
5beb4930 | 2737 | fput(new->vm_file); |
2aeadc30 | 2738 | unlink_anon_vmas(new); |
5beb4930 | 2739 | out_free_mpol: |
ef0855d3 | 2740 | mpol_put(vma_policy(new)); |
5beb4930 | 2741 | out_free_vma: |
3928d4f5 | 2742 | vm_area_free(new); |
d4af56c5 | 2743 | validate_mm_mt(mm); |
5beb4930 | 2744 | return err; |
1da177e4 LT |
2745 | } |
2746 | ||
659ace58 KM |
2747 | /* |
2748 | * Split a vma into two pieces at address 'addr', a new vma is allocated | |
2749 | * either for the first part or the tail. | |
2750 | */ | |
2751 | int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, | |
2752 | unsigned long addr, int new_below) | |
2753 | { | |
2754 | if (mm->map_count >= sysctl_max_map_count) | |
2755 | return -ENOMEM; | |
2756 | ||
2757 | return __split_vma(mm, vma, addr, new_below); | |
2758 | } | |
2759 | ||
1da177e4 LT |
2760 | /* Munmap is split into 2 main parts -- this part which finds |
2761 | * what needs doing, and the areas themselves, which do the | |
2762 | * work. This now handles partial unmappings. | |
2763 | * Jeremy Fitzhardinge <jeremy@goop.org> | |
2764 | */ | |
85a06835 YS |
2765 | int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, |
2766 | struct list_head *uf, bool downgrade) | |
1da177e4 LT |
2767 | { |
2768 | unsigned long end; | |
146425a3 | 2769 | struct vm_area_struct *vma, *prev, *last; |
d4af56c5 LH |
2770 | int error = -ENOMEM; |
2771 | MA_STATE(mas, &mm->mm_mt, 0, 0); | |
1da177e4 | 2772 | |
de1741a1 | 2773 | if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) |
1da177e4 LT |
2774 | return -EINVAL; |
2775 | ||
cc71aba3 | 2776 | len = PAGE_ALIGN(len); |
5a28fc94 | 2777 | end = start + len; |
cc71aba3 | 2778 | if (len == 0) |
1da177e4 LT |
2779 | return -EINVAL; |
2780 | ||
524e00b3 | 2781 | /* arch_unmap() might do unmaps itself. */ |
5a28fc94 DH |
2782 | arch_unmap(mm, start, end); |
2783 | ||
78d9cf60 GMJT |
2784 | /* Find the first overlapping VMA where start < vma->vm_end */ |
2785 | vma = find_vma_intersection(mm, start, end); | |
146425a3 | 2786 | if (!vma) |
1da177e4 | 2787 | return 0; |
d4af56c5 LH |
2788 | |
2789 | if (mas_preallocate(&mas, vma, GFP_KERNEL)) | |
2790 | return -ENOMEM; | |
9be34c9d | 2791 | prev = vma->vm_prev; |
524e00b3 LH |
2792 | /* we have start < vma->vm_end */ |
2793 | ||
1da177e4 LT |
2794 | /* |
2795 | * If we need to split any vma, do it now to save pain later. | |
2796 | * | |
2797 | * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially | |
2798 | * unmapped vm_area_struct will remain in use: so lower split_vma | |
2799 | * places tmp vma above, and higher split_vma places tmp vma below. | |
2800 | */ | |
146425a3 | 2801 | if (start > vma->vm_start) { |
659ace58 KM |
2802 | |
2803 | /* | |
2804 | * Make sure that map_count on return from munmap() will | |
2805 | * not exceed its limit; but let map_count go just above | |
2806 | * its limit temporarily, to help free resources as expected. | |
2807 | */ | |
2808 | if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) | |
d4af56c5 | 2809 | goto map_count_exceeded; |
659ace58 KM |
2810 | |
2811 | error = __split_vma(mm, vma, start, 0); | |
1da177e4 | 2812 | if (error) |
d4af56c5 | 2813 | goto split_failed; |
146425a3 | 2814 | prev = vma; |
1da177e4 LT |
2815 | } |
2816 | ||
2817 | /* Does it split the last one? */ | |
2818 | last = find_vma(mm, end); | |
2819 | if (last && end > last->vm_start) { | |
d4af56c5 | 2820 | error = __split_vma(mm, last, end, 1); |
1da177e4 | 2821 | if (error) |
d4af56c5 | 2822 | goto split_failed; |
1da177e4 | 2823 | } |
f39af059 | 2824 | vma = __vma_next(mm, prev); |
1da177e4 | 2825 | |
2376dd7c AA |
2826 | if (unlikely(uf)) { |
2827 | /* | |
2828 | * If userfaultfd_unmap_prep returns an error the vmas | |
f0953a1b | 2829 | * will remain split, but userland will get a |
2376dd7c AA |
2830 | * highly unexpected error anyway. This is no |
2831 | * different than the case where the first of the two | |
2832 | * __split_vma fails, but we don't undo the first | |
2833 | * split, despite we could. This is unlikely enough | |
2834 | * failure that it's not worth optimizing it for. | |
2835 | */ | |
d4af56c5 | 2836 | error = userfaultfd_unmap_prep(vma, start, end, uf); |
2376dd7c | 2837 | if (error) |
d4af56c5 | 2838 | goto userfaultfd_error; |
2376dd7c AA |
2839 | } |
2840 | ||
dd2283f2 | 2841 | /* Detach vmas from rbtree */ |
d4af56c5 | 2842 | if (!detach_vmas_to_be_unmapped(mm, &mas, vma, prev, end)) |
246c320a | 2843 | downgrade = false; |
1da177e4 | 2844 | |
dd2283f2 | 2845 | if (downgrade) |
d8ed45c5 | 2846 | mmap_write_downgrade(mm); |
dd2283f2 YS |
2847 | |
2848 | unmap_region(mm, vma, prev, start, end); | |
2849 | ||
1da177e4 | 2850 | /* Fix up all other VM information */ |
2c0b3814 | 2851 | remove_vma_list(mm, vma); |
1da177e4 | 2852 | |
524e00b3 LH |
2853 | |
2854 | validate_mm(mm); | |
dd2283f2 | 2855 | return downgrade ? 1 : 0; |
d4af56c5 LH |
2856 | |
2857 | map_count_exceeded: | |
2858 | split_failed: | |
2859 | userfaultfd_error: | |
2860 | mas_destroy(&mas); | |
2861 | return error; | |
1da177e4 | 2862 | } |
1da177e4 | 2863 | |
dd2283f2 YS |
2864 | int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, |
2865 | struct list_head *uf) | |
2866 | { | |
2867 | return __do_munmap(mm, start, len, uf, false); | |
2868 | } | |
2869 | ||
2870 | static int __vm_munmap(unsigned long start, size_t len, bool downgrade) | |
1da177e4 LT |
2871 | { |
2872 | int ret; | |
bfce281c | 2873 | struct mm_struct *mm = current->mm; |
897ab3e0 | 2874 | LIST_HEAD(uf); |
1da177e4 | 2875 | |
d8ed45c5 | 2876 | if (mmap_write_lock_killable(mm)) |
ae798783 MH |
2877 | return -EINTR; |
2878 | ||
dd2283f2 YS |
2879 | ret = __do_munmap(mm, start, len, &uf, downgrade); |
2880 | /* | |
c1e8d7c6 | 2881 | * Returning 1 indicates mmap_lock is downgraded. |
dd2283f2 YS |
2882 | * But 1 is not legal return value of vm_munmap() and munmap(), reset |
2883 | * it to 0 before return. | |
2884 | */ | |
2885 | if (ret == 1) { | |
d8ed45c5 | 2886 | mmap_read_unlock(mm); |
dd2283f2 YS |
2887 | ret = 0; |
2888 | } else | |
d8ed45c5 | 2889 | mmap_write_unlock(mm); |
dd2283f2 | 2890 | |
897ab3e0 | 2891 | userfaultfd_unmap_complete(mm, &uf); |
1da177e4 LT |
2892 | return ret; |
2893 | } | |
dd2283f2 YS |
2894 | |
2895 | int vm_munmap(unsigned long start, size_t len) | |
2896 | { | |
2897 | return __vm_munmap(start, len, false); | |
2898 | } | |
a46ef99d LT |
2899 | EXPORT_SYMBOL(vm_munmap); |
2900 | ||
2901 | SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) | |
2902 | { | |
ce18d171 | 2903 | addr = untagged_addr(addr); |
dd2283f2 | 2904 | return __vm_munmap(addr, len, true); |
a46ef99d | 2905 | } |
1da177e4 | 2906 | |
c8d78c18 KS |
2907 | |
2908 | /* | |
2909 | * Emulation of deprecated remap_file_pages() syscall. | |
2910 | */ | |
2911 | SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, | |
2912 | unsigned long, prot, unsigned long, pgoff, unsigned long, flags) | |
2913 | { | |
2914 | ||
2915 | struct mm_struct *mm = current->mm; | |
2916 | struct vm_area_struct *vma; | |
2917 | unsigned long populate = 0; | |
2918 | unsigned long ret = -EINVAL; | |
2919 | struct file *file; | |
2920 | ||
ee65728e | 2921 | pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n", |
756a025f | 2922 | current->comm, current->pid); |
c8d78c18 KS |
2923 | |
2924 | if (prot) | |
2925 | return ret; | |
2926 | start = start & PAGE_MASK; | |
2927 | size = size & PAGE_MASK; | |
2928 | ||
2929 | if (start + size <= start) | |
2930 | return ret; | |
2931 | ||
2932 | /* Does pgoff wrap? */ | |
2933 | if (pgoff + (size >> PAGE_SHIFT) < pgoff) | |
2934 | return ret; | |
2935 | ||
d8ed45c5 | 2936 | if (mmap_write_lock_killable(mm)) |
dc0ef0df MH |
2937 | return -EINTR; |
2938 | ||
9b593cb2 | 2939 | vma = vma_lookup(mm, start); |
c8d78c18 KS |
2940 | |
2941 | if (!vma || !(vma->vm_flags & VM_SHARED)) | |
2942 | goto out; | |
2943 | ||
48f7df32 KS |
2944 | if (start + size > vma->vm_end) { |
2945 | struct vm_area_struct *next; | |
2946 | ||
2947 | for (next = vma->vm_next; next; next = next->vm_next) { | |
2948 | /* hole between vmas ? */ | |
2949 | if (next->vm_start != next->vm_prev->vm_end) | |
2950 | goto out; | |
2951 | ||
2952 | if (next->vm_file != vma->vm_file) | |
2953 | goto out; | |
2954 | ||
2955 | if (next->vm_flags != vma->vm_flags) | |
2956 | goto out; | |
2957 | ||
2958 | if (start + size <= next->vm_end) | |
2959 | break; | |
2960 | } | |
2961 | ||
2962 | if (!next) | |
2963 | goto out; | |
c8d78c18 KS |
2964 | } |
2965 | ||
2966 | prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; | |
2967 | prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; | |
2968 | prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; | |
2969 | ||
2970 | flags &= MAP_NONBLOCK; | |
2971 | flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; | |
fce000b1 | 2972 | if (vma->vm_flags & VM_LOCKED) |
c8d78c18 | 2973 | flags |= MAP_LOCKED; |
48f7df32 | 2974 | |
c8d78c18 | 2975 | file = get_file(vma->vm_file); |
45e55300 | 2976 | ret = do_mmap(vma->vm_file, start, size, |
897ab3e0 | 2977 | prot, flags, pgoff, &populate, NULL); |
c8d78c18 KS |
2978 | fput(file); |
2979 | out: | |
d8ed45c5 | 2980 | mmap_write_unlock(mm); |
c8d78c18 KS |
2981 | if (populate) |
2982 | mm_populate(ret, populate); | |
2983 | if (!IS_ERR_VALUE(ret)) | |
2984 | ret = 0; | |
2985 | return ret; | |
2986 | } | |
2987 | ||
1da177e4 | 2988 | /* |
2e7ce7d3 LH |
2989 | * brk_munmap() - Unmap a parital vma. |
2990 | * @mas: The maple tree state. | |
2991 | * @vma: The vma to be modified | |
2992 | * @newbrk: the start of the address to unmap | |
2993 | * @oldbrk: The end of the address to unmap | |
2994 | * @uf: The userfaultfd list_head | |
2995 | * | |
2996 | * Returns: 1 on success. | |
2997 | * unmaps a partial VMA mapping. Does not handle alignment, downgrades lock if | |
2998 | * possible. | |
1da177e4 | 2999 | */ |
2e7ce7d3 LH |
3000 | static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, |
3001 | unsigned long newbrk, unsigned long oldbrk, | |
3002 | struct list_head *uf) | |
1da177e4 | 3003 | { |
2e7ce7d3 LH |
3004 | struct mm_struct *mm = vma->vm_mm; |
3005 | int ret; | |
3a459756 | 3006 | |
2e7ce7d3 LH |
3007 | arch_unmap(mm, newbrk, oldbrk); |
3008 | ret = __do_munmap(mm, newbrk, oldbrk - newbrk, uf, true); | |
3009 | validate_mm_mt(mm); | |
3010 | return ret; | |
3011 | } | |
1da177e4 | 3012 | |
2e7ce7d3 LH |
3013 | /* |
3014 | * do_brk_flags() - Increase the brk vma if the flags match. | |
3015 | * @mas: The maple tree state. | |
3016 | * @addr: The start address | |
3017 | * @len: The length of the increase | |
3018 | * @vma: The vma, | |
3019 | * @flags: The VMA Flags | |
3020 | * | |
3021 | * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags | |
3022 | * do not match then create a new anonymous VMA. Eventually we may be able to | |
3023 | * do some brk-specific accounting here. | |
3024 | */ | |
3025 | static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *vma, | |
3026 | unsigned long addr, unsigned long len, | |
3027 | unsigned long flags) | |
3028 | { | |
3029 | struct mm_struct *mm = current->mm; | |
3030 | struct vm_area_struct *prev = NULL; | |
1da177e4 | 3031 | |
2e7ce7d3 LH |
3032 | validate_mm_mt(mm); |
3033 | /* | |
3034 | * Check against address space limits by the changed size | |
3035 | * Note: This happens *after* clearing old mappings in some code paths. | |
3036 | */ | |
3037 | flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; | |
84638335 | 3038 | if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) |
1da177e4 LT |
3039 | return -ENOMEM; |
3040 | ||
3041 | if (mm->map_count > sysctl_max_map_count) | |
3042 | return -ENOMEM; | |
3043 | ||
191c5424 | 3044 | if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) |
1da177e4 LT |
3045 | return -ENOMEM; |
3046 | ||
1da177e4 | 3047 | /* |
2e7ce7d3 LH |
3048 | * Expand the existing vma if possible; Note that singular lists do not |
3049 | * occur after forking, so the expand will only happen on new VMAs. | |
1da177e4 | 3050 | */ |
2e7ce7d3 LH |
3051 | if (vma && |
3052 | (!vma->anon_vma || list_is_singular(&vma->anon_vma_chain)) && | |
3053 | ((vma->vm_flags & ~VM_SOFTDIRTY) == flags)) { | |
3054 | mas->index = vma->vm_start; | |
3055 | mas->last = addr + len - 1; | |
3056 | vma_adjust_trans_huge(vma, addr, addr + len, 0); | |
3057 | if (vma->anon_vma) { | |
3058 | anon_vma_lock_write(vma->anon_vma); | |
3059 | anon_vma_interval_tree_pre_update_vma(vma); | |
3060 | } | |
3061 | vma->vm_end = addr + len; | |
3062 | vma->vm_flags |= VM_SOFTDIRTY; | |
3063 | if (mas_store_gfp(mas, vma, GFP_KERNEL)) | |
3064 | goto mas_expand_failed; | |
3065 | ||
3066 | if (vma->anon_vma) { | |
3067 | anon_vma_interval_tree_post_update_vma(vma); | |
3068 | anon_vma_unlock_write(vma->anon_vma); | |
3069 | } | |
3070 | khugepaged_enter_vma(vma, flags); | |
3071 | goto out; | |
1da177e4 | 3072 | } |
2e7ce7d3 LH |
3073 | prev = vma; |
3074 | ||
3075 | /* create a vma struct for an anonymous mapping */ | |
3076 | vma = vm_area_alloc(mm); | |
3077 | if (!vma) | |
3078 | goto vma_alloc_fail; | |
1da177e4 | 3079 | |
bfd40eaf | 3080 | vma_set_anonymous(vma); |
1da177e4 LT |
3081 | vma->vm_start = addr; |
3082 | vma->vm_end = addr + len; | |
2e7ce7d3 | 3083 | vma->vm_pgoff = addr >> PAGE_SHIFT; |
1da177e4 | 3084 | vma->vm_flags = flags; |
3ed75eb8 | 3085 | vma->vm_page_prot = vm_get_page_prot(flags); |
2e7ce7d3 LH |
3086 | mas_set_range(mas, vma->vm_start, addr + len - 1); |
3087 | if (mas_store_gfp(mas, vma, GFP_KERNEL)) | |
3088 | goto mas_store_fail; | |
d4af56c5 | 3089 | |
2e7ce7d3 LH |
3090 | if (!prev) |
3091 | prev = mas_prev(mas, 0); | |
3092 | ||
3093 | __vma_link_list(mm, vma, prev); | |
3094 | mm->map_count++; | |
1da177e4 | 3095 | out: |
3af9e859 | 3096 | perf_event_mmap(vma); |
1da177e4 | 3097 | mm->total_vm += len >> PAGE_SHIFT; |
84638335 | 3098 | mm->data_vm += len >> PAGE_SHIFT; |
128557ff ML |
3099 | if (flags & VM_LOCKED) |
3100 | mm->locked_vm += (len >> PAGE_SHIFT); | |
d9104d1c | 3101 | vma->vm_flags |= VM_SOFTDIRTY; |
d4af56c5 | 3102 | validate_mm_mt(mm); |
5d22fc25 | 3103 | return 0; |
d4af56c5 | 3104 | |
2e7ce7d3 | 3105 | mas_store_fail: |
d4af56c5 | 3106 | vm_area_free(vma); |
2e7ce7d3 LH |
3107 | vma_alloc_fail: |
3108 | vm_unacct_memory(len >> PAGE_SHIFT); | |
3109 | return -ENOMEM; | |
3110 | ||
3111 | mas_expand_failed: | |
3112 | if (vma->anon_vma) { | |
3113 | anon_vma_interval_tree_post_update_vma(vma); | |
3114 | anon_vma_unlock_write(vma->anon_vma); | |
3115 | } | |
d4af56c5 | 3116 | return -ENOMEM; |
1da177e4 LT |
3117 | } |
3118 | ||
bb177a73 | 3119 | int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) |
e4eb1ff6 LT |
3120 | { |
3121 | struct mm_struct *mm = current->mm; | |
2e7ce7d3 | 3122 | struct vm_area_struct *vma = NULL; |
bb177a73 | 3123 | unsigned long len; |
5d22fc25 | 3124 | int ret; |
128557ff | 3125 | bool populate; |
897ab3e0 | 3126 | LIST_HEAD(uf); |
2e7ce7d3 | 3127 | MA_STATE(mas, &mm->mm_mt, addr, addr); |
e4eb1ff6 | 3128 | |
bb177a73 MH |
3129 | len = PAGE_ALIGN(request); |
3130 | if (len < request) | |
3131 | return -ENOMEM; | |
3132 | if (!len) | |
3133 | return 0; | |
3134 | ||
d8ed45c5 | 3135 | if (mmap_write_lock_killable(mm)) |
2d6c9282 MH |
3136 | return -EINTR; |
3137 | ||
2e7ce7d3 LH |
3138 | /* Until we need other flags, refuse anything except VM_EXEC. */ |
3139 | if ((flags & (~VM_EXEC)) != 0) | |
3140 | return -EINVAL; | |
3141 | ||
3142 | ret = check_brk_limits(addr, len); | |
3143 | if (ret) | |
3144 | goto limits_failed; | |
3145 | ||
3146 | if (find_vma_intersection(mm, addr, addr + len)) | |
3147 | ret = do_munmap(mm, addr, len, &uf); | |
3148 | ||
3149 | if (ret) | |
3150 | goto munmap_failed; | |
3151 | ||
3152 | vma = mas_prev(&mas, 0); | |
3153 | if (!vma || vma->vm_end != addr || vma_policy(vma) || | |
3154 | !can_vma_merge_after(vma, flags, NULL, NULL, | |
3155 | addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) | |
3156 | vma = NULL; | |
3157 | ||
3158 | ret = do_brk_flags(&mas, vma, addr, len, flags); | |
128557ff | 3159 | populate = ((mm->def_flags & VM_LOCKED) != 0); |
d8ed45c5 | 3160 | mmap_write_unlock(mm); |
897ab3e0 | 3161 | userfaultfd_unmap_complete(mm, &uf); |
5d22fc25 | 3162 | if (populate && !ret) |
128557ff | 3163 | mm_populate(addr, len); |
e4eb1ff6 | 3164 | return ret; |
2e7ce7d3 LH |
3165 | |
3166 | munmap_failed: | |
3167 | limits_failed: | |
3168 | mmap_write_unlock(mm); | |
3169 | return ret; | |
e4eb1ff6 | 3170 | } |
16e72e9b DV |
3171 | EXPORT_SYMBOL(vm_brk_flags); |
3172 | ||
3173 | int vm_brk(unsigned long addr, unsigned long len) | |
3174 | { | |
3175 | return vm_brk_flags(addr, len, 0); | |
3176 | } | |
e4eb1ff6 | 3177 | EXPORT_SYMBOL(vm_brk); |
1da177e4 LT |
3178 | |
3179 | /* Release all mmaps. */ | |
3180 | void exit_mmap(struct mm_struct *mm) | |
3181 | { | |
d16dfc55 | 3182 | struct mmu_gather tlb; |
ba470de4 | 3183 | struct vm_area_struct *vma; |
1da177e4 LT |
3184 | unsigned long nr_accounted = 0; |
3185 | ||
d6dd61c8 | 3186 | /* mm's last user has gone, and its about to be pulled down */ |
cddb8a5c | 3187 | mmu_notifier_release(mm); |
d6dd61c8 | 3188 | |
27ae357f DR |
3189 | if (unlikely(mm_is_oom_victim(mm))) { |
3190 | /* | |
3191 | * Manually reap the mm to free as much memory as possible. | |
3192 | * Then, as the oom reaper does, set MMF_OOM_SKIP to disregard | |
c1e8d7c6 | 3193 | * this mm from further consideration. Taking mm->mmap_lock for |
27ae357f | 3194 | * write after setting MMF_OOM_SKIP will guarantee that the oom |
c1e8d7c6 | 3195 | * reaper will not run on this mm again after mmap_lock is |
27ae357f DR |
3196 | * dropped. |
3197 | * | |
c1e8d7c6 | 3198 | * Nothing can be holding mm->mmap_lock here and the above call |
27ae357f DR |
3199 | * to mmu_notifier_release(mm) ensures mmu notifier callbacks in |
3200 | * __oom_reap_task_mm() will not block. | |
27ae357f | 3201 | */ |
93065ac7 | 3202 | (void)__oom_reap_task_mm(mm); |
27ae357f | 3203 | set_bit(MMF_OOM_SKIP, &mm->flags); |
27ae357f DR |
3204 | } |
3205 | ||
64591e86 | 3206 | mmap_write_lock(mm); |
9480c53e JF |
3207 | arch_exit_mmap(mm); |
3208 | ||
ba470de4 | 3209 | vma = mm->mmap; |
64591e86 SB |
3210 | if (!vma) { |
3211 | /* Can happen if dup_mmap() received an OOM */ | |
3212 | mmap_write_unlock(mm); | |
9480c53e | 3213 | return; |
64591e86 | 3214 | } |
9480c53e | 3215 | |
1da177e4 | 3216 | lru_add_drain(); |
1da177e4 | 3217 | flush_cache_mm(mm); |
d8b45053 | 3218 | tlb_gather_mmu_fullmm(&tlb, mm); |
901608d9 | 3219 | /* update_hiwater_rss(mm) here? but nobody should be looking */ |
e0da382c | 3220 | /* Use -1 here to ensure all VMAs in the mm are unmapped */ |
4f74d2c8 | 3221 | unmap_vmas(&tlb, vma, 0, -1); |
6ee8630e | 3222 | free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); |
ae8eba8b | 3223 | tlb_finish_mmu(&tlb); |
1da177e4 | 3224 | |
64591e86 | 3225 | /* Walk the list again, actually closing and freeing it. */ |
4f74d2c8 LT |
3226 | while (vma) { |
3227 | if (vma->vm_flags & VM_ACCOUNT) | |
3228 | nr_accounted += vma_pages(vma); | |
a8fb5618 | 3229 | vma = remove_vma(vma); |
0a3b3c25 | 3230 | cond_resched(); |
4f74d2c8 | 3231 | } |
d4af56c5 LH |
3232 | |
3233 | trace_exit_mmap(mm); | |
3234 | __mt_destroy(&mm->mm_mt); | |
f798a1d4 | 3235 | mm->mmap = NULL; |
64591e86 | 3236 | mmap_write_unlock(mm); |
4f74d2c8 | 3237 | vm_unacct_memory(nr_accounted); |
1da177e4 LT |
3238 | } |
3239 | ||
3240 | /* Insert vm structure into process list sorted by address | |
3241 | * and into the inode's i_mmap tree. If vm_file is non-NULL | |
c8c06efa | 3242 | * then i_mmap_rwsem is taken here. |
1da177e4 | 3243 | */ |
6597d783 | 3244 | int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) |
1da177e4 | 3245 | { |
6597d783 | 3246 | struct vm_area_struct *prev; |
d4af56c5 | 3247 | unsigned long charged = vma_pages(vma); |
1da177e4 | 3248 | |
d4af56c5 | 3249 | |
524e00b3 | 3250 | if (range_has_overlap(mm, vma->vm_start, vma->vm_end, &prev)) |
c9d13f5f | 3251 | return -ENOMEM; |
d4af56c5 | 3252 | |
c9d13f5f | 3253 | if ((vma->vm_flags & VM_ACCOUNT) && |
d4af56c5 | 3254 | security_vm_enough_memory_mm(mm, charged)) |
c9d13f5f CG |
3255 | return -ENOMEM; |
3256 | ||
1da177e4 LT |
3257 | /* |
3258 | * The vm_pgoff of a purely anonymous vma should be irrelevant | |
3259 | * until its first write fault, when page's anon_vma and index | |
3260 | * are set. But now set the vm_pgoff it will almost certainly | |
3261 | * end up with (unless mremap moves it elsewhere before that | |
3262 | * first wfault), so /proc/pid/maps tells a consistent story. | |
3263 | * | |
3264 | * By setting it to reflect the virtual start address of the | |
3265 | * vma, merges and splits can happen in a seamless way, just | |
3266 | * using the existing file pgoff checks and manipulations. | |
8332326e | 3267 | * Similarly in do_mmap and in do_brk_flags. |
1da177e4 | 3268 | */ |
8a9cc3b5 | 3269 | if (vma_is_anonymous(vma)) { |
1da177e4 LT |
3270 | BUG_ON(vma->anon_vma); |
3271 | vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; | |
3272 | } | |
2b144498 | 3273 | |
524e00b3 | 3274 | if (vma_link(mm, vma, prev)) { |
d4af56c5 LH |
3275 | vm_unacct_memory(charged); |
3276 | return -ENOMEM; | |
3277 | } | |
3278 | ||
1da177e4 LT |
3279 | return 0; |
3280 | } | |
3281 | ||
3282 | /* | |
3283 | * Copy the vma structure to a new location in the same mm, | |
3284 | * prior to moving page table entries, to effect an mremap move. | |
3285 | */ | |
3286 | struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, | |
38a76013 ML |
3287 | unsigned long addr, unsigned long len, pgoff_t pgoff, |
3288 | bool *need_rmap_locks) | |
1da177e4 LT |
3289 | { |
3290 | struct vm_area_struct *vma = *vmap; | |
3291 | unsigned long vma_start = vma->vm_start; | |
3292 | struct mm_struct *mm = vma->vm_mm; | |
3293 | struct vm_area_struct *new_vma, *prev; | |
948f017b | 3294 | bool faulted_in_anon_vma = true; |
1da177e4 | 3295 | |
d4af56c5 | 3296 | validate_mm_mt(mm); |
1da177e4 LT |
3297 | /* |
3298 | * If anonymous vma has not yet been faulted, update new pgoff | |
3299 | * to match new location, to increase its chance of merging. | |
3300 | */ | |
ce75799b | 3301 | if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { |
1da177e4 | 3302 | pgoff = addr >> PAGE_SHIFT; |
948f017b AA |
3303 | faulted_in_anon_vma = false; |
3304 | } | |
1da177e4 | 3305 | |
524e00b3 | 3306 | if (range_has_overlap(mm, addr, addr + len, &prev)) |
6597d783 | 3307 | return NULL; /* should never get here */ |
524e00b3 | 3308 | |
1da177e4 | 3309 | new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, |
19a809af | 3310 | vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), |
5c26f6ac | 3311 | vma->vm_userfaultfd_ctx, anon_vma_name(vma)); |
1da177e4 LT |
3312 | if (new_vma) { |
3313 | /* | |
3314 | * Source vma may have been merged into new_vma | |
3315 | */ | |
948f017b AA |
3316 | if (unlikely(vma_start >= new_vma->vm_start && |
3317 | vma_start < new_vma->vm_end)) { | |
3318 | /* | |
3319 | * The only way we can get a vma_merge with | |
3320 | * self during an mremap is if the vma hasn't | |
3321 | * been faulted in yet and we were allowed to | |
3322 | * reset the dst vma->vm_pgoff to the | |
3323 | * destination address of the mremap to allow | |
3324 | * the merge to happen. mremap must change the | |
3325 | * vm_pgoff linearity between src and dst vmas | |
3326 | * (in turn preventing a vma_merge) to be | |
3327 | * safe. It is only safe to keep the vm_pgoff | |
3328 | * linear if there are no pages mapped yet. | |
3329 | */ | |
81d1b09c | 3330 | VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); |
38a76013 | 3331 | *vmap = vma = new_vma; |
108d6642 | 3332 | } |
38a76013 | 3333 | *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); |
1da177e4 | 3334 | } else { |
3928d4f5 | 3335 | new_vma = vm_area_dup(vma); |
e3975891 CG |
3336 | if (!new_vma) |
3337 | goto out; | |
e3975891 CG |
3338 | new_vma->vm_start = addr; |
3339 | new_vma->vm_end = addr + len; | |
3340 | new_vma->vm_pgoff = pgoff; | |
3341 | if (vma_dup_policy(vma, new_vma)) | |
3342 | goto out_free_vma; | |
e3975891 CG |
3343 | if (anon_vma_clone(new_vma, vma)) |
3344 | goto out_free_mempol; | |
3345 | if (new_vma->vm_file) | |
3346 | get_file(new_vma->vm_file); | |
3347 | if (new_vma->vm_ops && new_vma->vm_ops->open) | |
3348 | new_vma->vm_ops->open(new_vma); | |
524e00b3 LH |
3349 | if (vma_link(mm, new_vma, prev)) |
3350 | goto out_vma_link; | |
e3975891 | 3351 | *need_rmap_locks = false; |
1da177e4 | 3352 | } |
d4af56c5 | 3353 | validate_mm_mt(mm); |
1da177e4 | 3354 | return new_vma; |
5beb4930 | 3355 | |
524e00b3 LH |
3356 | out_vma_link: |
3357 | if (new_vma->vm_ops && new_vma->vm_ops->close) | |
3358 | new_vma->vm_ops->close(new_vma); | |
e3975891 | 3359 | out_free_mempol: |
ef0855d3 | 3360 | mpol_put(vma_policy(new_vma)); |
e3975891 | 3361 | out_free_vma: |
3928d4f5 | 3362 | vm_area_free(new_vma); |
e3975891 | 3363 | out: |
d4af56c5 | 3364 | validate_mm_mt(mm); |
5beb4930 | 3365 | return NULL; |
1da177e4 | 3366 | } |
119f657c | 3367 | |
3368 | /* | |
3369 | * Return true if the calling process may expand its vm space by the passed | |
3370 | * number of pages | |
3371 | */ | |
84638335 | 3372 | bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages) |
119f657c | 3373 | { |
84638335 KK |
3374 | if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) |
3375 | return false; | |
119f657c | 3376 | |
d977d56c KK |
3377 | if (is_data_mapping(flags) && |
3378 | mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { | |
f4fcd558 KK |
3379 | /* Workaround for Valgrind */ |
3380 | if (rlimit(RLIMIT_DATA) == 0 && | |
3381 | mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) | |
3382 | return true; | |
57a7702b DW |
3383 | |
3384 | pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", | |
3385 | current->comm, current->pid, | |
3386 | (mm->data_vm + npages) << PAGE_SHIFT, | |
3387 | rlimit(RLIMIT_DATA), | |
3388 | ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); | |
3389 | ||
3390 | if (!ignore_rlimit_data) | |
d977d56c KK |
3391 | return false; |
3392 | } | |
119f657c | 3393 | |
84638335 KK |
3394 | return true; |
3395 | } | |
3396 | ||
3397 | void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages) | |
3398 | { | |
7866076b | 3399 | WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages); |
84638335 | 3400 | |
d977d56c | 3401 | if (is_exec_mapping(flags)) |
84638335 | 3402 | mm->exec_vm += npages; |
d977d56c | 3403 | else if (is_stack_mapping(flags)) |
84638335 | 3404 | mm->stack_vm += npages; |
d977d56c | 3405 | else if (is_data_mapping(flags)) |
84638335 | 3406 | mm->data_vm += npages; |
119f657c | 3407 | } |
fa5dc22f | 3408 | |
b3ec9f33 | 3409 | static vm_fault_t special_mapping_fault(struct vm_fault *vmf); |
a62c34bd AL |
3410 | |
3411 | /* | |
3412 | * Having a close hook prevents vma merging regardless of flags. | |
3413 | */ | |
3414 | static void special_mapping_close(struct vm_area_struct *vma) | |
3415 | { | |
3416 | } | |
3417 | ||
3418 | static const char *special_mapping_name(struct vm_area_struct *vma) | |
3419 | { | |
3420 | return ((struct vm_special_mapping *)vma->vm_private_data)->name; | |
3421 | } | |
3422 | ||
14d07113 | 3423 | static int special_mapping_mremap(struct vm_area_struct *new_vma) |
b059a453 DS |
3424 | { |
3425 | struct vm_special_mapping *sm = new_vma->vm_private_data; | |
3426 | ||
280e87e9 DS |
3427 | if (WARN_ON_ONCE(current->mm != new_vma->vm_mm)) |
3428 | return -EFAULT; | |
3429 | ||
b059a453 DS |
3430 | if (sm->mremap) |
3431 | return sm->mremap(sm, new_vma); | |
280e87e9 | 3432 | |
b059a453 DS |
3433 | return 0; |
3434 | } | |
3435 | ||
871402e0 DS |
3436 | static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) |
3437 | { | |
3438 | /* | |
3439 | * Forbid splitting special mappings - kernel has expectations over | |
3440 | * the number of pages in mapping. Together with VM_DONTEXPAND | |
3441 | * the size of vma should stay the same over the special mapping's | |
3442 | * lifetime. | |
3443 | */ | |
3444 | return -EINVAL; | |
3445 | } | |
3446 | ||
a62c34bd AL |
3447 | static const struct vm_operations_struct special_mapping_vmops = { |
3448 | .close = special_mapping_close, | |
3449 | .fault = special_mapping_fault, | |
b059a453 | 3450 | .mremap = special_mapping_mremap, |
a62c34bd | 3451 | .name = special_mapping_name, |
af34ebeb DS |
3452 | /* vDSO code relies that VVAR can't be accessed remotely */ |
3453 | .access = NULL, | |
871402e0 | 3454 | .may_split = special_mapping_split, |
a62c34bd AL |
3455 | }; |
3456 | ||
3457 | static const struct vm_operations_struct legacy_special_mapping_vmops = { | |
3458 | .close = special_mapping_close, | |
3459 | .fault = special_mapping_fault, | |
3460 | }; | |
fa5dc22f | 3461 | |
b3ec9f33 | 3462 | static vm_fault_t special_mapping_fault(struct vm_fault *vmf) |
fa5dc22f | 3463 | { |
11bac800 | 3464 | struct vm_area_struct *vma = vmf->vma; |
b1d0e4f5 | 3465 | pgoff_t pgoff; |
fa5dc22f RM |
3466 | struct page **pages; |
3467 | ||
f872f540 | 3468 | if (vma->vm_ops == &legacy_special_mapping_vmops) { |
a62c34bd | 3469 | pages = vma->vm_private_data; |
f872f540 AL |
3470 | } else { |
3471 | struct vm_special_mapping *sm = vma->vm_private_data; | |
3472 | ||
3473 | if (sm->fault) | |
11bac800 | 3474 | return sm->fault(sm, vmf->vma, vmf); |
f872f540 AL |
3475 | |
3476 | pages = sm->pages; | |
3477 | } | |
a62c34bd | 3478 | |
8a9cc3b5 | 3479 | for (pgoff = vmf->pgoff; pgoff && *pages; ++pages) |
b1d0e4f5 | 3480 | pgoff--; |
fa5dc22f RM |
3481 | |
3482 | if (*pages) { | |
3483 | struct page *page = *pages; | |
3484 | get_page(page); | |
b1d0e4f5 NP |
3485 | vmf->page = page; |
3486 | return 0; | |
fa5dc22f RM |
3487 | } |
3488 | ||
b1d0e4f5 | 3489 | return VM_FAULT_SIGBUS; |
fa5dc22f RM |
3490 | } |
3491 | ||
a62c34bd AL |
3492 | static struct vm_area_struct *__install_special_mapping( |
3493 | struct mm_struct *mm, | |
3494 | unsigned long addr, unsigned long len, | |
27f28b97 CG |
3495 | unsigned long vm_flags, void *priv, |
3496 | const struct vm_operations_struct *ops) | |
fa5dc22f | 3497 | { |
462e635e | 3498 | int ret; |
fa5dc22f RM |
3499 | struct vm_area_struct *vma; |
3500 | ||
d4af56c5 | 3501 | validate_mm_mt(mm); |
490fc053 | 3502 | vma = vm_area_alloc(mm); |
fa5dc22f | 3503 | if (unlikely(vma == NULL)) |
3935ed6a | 3504 | return ERR_PTR(-ENOMEM); |
fa5dc22f | 3505 | |
fa5dc22f RM |
3506 | vma->vm_start = addr; |
3507 | vma->vm_end = addr + len; | |
3508 | ||
d9104d1c | 3509 | vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY; |
1fc09228 | 3510 | vma->vm_flags &= VM_LOCKED_CLEAR_MASK; |
3ed75eb8 | 3511 | vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); |
fa5dc22f | 3512 | |
a62c34bd AL |
3513 | vma->vm_ops = ops; |
3514 | vma->vm_private_data = priv; | |
fa5dc22f | 3515 | |
462e635e TO |
3516 | ret = insert_vm_struct(mm, vma); |
3517 | if (ret) | |
3518 | goto out; | |
fa5dc22f | 3519 | |
84638335 | 3520 | vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); |
fa5dc22f | 3521 | |
cdd6c482 | 3522 | perf_event_mmap(vma); |
089dd79d | 3523 | |
d4af56c5 | 3524 | validate_mm_mt(mm); |
3935ed6a | 3525 | return vma; |
462e635e TO |
3526 | |
3527 | out: | |
3928d4f5 | 3528 | vm_area_free(vma); |
d4af56c5 | 3529 | validate_mm_mt(mm); |
3935ed6a SS |
3530 | return ERR_PTR(ret); |
3531 | } | |
3532 | ||
2eefd878 DS |
3533 | bool vma_is_special_mapping(const struct vm_area_struct *vma, |
3534 | const struct vm_special_mapping *sm) | |
3535 | { | |
3536 | return vma->vm_private_data == sm && | |
3537 | (vma->vm_ops == &special_mapping_vmops || | |
3538 | vma->vm_ops == &legacy_special_mapping_vmops); | |
3539 | } | |
3540 | ||
a62c34bd | 3541 | /* |
c1e8d7c6 | 3542 | * Called with mm->mmap_lock held for writing. |
a62c34bd AL |
3543 | * Insert a new vma covering the given region, with the given flags. |
3544 | * Its pages are supplied by the given array of struct page *. | |
3545 | * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated. | |
3546 | * The region past the last page supplied will always produce SIGBUS. | |
3547 | * The array pointer and the pages it points to are assumed to stay alive | |
3548 | * for as long as this mapping might exist. | |
3549 | */ | |
3550 | struct vm_area_struct *_install_special_mapping( | |
3551 | struct mm_struct *mm, | |
3552 | unsigned long addr, unsigned long len, | |
3553 | unsigned long vm_flags, const struct vm_special_mapping *spec) | |
3554 | { | |
27f28b97 CG |
3555 | return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec, |
3556 | &special_mapping_vmops); | |
a62c34bd AL |
3557 | } |
3558 | ||
3935ed6a SS |
3559 | int install_special_mapping(struct mm_struct *mm, |
3560 | unsigned long addr, unsigned long len, | |
3561 | unsigned long vm_flags, struct page **pages) | |
3562 | { | |
a62c34bd | 3563 | struct vm_area_struct *vma = __install_special_mapping( |
27f28b97 CG |
3564 | mm, addr, len, vm_flags, (void *)pages, |
3565 | &legacy_special_mapping_vmops); | |
3935ed6a | 3566 | |
14bd5b45 | 3567 | return PTR_ERR_OR_ZERO(vma); |
fa5dc22f | 3568 | } |
7906d00c AA |
3569 | |
3570 | static DEFINE_MUTEX(mm_all_locks_mutex); | |
3571 | ||
454ed842 | 3572 | static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) |
7906d00c | 3573 | { |
f808c13f | 3574 | if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { |
7906d00c AA |
3575 | /* |
3576 | * The LSB of head.next can't change from under us | |
3577 | * because we hold the mm_all_locks_mutex. | |
3578 | */ | |
da1c55f1 | 3579 | down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock); |
7906d00c AA |
3580 | /* |
3581 | * We can safely modify head.next after taking the | |
5a505085 | 3582 | * anon_vma->root->rwsem. If some other vma in this mm shares |
7906d00c AA |
3583 | * the same anon_vma we won't take it again. |
3584 | * | |
3585 | * No need of atomic instructions here, head.next | |
3586 | * can't change from under us thanks to the | |
5a505085 | 3587 | * anon_vma->root->rwsem. |
7906d00c AA |
3588 | */ |
3589 | if (__test_and_set_bit(0, (unsigned long *) | |
f808c13f | 3590 | &anon_vma->root->rb_root.rb_root.rb_node)) |
7906d00c AA |
3591 | BUG(); |
3592 | } | |
3593 | } | |
3594 | ||
454ed842 | 3595 | static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) |
7906d00c AA |
3596 | { |
3597 | if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { | |
3598 | /* | |
3599 | * AS_MM_ALL_LOCKS can't change from under us because | |
3600 | * we hold the mm_all_locks_mutex. | |
3601 | * | |
3602 | * Operations on ->flags have to be atomic because | |
3603 | * even if AS_MM_ALL_LOCKS is stable thanks to the | |
3604 | * mm_all_locks_mutex, there may be other cpus | |
3605 | * changing other bitflags in parallel to us. | |
3606 | */ | |
3607 | if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) | |
3608 | BUG(); | |
da1c55f1 | 3609 | down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock); |
7906d00c AA |
3610 | } |
3611 | } | |
3612 | ||
3613 | /* | |
3614 | * This operation locks against the VM for all pte/vma/mm related | |
3615 | * operations that could ever happen on a certain mm. This includes | |
3616 | * vmtruncate, try_to_unmap, and all page faults. | |
3617 | * | |
c1e8d7c6 | 3618 | * The caller must take the mmap_lock in write mode before calling |
7906d00c | 3619 | * mm_take_all_locks(). The caller isn't allowed to release the |
c1e8d7c6 | 3620 | * mmap_lock until mm_drop_all_locks() returns. |
7906d00c | 3621 | * |
c1e8d7c6 | 3622 | * mmap_lock in write mode is required in order to block all operations |
7906d00c | 3623 | * that could modify pagetables and free pages without need of |
27ba0644 | 3624 | * altering the vma layout. It's also needed in write mode to avoid new |
7906d00c AA |
3625 | * anon_vmas to be associated with existing vmas. |
3626 | * | |
3627 | * A single task can't take more than one mm_take_all_locks() in a row | |
3628 | * or it would deadlock. | |
3629 | * | |
bf181b9f | 3630 | * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in |
7906d00c AA |
3631 | * mapping->flags avoid to take the same lock twice, if more than one |
3632 | * vma in this mm is backed by the same anon_vma or address_space. | |
3633 | * | |
88f306b6 KS |
3634 | * We take locks in following order, accordingly to comment at beginning |
3635 | * of mm/rmap.c: | |
3636 | * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for | |
3637 | * hugetlb mapping); | |
3638 | * - all i_mmap_rwsem locks; | |
3639 | * - all anon_vma->rwseml | |
3640 | * | |
3641 | * We can take all locks within these types randomly because the VM code | |
3642 | * doesn't nest them and we protected from parallel mm_take_all_locks() by | |
3643 | * mm_all_locks_mutex. | |
7906d00c AA |
3644 | * |
3645 | * mm_take_all_locks() and mm_drop_all_locks are expensive operations | |
3646 | * that may have to take thousand of locks. | |
3647 | * | |
3648 | * mm_take_all_locks() can fail if it's interrupted by signals. | |
3649 | */ | |
3650 | int mm_take_all_locks(struct mm_struct *mm) | |
3651 | { | |
3652 | struct vm_area_struct *vma; | |
5beb4930 | 3653 | struct anon_vma_chain *avc; |
7906d00c | 3654 | |
325bca1f | 3655 | mmap_assert_write_locked(mm); |
7906d00c AA |
3656 | |
3657 | mutex_lock(&mm_all_locks_mutex); | |
3658 | ||
3659 | for (vma = mm->mmap; vma; vma = vma->vm_next) { | |
3660 | if (signal_pending(current)) | |
3661 | goto out_unlock; | |
88f306b6 KS |
3662 | if (vma->vm_file && vma->vm_file->f_mapping && |
3663 | is_vm_hugetlb_page(vma)) | |
3664 | vm_lock_mapping(mm, vma->vm_file->f_mapping); | |
3665 | } | |
3666 | ||
3667 | for (vma = mm->mmap; vma; vma = vma->vm_next) { | |
3668 | if (signal_pending(current)) | |
3669 | goto out_unlock; | |
3670 | if (vma->vm_file && vma->vm_file->f_mapping && | |
3671 | !is_vm_hugetlb_page(vma)) | |
454ed842 | 3672 | vm_lock_mapping(mm, vma->vm_file->f_mapping); |
7906d00c | 3673 | } |
7cd5a02f PZ |
3674 | |
3675 | for (vma = mm->mmap; vma; vma = vma->vm_next) { | |
3676 | if (signal_pending(current)) | |
3677 | goto out_unlock; | |
3678 | if (vma->anon_vma) | |
5beb4930 RR |
3679 | list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) |
3680 | vm_lock_anon_vma(mm, avc->anon_vma); | |
7906d00c | 3681 | } |
7cd5a02f | 3682 | |
584cff54 | 3683 | return 0; |
7906d00c AA |
3684 | |
3685 | out_unlock: | |
584cff54 KC |
3686 | mm_drop_all_locks(mm); |
3687 | return -EINTR; | |
7906d00c AA |
3688 | } |
3689 | ||
3690 | static void vm_unlock_anon_vma(struct anon_vma *anon_vma) | |
3691 | { | |
f808c13f | 3692 | if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { |
7906d00c AA |
3693 | /* |
3694 | * The LSB of head.next can't change to 0 from under | |
3695 | * us because we hold the mm_all_locks_mutex. | |
3696 | * | |
3697 | * We must however clear the bitflag before unlocking | |
bf181b9f | 3698 | * the vma so the users using the anon_vma->rb_root will |
7906d00c AA |
3699 | * never see our bitflag. |
3700 | * | |
3701 | * No need of atomic instructions here, head.next | |
3702 | * can't change from under us until we release the | |
5a505085 | 3703 | * anon_vma->root->rwsem. |
7906d00c AA |
3704 | */ |
3705 | if (!__test_and_clear_bit(0, (unsigned long *) | |
f808c13f | 3706 | &anon_vma->root->rb_root.rb_root.rb_node)) |
7906d00c | 3707 | BUG(); |
08b52706 | 3708 | anon_vma_unlock_write(anon_vma); |
7906d00c AA |
3709 | } |
3710 | } | |
3711 | ||
3712 | static void vm_unlock_mapping(struct address_space *mapping) | |
3713 | { | |
3714 | if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { | |
3715 | /* | |
3716 | * AS_MM_ALL_LOCKS can't change to 0 from under us | |
3717 | * because we hold the mm_all_locks_mutex. | |
3718 | */ | |
83cde9e8 | 3719 | i_mmap_unlock_write(mapping); |
7906d00c AA |
3720 | if (!test_and_clear_bit(AS_MM_ALL_LOCKS, |
3721 | &mapping->flags)) | |
3722 | BUG(); | |
3723 | } | |
3724 | } | |
3725 | ||
3726 | /* | |
c1e8d7c6 | 3727 | * The mmap_lock cannot be released by the caller until |
7906d00c AA |
3728 | * mm_drop_all_locks() returns. |
3729 | */ | |
3730 | void mm_drop_all_locks(struct mm_struct *mm) | |
3731 | { | |
3732 | struct vm_area_struct *vma; | |
5beb4930 | 3733 | struct anon_vma_chain *avc; |
7906d00c | 3734 | |
325bca1f | 3735 | mmap_assert_write_locked(mm); |
7906d00c AA |
3736 | BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); |
3737 | ||
3738 | for (vma = mm->mmap; vma; vma = vma->vm_next) { | |
3739 | if (vma->anon_vma) | |
5beb4930 RR |
3740 | list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) |
3741 | vm_unlock_anon_vma(avc->anon_vma); | |
7906d00c AA |
3742 | if (vma->vm_file && vma->vm_file->f_mapping) |
3743 | vm_unlock_mapping(vma->vm_file->f_mapping); | |
3744 | } | |
3745 | ||
3746 | mutex_unlock(&mm_all_locks_mutex); | |
3747 | } | |
8feae131 DH |
3748 | |
3749 | /* | |
3edf41d8 | 3750 | * initialise the percpu counter for VM |
8feae131 DH |
3751 | */ |
3752 | void __init mmap_init(void) | |
3753 | { | |
00a62ce9 KM |
3754 | int ret; |
3755 | ||
908c7f19 | 3756 | ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL); |
00a62ce9 | 3757 | VM_BUG_ON(ret); |
8feae131 | 3758 | } |
c9b1d098 AS |
3759 | |
3760 | /* | |
3761 | * Initialise sysctl_user_reserve_kbytes. | |
3762 | * | |
3763 | * This is intended to prevent a user from starting a single memory hogging | |
3764 | * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER | |
3765 | * mode. | |
3766 | * | |
3767 | * The default value is min(3% of free memory, 128MB) | |
3768 | * 128MB is enough to recover with sshd/login, bash, and top/kill. | |
3769 | */ | |
1640879a | 3770 | static int init_user_reserve(void) |
c9b1d098 AS |
3771 | { |
3772 | unsigned long free_kbytes; | |
3773 | ||
c41f012a | 3774 | free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); |
c9b1d098 AS |
3775 | |
3776 | sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17); | |
3777 | return 0; | |
3778 | } | |
a64fb3cd | 3779 | subsys_initcall(init_user_reserve); |
4eeab4f5 AS |
3780 | |
3781 | /* | |
3782 | * Initialise sysctl_admin_reserve_kbytes. | |
3783 | * | |
3784 | * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin | |
3785 | * to log in and kill a memory hogging process. | |
3786 | * | |
3787 | * Systems with more than 256MB will reserve 8MB, enough to recover | |
3788 | * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will | |
3789 | * only reserve 3% of free pages by default. | |
3790 | */ | |
1640879a | 3791 | static int init_admin_reserve(void) |
4eeab4f5 AS |
3792 | { |
3793 | unsigned long free_kbytes; | |
3794 | ||
c41f012a | 3795 | free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); |
4eeab4f5 AS |
3796 | |
3797 | sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13); | |
3798 | return 0; | |
3799 | } | |
a64fb3cd | 3800 | subsys_initcall(init_admin_reserve); |
1640879a AS |
3801 | |
3802 | /* | |
3803 | * Reinititalise user and admin reserves if memory is added or removed. | |
3804 | * | |
3805 | * The default user reserve max is 128MB, and the default max for the | |
3806 | * admin reserve is 8MB. These are usually, but not always, enough to | |
3807 | * enable recovery from a memory hogging process using login/sshd, a shell, | |
3808 | * and tools like top. It may make sense to increase or even disable the | |
3809 | * reserve depending on the existence of swap or variations in the recovery | |
3810 | * tools. So, the admin may have changed them. | |
3811 | * | |
3812 | * If memory is added and the reserves have been eliminated or increased above | |
3813 | * the default max, then we'll trust the admin. | |
3814 | * | |
3815 | * If memory is removed and there isn't enough free memory, then we | |
3816 | * need to reset the reserves. | |
3817 | * | |
3818 | * Otherwise keep the reserve set by the admin. | |
3819 | */ | |
3820 | static int reserve_mem_notifier(struct notifier_block *nb, | |
3821 | unsigned long action, void *data) | |
3822 | { | |
3823 | unsigned long tmp, free_kbytes; | |
3824 | ||
3825 | switch (action) { | |
3826 | case MEM_ONLINE: | |
3827 | /* Default max is 128MB. Leave alone if modified by operator. */ | |
3828 | tmp = sysctl_user_reserve_kbytes; | |
3829 | if (0 < tmp && tmp < (1UL << 17)) | |
3830 | init_user_reserve(); | |
3831 | ||
3832 | /* Default max is 8MB. Leave alone if modified by operator. */ | |
3833 | tmp = sysctl_admin_reserve_kbytes; | |
3834 | if (0 < tmp && tmp < (1UL << 13)) | |
3835 | init_admin_reserve(); | |
3836 | ||
3837 | break; | |
3838 | case MEM_OFFLINE: | |
c41f012a | 3839 | free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); |
1640879a AS |
3840 | |
3841 | if (sysctl_user_reserve_kbytes > free_kbytes) { | |
3842 | init_user_reserve(); | |
3843 | pr_info("vm.user_reserve_kbytes reset to %lu\n", | |
3844 | sysctl_user_reserve_kbytes); | |
3845 | } | |
3846 | ||
3847 | if (sysctl_admin_reserve_kbytes > free_kbytes) { | |
3848 | init_admin_reserve(); | |
3849 | pr_info("vm.admin_reserve_kbytes reset to %lu\n", | |
3850 | sysctl_admin_reserve_kbytes); | |
3851 | } | |
3852 | break; | |
3853 | default: | |
3854 | break; | |
3855 | } | |
3856 | return NOTIFY_OK; | |
3857 | } | |
3858 | ||
3859 | static struct notifier_block reserve_mem_nb = { | |
3860 | .notifier_call = reserve_mem_notifier, | |
3861 | }; | |
3862 | ||
3863 | static int __meminit init_reserve_notifier(void) | |
3864 | { | |
3865 | if (register_hotmemory_notifier(&reserve_mem_nb)) | |
b1de0d13 | 3866 | pr_err("Failed registering memory add/remove notifier for admin reserve\n"); |
1640879a AS |
3867 | |
3868 | return 0; | |
3869 | } | |
a64fb3cd | 3870 | subsys_initcall(init_reserve_notifier); |