]>
Commit | Line | Data |
---|---|---|
2cb7cef9 BS |
1 | From: Andrea Arcangeli <andrea@suse.de> |
2 | Subject: avoid silent stack overflow over the heap | |
3 | Patch-mainline: no | |
4 | References: bnc#44807 bnc#211997 | |
5 | ||
6 | x | |
7 | ||
8 | Signed-off-by: Andrea Arcangeli <andrea@suse.de> | |
9 | Updated-by: Jeff Mahoney <jeffm@suse.com> | |
10 | --- | |
11 | ||
12 | arch/alpha/mm/fault.c | 2 - | |
13 | arch/arm/mm/fault.c | 2 - | |
14 | arch/cris/mm/fault.c | 2 - | |
15 | arch/frv/mm/fault.c | 2 - | |
16 | arch/ia64/mm/fault.c | 2 - | |
17 | arch/m32r/mm/fault.c | 2 - | |
18 | arch/m68k/mm/fault.c | 2 - | |
19 | arch/mips/mm/fault.c | 2 - | |
20 | arch/parisc/mm/fault.c | 2 - | |
21 | arch/powerpc/mm/fault.c | 5 ++- | |
22 | arch/powerpc/platforms/cell/spu_fault.c | 2 - | |
23 | arch/s390/mm/fault.c | 2 - | |
24 | arch/sh/mm/fault_32.c | 2 - | |
25 | arch/sh/mm/tlbflush_64.c | 2 - | |
26 | arch/sparc/mm/fault.c | 4 +-- | |
27 | arch/sparc64/mm/fault.c | 2 - | |
28 | arch/um/kernel/trap.c | 9 ++++-- | |
29 | arch/x86/kernel/sys_x86_64.c | 10 ++++++- | |
30 | arch/x86/mm/fault.c | 10 ++++++- | |
31 | arch/xtensa/mm/fault.c | 2 - | |
32 | include/linux/mm.h | 5 +++ | |
33 | kernel/sysctl.c | 8 ++++++ | |
34 | mm/mmap.c | 42 +++++++++++++++++++++++++------- | |
35 | 23 files changed, 88 insertions(+), 35 deletions(-) | |
36 | ||
37 | --- a/arch/alpha/mm/fault.c | |
38 | +++ b/arch/alpha/mm/fault.c | |
39 | @@ -123,7 +123,7 @@ do_page_fault(unsigned long address, uns | |
40 | goto good_area; | |
41 | if (!(vma->vm_flags & VM_GROWSDOWN)) | |
42 | goto bad_area; | |
43 | - if (expand_stack(vma, address)) | |
44 | + if (expand_stack(vma, address, NULL)) | |
45 | goto bad_area; | |
46 | ||
47 | /* Ok, we have a good vm_area for this memory access, so | |
48 | --- a/arch/arm/mm/fault.c | |
49 | +++ b/arch/arm/mm/fault.c | |
50 | @@ -233,7 +233,7 @@ out_of_memory: | |
51 | goto survive; | |
52 | ||
53 | check_stack: | |
54 | - if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) | |
55 | + if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr, NULL)) | |
56 | goto good_area; | |
57 | out: | |
58 | return fault; | |
59 | --- a/arch/cris/mm/fault.c | |
60 | +++ b/arch/cris/mm/fault.c | |
61 | @@ -133,7 +133,7 @@ do_page_fault(unsigned long address, str | |
62 | if (address + PAGE_SIZE < rdusp()) | |
63 | goto bad_area; | |
64 | } | |
65 | - if (expand_stack(vma, address)) | |
66 | + if (expand_stack(vma, address, NULL)) | |
67 | goto bad_area; | |
68 | ||
69 | /* | |
70 | --- a/arch/frv/mm/fault.c | |
71 | +++ b/arch/frv/mm/fault.c | |
72 | @@ -121,7 +121,7 @@ asmlinkage void do_page_fault(int datamm | |
73 | } | |
74 | } | |
75 | ||
76 | - if (expand_stack(vma, ear0)) | |
77 | + if (expand_stack(vma, ear0, NULL)) | |
78 | goto bad_area; | |
79 | ||
80 | /* | |
81 | --- a/arch/ia64/mm/fault.c | |
82 | +++ b/arch/ia64/mm/fault.c | |
83 | @@ -185,7 +185,7 @@ ia64_do_page_fault (unsigned long addres | |
84 | if (REGION_NUMBER(address) != REGION_NUMBER(vma->vm_start) | |
85 | || REGION_OFFSET(address) >= RGN_MAP_LIMIT) | |
86 | goto bad_area; | |
87 | - if (expand_stack(vma, address)) | |
88 | + if (expand_stack(vma, address, NULL /* FIXME? */)) | |
89 | goto bad_area; | |
90 | } else { | |
91 | vma = prev_vma; | |
92 | --- a/arch/m32r/mm/fault.c | |
93 | +++ b/arch/m32r/mm/fault.c | |
94 | @@ -159,7 +159,7 @@ asmlinkage void do_page_fault(struct pt_ | |
95 | goto bad_area; | |
96 | } | |
97 | ||
98 | - if (expand_stack(vma, address)) | |
99 | + if (expand_stack(vma, address, NULL)) | |
100 | goto bad_area; | |
101 | /* | |
102 | * Ok, we have a good vm_area for this memory access, so | |
103 | --- a/arch/m68k/mm/fault.c | |
104 | +++ b/arch/m68k/mm/fault.c | |
105 | @@ -121,7 +121,7 @@ int do_page_fault(struct pt_regs *regs, | |
106 | if (address + 256 < rdusp()) | |
107 | goto map_err; | |
108 | } | |
109 | - if (expand_stack(vma, address)) | |
110 | + if (expand_stack(vma, address, NULL)) | |
111 | goto map_err; | |
112 | ||
113 | /* | |
114 | --- a/arch/mips/mm/fault.c | |
115 | +++ b/arch/mips/mm/fault.c | |
116 | @@ -80,7 +80,7 @@ asmlinkage void do_page_fault(struct pt_ | |
117 | goto good_area; | |
118 | if (!(vma->vm_flags & VM_GROWSDOWN)) | |
119 | goto bad_area; | |
120 | - if (expand_stack(vma, address)) | |
121 | + if (expand_stack(vma, address, NULL)) | |
122 | goto bad_area; | |
123 | /* | |
124 | * Ok, we have a good vm_area for this memory access, so | |
125 | --- a/arch/parisc/mm/fault.c | |
126 | +++ b/arch/parisc/mm/fault.c | |
127 | @@ -196,7 +196,7 @@ good_area: | |
128 | ||
129 | check_expansion: | |
130 | vma = prev_vma; | |
131 | - if (vma && (expand_stack(vma, address) == 0)) | |
132 | + if (vma && (expand_stack(vma, address, NULL) == 0)) | |
133 | goto good_area; | |
134 | ||
135 | /* | |
136 | --- a/arch/powerpc/mm/fault.c | |
137 | +++ b/arch/powerpc/mm/fault.c | |
138 | @@ -116,7 +116,7 @@ static int store_updates_sp(struct pt_re | |
139 | int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address, | |
140 | unsigned long error_code) | |
141 | { | |
142 | - struct vm_area_struct * vma; | |
143 | + struct vm_area_struct * vma, * prev_vma; | |
144 | struct mm_struct *mm = current->mm; | |
145 | siginfo_t info; | |
146 | int code = SEGV_MAPERR; | |
147 | @@ -230,7 +230,8 @@ int __kprobes do_page_fault(struct pt_re | |
148 | && (!user_mode(regs) || !store_updates_sp(regs))) | |
149 | goto bad_area; | |
150 | } | |
151 | - if (expand_stack(vma, address)) | |
152 | + find_vma_prev(mm, address, &prev_vma); | |
153 | + if (expand_stack(vma, address, prev_vma)) | |
154 | goto bad_area; | |
155 | ||
156 | good_area: | |
157 | --- a/arch/powerpc/platforms/cell/spu_fault.c | |
158 | +++ b/arch/powerpc/platforms/cell/spu_fault.c | |
159 | @@ -59,7 +59,7 @@ int spu_handle_mm_fault(struct mm_struct | |
160 | goto good_area; | |
161 | if (!(vma->vm_flags & VM_GROWSDOWN)) | |
162 | goto bad_area; | |
163 | - if (expand_stack(vma, ea)) | |
164 | + if (expand_stack(vma, ea, NULL)) | |
165 | goto bad_area; | |
166 | good_area: | |
167 | is_write = dsisr & MFC_DSISR_ACCESS_PUT; | |
168 | --- a/arch/s390/mm/fault.c | |
169 | +++ b/arch/s390/mm/fault.c | |
170 | @@ -350,7 +350,7 @@ do_exception(struct pt_regs *regs, unsig | |
171 | goto good_area; | |
172 | if (!(vma->vm_flags & VM_GROWSDOWN)) | |
173 | goto bad_area; | |
174 | - if (expand_stack(vma, address)) | |
175 | + if (expand_stack(vma, address, NULL /* FIXME? */)) | |
176 | goto bad_area; | |
177 | /* | |
178 | * Ok, we have a good vm_area for this memory access, so | |
179 | --- a/arch/sh/mm/fault_32.c | |
180 | +++ b/arch/sh/mm/fault_32.c | |
181 | @@ -108,7 +108,7 @@ asmlinkage void __kprobes do_page_fault( | |
182 | goto good_area; | |
183 | if (!(vma->vm_flags & VM_GROWSDOWN)) | |
184 | goto bad_area; | |
185 | - if (expand_stack(vma, address)) | |
186 | + if (expand_stack(vma, address, NULL)) | |
187 | goto bad_area; | |
188 | /* | |
189 | * Ok, we have a good vm_area for this memory access, so | |
190 | --- a/arch/sh/mm/tlbflush_64.c | |
191 | +++ b/arch/sh/mm/tlbflush_64.c | |
192 | @@ -153,7 +153,7 @@ asmlinkage void do_page_fault(struct pt_ | |
193 | #endif | |
194 | goto bad_area; | |
195 | } | |
196 | - if (expand_stack(vma, address)) { | |
197 | + if (expand_stack(vma, address, NULL)) { | |
198 | #ifdef DEBUG_FAULT | |
199 | print_task(tsk); | |
200 | printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", | |
201 | --- a/arch/sparc/mm/fault.c | |
202 | +++ b/arch/sparc/mm/fault.c | |
203 | @@ -219,7 +219,7 @@ asmlinkage void do_sparc_fault(struct pt | |
204 | goto good_area; | |
205 | if(!(vma->vm_flags & VM_GROWSDOWN)) | |
206 | goto bad_area; | |
207 | - if(expand_stack(vma, address)) | |
208 | + if(expand_stack(vma, address, NULL)) | |
209 | goto bad_area; | |
210 | /* | |
211 | * Ok, we have a good vm_area for this memory access, so | |
212 | @@ -472,7 +472,7 @@ static void force_user_fault(unsigned lo | |
213 | goto good_area; | |
214 | if(!(vma->vm_flags & VM_GROWSDOWN)) | |
215 | goto bad_area; | |
216 | - if(expand_stack(vma, address)) | |
217 | + if(expand_stack(vma, address, NULL)) | |
218 | goto bad_area; | |
219 | good_area: | |
220 | info.si_code = SEGV_ACCERR; | |
221 | --- a/arch/sparc64/mm/fault.c | |
222 | +++ b/arch/sparc64/mm/fault.c | |
223 | @@ -367,7 +367,7 @@ continue_fault: | |
224 | goto bad_area; | |
225 | } | |
226 | } | |
227 | - if (expand_stack(vma, address)) | |
228 | + if (expand_stack(vma, address, NULL)) | |
229 | goto bad_area; | |
230 | /* | |
231 | * Ok, we have a good vm_area for this memory access, so | |
232 | --- a/arch/um/kernel/trap.c | |
233 | +++ b/arch/um/kernel/trap.c | |
234 | @@ -24,7 +24,7 @@ int handle_page_fault(unsigned long addr | |
235 | int is_write, int is_user, int *code_out) | |
236 | { | |
237 | struct mm_struct *mm = current->mm; | |
238 | - struct vm_area_struct *vma; | |
239 | + struct vm_area_struct *vma, *prev_vma; | |
240 | pgd_t *pgd; | |
241 | pud_t *pud; | |
242 | pmd_t *pmd; | |
243 | @@ -50,8 +50,11 @@ int handle_page_fault(unsigned long addr | |
244 | goto out; | |
245 | else if (is_user && !ARCH_IS_STACKGROW(address)) | |
246 | goto out; | |
247 | - else if (expand_stack(vma, address)) | |
248 | - goto out; | |
249 | + else { | |
250 | + find_vma_prev(mm, address, &prev_vma); | |
251 | + if(expand_stack(vma, address, prev_vma)) | |
252 | + goto out; | |
253 | + } | |
254 | ||
255 | good_area: | |
256 | *code_out = SEGV_ACCERR; | |
257 | --- a/arch/x86/kernel/sys_x86_64.c | |
258 | +++ b/arch/x86/kernel/sys_x86_64.c | |
259 | @@ -106,6 +106,7 @@ arch_get_unmapped_area(struct file *filp | |
260 | ||
261 | full_search: | |
262 | for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { | |
263 | + unsigned long __heap_stack_gap; | |
264 | /* At this point: (!vma || addr < vma->vm_end). */ | |
265 | if (end - len < addr) { | |
266 | /* | |
267 | @@ -119,7 +120,14 @@ full_search: | |
268 | } | |
269 | return -ENOMEM; | |
270 | } | |
271 | - if (!vma || addr + len <= vma->vm_start) { | |
272 | + if (!vma) | |
273 | + goto got_it; | |
274 | + __heap_stack_gap = 0; | |
275 | + if (vma->vm_flags & VM_GROWSDOWN) | |
276 | + __heap_stack_gap = min(end-(addr+len), | |
277 | + (unsigned long) heap_stack_gap << PAGE_SHIFT); | |
278 | + if (addr + len + __heap_stack_gap <= vma->vm_start) { | |
279 | + got_it: | |
280 | /* | |
281 | * Remember the place where we stopped the search: | |
282 | */ | |
283 | --- a/arch/x86/mm/fault.c | |
284 | +++ b/arch/x86/mm/fault.c | |
285 | @@ -585,7 +585,7 @@ void __kprobes do_page_fault(struct pt_r | |
286 | { | |
287 | struct task_struct *tsk; | |
288 | struct mm_struct *mm; | |
289 | - struct vm_area_struct *vma; | |
290 | + struct vm_area_struct *vma, *prev_vma; | |
291 | unsigned long address; | |
292 | int write, si_code; | |
293 | int fault; | |
294 | @@ -719,7 +719,13 @@ again: | |
295 | if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp) | |
296 | goto bad_area; | |
297 | } | |
298 | - if (expand_stack(vma, address)) | |
299 | + /* | |
300 | + * find_vma_prev is just a bit slower, because it cannot | |
301 | + * use the mmap_cache, so we run it only in the growsdown | |
302 | + * slow path and we leave find_vma in the fast path. | |
303 | + */ | |
304 | + find_vma_prev(current->mm, address, &prev_vma); | |
305 | + if (expand_stack(vma, address, prev_vma)) | |
306 | goto bad_area; | |
307 | /* | |
308 | * Ok, we have a good vm_area for this memory access, so | |
309 | --- a/arch/xtensa/mm/fault.c | |
310 | +++ b/arch/xtensa/mm/fault.c | |
311 | @@ -80,7 +80,7 @@ void do_page_fault(struct pt_regs *regs) | |
312 | goto good_area; | |
313 | if (!(vma->vm_flags & VM_GROWSDOWN)) | |
314 | goto bad_area; | |
315 | - if (expand_stack(vma, address)) | |
316 | + if (expand_stack(vma, address, NULL)) | |
317 | goto bad_area; | |
318 | ||
319 | /* Ok, we have a good vm_area for this memory access, so | |
320 | --- a/include/linux/mm.h | |
321 | +++ b/include/linux/mm.h | |
322 | @@ -1208,7 +1208,10 @@ void page_cache_async_readahead(struct a | |
323 | unsigned long max_sane_readahead(unsigned long nr); | |
324 | ||
325 | /* Do stack extension */ | |
326 | -extern int expand_stack(struct vm_area_struct *vma, unsigned long address); | |
327 | +#define EXPAND_STACK_HAS_3_ARGS | |
328 | +extern int heap_stack_gap; | |
329 | +extern int expand_stack(struct vm_area_struct * vma, unsigned long address, | |
330 | + struct vm_area_struct * prev_vma); | |
331 | #ifdef CONFIG_IA64 | |
332 | extern int expand_upwards(struct vm_area_struct *vma, unsigned long address); | |
333 | #endif | |
334 | --- a/kernel/sysctl.c | |
335 | +++ b/kernel/sysctl.c | |
336 | @@ -1208,6 +1208,14 @@ static struct ctl_table vm_table[] = { | |
337 | .extra2 = &one, | |
338 | }, | |
339 | #endif | |
340 | + { | |
341 | + .ctl_name = CTL_UNNUMBERED, | |
342 | + .procname = "heap-stack-gap", | |
343 | + .data = &heap_stack_gap, | |
344 | + .maxlen = sizeof(int), | |
345 | + .mode = 0644, | |
346 | + .proc_handler = &proc_dointvec, | |
347 | + }, | |
348 | /* | |
349 | * NOTE: do not add new entries to this table unless you have read | |
350 | * Documentation/sysctl/ctl_unnumbered.txt | |
351 | --- a/mm/mmap.c | |
352 | +++ b/mm/mmap.c | |
353 | @@ -85,6 +85,7 @@ int sysctl_overcommit_memory = OVERCOMMI | |
354 | int sysctl_overcommit_ratio = 50; /* default is 50% */ | |
355 | int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT; | |
356 | atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0); | |
357 | +int heap_stack_gap = 1; | |
358 | ||
359 | /* | |
360 | * Check that a process has enough memory to allocate a new virtual | |
361 | @@ -1291,6 +1292,7 @@ arch_get_unmapped_area(struct file *filp | |
362 | full_search: | |
363 | for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { | |
364 | /* At this point: (!vma || addr < vma->vm_end). */ | |
365 | + unsigned long __heap_stack_gap; | |
366 | if (TASK_SIZE - len < addr) { | |
367 | /* | |
368 | * Start a new search - just in case we missed | |
369 | @@ -1304,7 +1306,14 @@ full_search: | |
370 | } | |
371 | return -ENOMEM; | |
372 | } | |
373 | - if (!vma || addr + len <= vma->vm_start) { | |
374 | + if (!vma) | |
375 | + goto got_it; | |
376 | + __heap_stack_gap = 0; | |
377 | + if (vma->vm_flags & VM_GROWSDOWN) | |
378 | + __heap_stack_gap = min(TASK_SIZE-(addr+len), | |
379 | + (unsigned long) heap_stack_gap << PAGE_SHIFT); | |
380 | + if (addr + len + __heap_stack_gap <= vma->vm_start) { | |
381 | + got_it: | |
382 | /* | |
383 | * Remember the place where we stopped the search: | |
384 | */ | |
385 | @@ -1633,11 +1642,9 @@ int expand_upwards(struct vm_area_struct | |
386 | } | |
387 | #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ | |
388 | ||
389 | -/* | |
390 | - * vma is the first one with address < vma->vm_start. Have to extend vma. | |
391 | - */ | |
392 | static inline int expand_downwards(struct vm_area_struct *vma, | |
393 | - unsigned long address) | |
394 | + unsigned long address, | |
395 | + struct vm_area_struct *prev_vma) | |
396 | { | |
397 | int error; | |
398 | ||
399 | @@ -1665,6 +1672,13 @@ static inline int expand_downwards(struc | |
400 | if (address < vma->vm_start) { | |
401 | unsigned long size, grow; | |
402 | ||
403 | + error = -ENOMEM; | |
404 | + if (prev_vma) { | |
405 | + unsigned long __heap_stack_gap = min(TASK_SIZE-prev_vma->vm_end, | |
406 | + (unsigned long) heap_stack_gap << PAGE_SHIFT); | |
407 | + if (unlikely(prev_vma->vm_end + __heap_stack_gap > address)) | |
408 | + goto out_unlock; | |
409 | + } | |
410 | size = vma->vm_end - address; | |
411 | grow = (vma->vm_start - address) >> PAGE_SHIFT; | |
412 | ||
413 | @@ -1674,6 +1688,7 @@ static inline int expand_downwards(struc | |
414 | vma->vm_pgoff -= grow; | |
415 | } | |
416 | } | |
417 | + out_unlock: | |
418 | anon_vma_unlock(vma); | |
419 | return error; | |
420 | } | |
421 | @@ -1684,8 +1699,16 @@ int expand_stack_downwards(struct vm_are | |
422 | } | |
423 | ||
424 | #ifdef CONFIG_STACK_GROWSUP | |
425 | -int expand_stack(struct vm_area_struct *vma, unsigned long address) | |
426 | +int expand_stack(struct vm_area_struct * vma, unsigned long address, | |
427 | + struct vm_area_struct * prev_vma) | |
428 | { | |
429 | + /* | |
430 | + * If you re-use the heap-stack-gap for a growsup stack you | |
431 | + * should implement the feature for growsup too and remove | |
432 | + * this WARN_ON. | |
433 | + */ | |
434 | + WARN_ON(prev_vma); | |
435 | + | |
436 | return expand_upwards(vma, address); | |
437 | } | |
438 | ||
439 | @@ -1698,7 +1721,7 @@ find_extend_vma(struct mm_struct *mm, un | |
440 | vma = find_vma_prev(mm, addr, &prev); | |
441 | if (vma && (vma->vm_start <= addr)) | |
442 | return vma; | |
443 | - if (!prev || expand_stack(prev, addr)) | |
444 | + if (!prev || expand_stack(prev, addr, NULL)) | |
445 | return NULL; | |
446 | if (prev->vm_flags & VM_LOCKED) | |
447 | make_pages_present(addr, prev->vm_end); | |
448 | @@ -1713,7 +1736,7 @@ int expand_stack(struct vm_area_struct * | |
449 | struct vm_area_struct * | |
450 | find_extend_vma(struct mm_struct * mm, unsigned long addr) | |
451 | { | |
452 | - struct vm_area_struct * vma; | |
453 | + struct vm_area_struct * vma, * prev_vma; | |
454 | unsigned long start; | |
455 | ||
456 | addr &= PAGE_MASK; | |
457 | @@ -1725,7 +1748,8 @@ find_extend_vma(struct mm_struct * mm, u | |
458 | if (!(vma->vm_flags & VM_GROWSDOWN)) | |
459 | return NULL; | |
460 | start = vma->vm_start; | |
461 | - if (expand_stack(vma, addr)) | |
462 | + find_vma_prev(mm, addr, &prev_vma); | |
463 | + if (expand_stack(vma, addr, prev_vma)) | |
464 | return NULL; | |
465 | if (vma->vm_flags & VM_LOCKED) | |
466 | make_pages_present(addr, start); |