1 From: Andrea Arcangeli <andrea@suse.de>
2 Subject: avoid silent stack overflow over the heap
4 References: bnc#44807 bnc#211997
8 Signed-off-by: Andrea Arcangeli <andrea@suse.de>
9 Updated-by: Jeff Mahoney <jeffm@suse.com>
12 arch/alpha/mm/fault.c | 2 -
13 arch/arm/mm/fault.c | 2 -
14 arch/cris/mm/fault.c | 2 -
15 arch/frv/mm/fault.c | 2 -
16 arch/ia64/mm/fault.c | 2 -
17 arch/m32r/mm/fault.c | 2 -
18 arch/m68k/mm/fault.c | 2 -
19 arch/mips/mm/fault.c | 2 -
20 arch/parisc/mm/fault.c | 2 -
21 arch/powerpc/mm/fault.c | 5 ++-
22 arch/powerpc/platforms/cell/spu_fault.c | 2 -
23 arch/s390/mm/fault.c | 2 -
24 arch/sh/mm/fault_32.c | 2 -
25 arch/sh/mm/tlbflush_64.c | 2 -
26 arch/sparc/mm/fault.c | 4 +--
27 arch/sparc64/mm/fault.c | 2 -
28 arch/um/kernel/trap.c | 9 ++++--
29 arch/x86/kernel/sys_x86_64.c | 10 ++++++-
30 arch/x86/mm/fault.c | 10 ++++++-
31 arch/xtensa/mm/fault.c | 2 -
32 include/linux/mm.h | 5 +++
33 kernel/sysctl.c | 8 ++++++
34 mm/mmap.c | 42 +++++++++++++++++++++++++-------
35 23 files changed, 88 insertions(+), 35 deletions(-)
37 --- a/arch/alpha/mm/fault.c
38 +++ b/arch/alpha/mm/fault.c
39 @@ -123,7 +123,7 @@ do_page_fault(unsigned long address, uns
41 if (!(vma->vm_flags & VM_GROWSDOWN))
43 - if (expand_stack(vma, address))
44 + if (expand_stack(vma, address, NULL))
47 /* Ok, we have a good vm_area for this memory access, so
48 --- a/arch/arm/mm/fault.c
49 +++ b/arch/arm/mm/fault.c
50 @@ -233,7 +233,7 @@ out_of_memory:
54 - if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
55 + if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr, NULL))
59 --- a/arch/cris/mm/fault.c
60 +++ b/arch/cris/mm/fault.c
61 @@ -133,7 +133,7 @@ do_page_fault(unsigned long address, str
62 if (address + PAGE_SIZE < rdusp())
65 - if (expand_stack(vma, address))
66 + if (expand_stack(vma, address, NULL))
70 --- a/arch/frv/mm/fault.c
71 +++ b/arch/frv/mm/fault.c
72 @@ -121,7 +121,7 @@ asmlinkage void do_page_fault(int datamm
76 - if (expand_stack(vma, ear0))
77 + if (expand_stack(vma, ear0, NULL))
81 --- a/arch/ia64/mm/fault.c
82 +++ b/arch/ia64/mm/fault.c
83 @@ -185,7 +185,7 @@ ia64_do_page_fault (unsigned long addres
84 if (REGION_NUMBER(address) != REGION_NUMBER(vma->vm_start)
85 || REGION_OFFSET(address) >= RGN_MAP_LIMIT)
87 - if (expand_stack(vma, address))
88 + if (expand_stack(vma, address, NULL /* FIXME? */))
92 --- a/arch/m32r/mm/fault.c
93 +++ b/arch/m32r/mm/fault.c
94 @@ -159,7 +159,7 @@ asmlinkage void do_page_fault(struct pt_
98 - if (expand_stack(vma, address))
99 + if (expand_stack(vma, address, NULL))
102 * Ok, we have a good vm_area for this memory access, so
103 --- a/arch/m68k/mm/fault.c
104 +++ b/arch/m68k/mm/fault.c
105 @@ -121,7 +121,7 @@ int do_page_fault(struct pt_regs *regs,
106 if (address + 256 < rdusp())
109 - if (expand_stack(vma, address))
110 + if (expand_stack(vma, address, NULL))
114 --- a/arch/mips/mm/fault.c
115 +++ b/arch/mips/mm/fault.c
116 @@ -80,7 +80,7 @@ asmlinkage void do_page_fault(struct pt_
118 if (!(vma->vm_flags & VM_GROWSDOWN))
120 - if (expand_stack(vma, address))
121 + if (expand_stack(vma, address, NULL))
124 * Ok, we have a good vm_area for this memory access, so
125 --- a/arch/parisc/mm/fault.c
126 +++ b/arch/parisc/mm/fault.c
127 @@ -196,7 +196,7 @@ good_area:
131 - if (vma && (expand_stack(vma, address) == 0))
132 + if (vma && (expand_stack(vma, address, NULL) == 0))
136 --- a/arch/powerpc/mm/fault.c
137 +++ b/arch/powerpc/mm/fault.c
138 @@ -116,7 +116,7 @@ static int store_updates_sp(struct pt_re
139 int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
140 unsigned long error_code)
142 - struct vm_area_struct * vma;
143 + struct vm_area_struct * vma, * prev_vma;
144 struct mm_struct *mm = current->mm;
146 int code = SEGV_MAPERR;
147 @@ -230,7 +230,8 @@ int __kprobes do_page_fault(struct pt_re
148 && (!user_mode(regs) || !store_updates_sp(regs)))
151 - if (expand_stack(vma, address))
152 + find_vma_prev(mm, address, &prev_vma);
153 + if (expand_stack(vma, address, prev_vma))
157 --- a/arch/powerpc/platforms/cell/spu_fault.c
158 +++ b/arch/powerpc/platforms/cell/spu_fault.c
159 @@ -59,7 +59,7 @@ int spu_handle_mm_fault(struct mm_struct
161 if (!(vma->vm_flags & VM_GROWSDOWN))
163 - if (expand_stack(vma, ea))
164 + if (expand_stack(vma, ea, NULL))
167 is_write = dsisr & MFC_DSISR_ACCESS_PUT;
168 --- a/arch/s390/mm/fault.c
169 +++ b/arch/s390/mm/fault.c
170 @@ -350,7 +350,7 @@ do_exception(struct pt_regs *regs, unsig
172 if (!(vma->vm_flags & VM_GROWSDOWN))
174 - if (expand_stack(vma, address))
175 + if (expand_stack(vma, address, NULL /* FIXME? */))
178 * Ok, we have a good vm_area for this memory access, so
179 --- a/arch/sh/mm/fault_32.c
180 +++ b/arch/sh/mm/fault_32.c
181 @@ -108,7 +108,7 @@ asmlinkage void __kprobes do_page_fault(
183 if (!(vma->vm_flags & VM_GROWSDOWN))
185 - if (expand_stack(vma, address))
186 + if (expand_stack(vma, address, NULL))
189 * Ok, we have a good vm_area for this memory access, so
190 --- a/arch/sh/mm/tlbflush_64.c
191 +++ b/arch/sh/mm/tlbflush_64.c
192 @@ -153,7 +153,7 @@ asmlinkage void do_page_fault(struct pt_
196 - if (expand_stack(vma, address)) {
197 + if (expand_stack(vma, address, NULL)) {
200 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
201 --- a/arch/sparc/mm/fault.c
202 +++ b/arch/sparc/mm/fault.c
203 @@ -219,7 +219,7 @@ asmlinkage void do_sparc_fault(struct pt
205 if(!(vma->vm_flags & VM_GROWSDOWN))
207 - if(expand_stack(vma, address))
208 + if(expand_stack(vma, address, NULL))
211 * Ok, we have a good vm_area for this memory access, so
212 @@ -472,7 +472,7 @@ static void force_user_fault(unsigned lo
214 if(!(vma->vm_flags & VM_GROWSDOWN))
216 - if(expand_stack(vma, address))
217 + if(expand_stack(vma, address, NULL))
220 info.si_code = SEGV_ACCERR;
221 --- a/arch/sparc64/mm/fault.c
222 +++ b/arch/sparc64/mm/fault.c
223 @@ -367,7 +367,7 @@ continue_fault:
227 - if (expand_stack(vma, address))
228 + if (expand_stack(vma, address, NULL))
231 * Ok, we have a good vm_area for this memory access, so
232 --- a/arch/um/kernel/trap.c
233 +++ b/arch/um/kernel/trap.c
234 @@ -24,7 +24,7 @@ int handle_page_fault(unsigned long addr
235 int is_write, int is_user, int *code_out)
237 struct mm_struct *mm = current->mm;
238 - struct vm_area_struct *vma;
239 + struct vm_area_struct *vma, *prev_vma;
243 @@ -50,8 +50,11 @@ int handle_page_fault(unsigned long addr
245 else if (is_user && !ARCH_IS_STACKGROW(address))
247 - else if (expand_stack(vma, address))
250 + find_vma_prev(mm, address, &prev_vma);
251 + if(expand_stack(vma, address, prev_vma))
256 *code_out = SEGV_ACCERR;
257 --- a/arch/x86/kernel/sys_x86_64.c
258 +++ b/arch/x86/kernel/sys_x86_64.c
259 @@ -106,6 +106,7 @@ arch_get_unmapped_area(struct file *filp
262 for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
263 + unsigned long __heap_stack_gap;
264 /* At this point: (!vma || addr < vma->vm_end). */
265 if (end - len < addr) {
267 @@ -119,7 +120,14 @@ full_search:
271 - if (!vma || addr + len <= vma->vm_start) {
274 + __heap_stack_gap = 0;
275 + if (vma->vm_flags & VM_GROWSDOWN)
276 + __heap_stack_gap = min(end-(addr+len),
277 + (unsigned long) heap_stack_gap << PAGE_SHIFT);
278 + if (addr + len + __heap_stack_gap <= vma->vm_start) {
281 * Remember the place where we stopped the search:
283 --- a/arch/x86/mm/fault.c
284 +++ b/arch/x86/mm/fault.c
285 @@ -585,7 +585,7 @@ void __kprobes do_page_fault(struct pt_r
287 struct task_struct *tsk;
288 struct mm_struct *mm;
289 - struct vm_area_struct *vma;
290 + struct vm_area_struct *vma, *prev_vma;
291 unsigned long address;
294 @@ -719,7 +719,13 @@ again:
295 if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)
298 - if (expand_stack(vma, address))
300 + * find_vma_prev is just a bit slower, because it cannot
301 + * use the mmap_cache, so we run it only in the growsdown
302 + * slow path and we leave find_vma in the fast path.
304 + find_vma_prev(current->mm, address, &prev_vma);
305 + if (expand_stack(vma, address, prev_vma))
308 * Ok, we have a good vm_area for this memory access, so
309 --- a/arch/xtensa/mm/fault.c
310 +++ b/arch/xtensa/mm/fault.c
311 @@ -80,7 +80,7 @@ void do_page_fault(struct pt_regs *regs)
313 if (!(vma->vm_flags & VM_GROWSDOWN))
315 - if (expand_stack(vma, address))
316 + if (expand_stack(vma, address, NULL))
319 /* Ok, we have a good vm_area for this memory access, so
320 --- a/include/linux/mm.h
321 +++ b/include/linux/mm.h
322 @@ -1208,7 +1208,10 @@ void page_cache_async_readahead(struct a
323 unsigned long max_sane_readahead(unsigned long nr);
325 /* Do stack extension */
326 -extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
327 +#define EXPAND_STACK_HAS_3_ARGS
328 +extern int heap_stack_gap;
329 +extern int expand_stack(struct vm_area_struct * vma, unsigned long address,
330 + struct vm_area_struct * prev_vma);
332 extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
334 --- a/kernel/sysctl.c
335 +++ b/kernel/sysctl.c
336 @@ -1208,6 +1208,14 @@ static struct ctl_table vm_table[] = {
341 + .ctl_name = CTL_UNNUMBERED,
342 + .procname = "heap-stack-gap",
343 + .data = &heap_stack_gap,
344 + .maxlen = sizeof(int),
346 + .proc_handler = &proc_dointvec,
349 * NOTE: do not add new entries to this table unless you have read
350 * Documentation/sysctl/ctl_unnumbered.txt
353 @@ -85,6 +85,7 @@ int sysctl_overcommit_memory = OVERCOMMI
354 int sysctl_overcommit_ratio = 50; /* default is 50% */
355 int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
356 atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0);
357 +int heap_stack_gap = 1;
360 * Check that a process has enough memory to allocate a new virtual
361 @@ -1291,6 +1292,7 @@ arch_get_unmapped_area(struct file *filp
363 for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
364 /* At this point: (!vma || addr < vma->vm_end). */
365 + unsigned long __heap_stack_gap;
366 if (TASK_SIZE - len < addr) {
368 * Start a new search - just in case we missed
369 @@ -1304,7 +1306,14 @@ full_search:
373 - if (!vma || addr + len <= vma->vm_start) {
376 + __heap_stack_gap = 0;
377 + if (vma->vm_flags & VM_GROWSDOWN)
378 + __heap_stack_gap = min(TASK_SIZE-(addr+len),
379 + (unsigned long) heap_stack_gap << PAGE_SHIFT);
380 + if (addr + len + __heap_stack_gap <= vma->vm_start) {
383 * Remember the place where we stopped the search:
385 @@ -1633,11 +1642,9 @@ int expand_upwards(struct vm_area_struct
387 #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
390 - * vma is the first one with address < vma->vm_start. Have to extend vma.
392 static inline int expand_downwards(struct vm_area_struct *vma,
393 - unsigned long address)
394 + unsigned long address,
395 + struct vm_area_struct *prev_vma)
399 @@ -1665,6 +1672,13 @@ static inline int expand_downwards(struc
400 if (address < vma->vm_start) {
401 unsigned long size, grow;
405 + unsigned long __heap_stack_gap = min(TASK_SIZE-prev_vma->vm_end,
406 + (unsigned long) heap_stack_gap << PAGE_SHIFT);
407 + if (unlikely(prev_vma->vm_end + __heap_stack_gap > address))
410 size = vma->vm_end - address;
411 grow = (vma->vm_start - address) >> PAGE_SHIFT;
413 @@ -1674,6 +1688,7 @@ static inline int expand_downwards(struc
414 vma->vm_pgoff -= grow;
418 anon_vma_unlock(vma);
421 @@ -1684,8 +1699,16 @@ int expand_stack_downwards(struct vm_are
424 #ifdef CONFIG_STACK_GROWSUP
425 -int expand_stack(struct vm_area_struct *vma, unsigned long address)
426 +int expand_stack(struct vm_area_struct * vma, unsigned long address,
427 + struct vm_area_struct * prev_vma)
430 + * If you re-use the heap-stack-gap for a growsup stack you
431 + * should implement the feature for growsup too and remove
436 return expand_upwards(vma, address);
439 @@ -1698,7 +1721,7 @@ find_extend_vma(struct mm_struct *mm, un
440 vma = find_vma_prev(mm, addr, &prev);
441 if (vma && (vma->vm_start <= addr))
443 - if (!prev || expand_stack(prev, addr))
444 + if (!prev || expand_stack(prev, addr, NULL))
446 if (prev->vm_flags & VM_LOCKED)
447 make_pages_present(addr, prev->vm_end);
448 @@ -1713,7 +1736,7 @@ int expand_stack(struct vm_area_struct *
449 struct vm_area_struct *
450 find_extend_vma(struct mm_struct * mm, unsigned long addr)
452 - struct vm_area_struct * vma;
453 + struct vm_area_struct * vma, * prev_vma;
457 @@ -1725,7 +1748,8 @@ find_extend_vma(struct mm_struct * mm, u
458 if (!(vma->vm_flags & VM_GROWSDOWN))
460 start = vma->vm_start;
461 - if (expand_stack(vma, addr))
462 + find_vma_prev(mm, addr, &prev_vma);
463 + if (expand_stack(vma, addr, prev_vma))
465 if (vma->vm_flags & VM_LOCKED)
466 make_pages_present(addr, start);