]>
Commit | Line | Data |
---|---|---|
2cb7cef9 BS |
1 | From: Peter Zijlstra <a.p.zijlstra@chello.nl> |
2 | Subject: mm: allow PF_MEMALLOC from softirq context | |
3 | Patch-mainline: No | |
4 | References: FATE#303834 | |
5 | ||
6 | This is needed to allow network softirq packet processing to make use of | |
7 | PF_MEMALLOC. | |
8 | ||
9 | Currently softirq context cannot use PF_MEMALLOC due to it not being associated | |
10 | with a task, and therefore not having task flags to fiddle with - thus the gfp | |
11 | to alloc flag mapping ignores the task flags when in interrupts (hard or soft) | |
12 | context. | |
13 | ||
14 | Allowing softirqs to make use of PF_MEMALLOC therefore requires some trickery. | |
15 | We basically borrow the task flags from whatever process happens to be | |
16 | preempted by the softirq. | |
17 | ||
18 | So we modify the gfp to alloc flags mapping to not exclude task flags in | |
19 | softirq context, and modify the softirq code to save, clear and restore the | |
20 | PF_MEMALLOC flag. | |
21 | ||
22 | The save and clear, ensures the preempted task's PF_MEMALLOC flag doesn't | |
23 | leak into the softirq. The restore ensures a softirq's PF_MEMALLOC flag cannot | |
24 | leak back into the preempted process. | |
25 | ||
26 | Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> | |
27 | Acked-by: Neil Brown <neilb@suse.de> | |
28 | Acked-by: Suresh Jayaraman <sjayaraman@suse.de> | |
29 | ||
30 | --- | |
31 | include/linux/sched.h | 7 +++++++ | |
32 | kernel/softirq.c | 3 +++ | |
33 | mm/page_alloc.c | 7 ++++--- | |
34 | 3 files changed, 14 insertions(+), 3 deletions(-) | |
35 | ||
36 | --- a/include/linux/sched.h | |
37 | +++ b/include/linux/sched.h | |
38 | @@ -1555,6 +1555,13 @@ extern cputime_t task_gtime(struct task_ | |
39 | #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) | |
40 | #define used_math() tsk_used_math(current) | |
41 | ||
42 | +static inline void tsk_restore_flags(struct task_struct *p, | |
43 | + unsigned long pflags, unsigned long mask) | |
44 | +{ | |
45 | + p->flags &= ~mask; | |
46 | + p->flags |= pflags & mask; | |
47 | +} | |
48 | + | |
49 | #ifdef CONFIG_SMP | |
50 | extern int set_cpus_allowed_ptr(struct task_struct *p, | |
51 | const cpumask_t *new_mask); | |
52 | --- a/kernel/softirq.c | |
53 | +++ b/kernel/softirq.c | |
54 | @@ -188,6 +188,8 @@ asmlinkage void __do_softirq(void) | |
55 | __u32 pending; | |
56 | int max_restart = MAX_SOFTIRQ_RESTART; | |
57 | int cpu; | |
58 | + unsigned long pflags = current->flags; | |
59 | + current->flags &= ~PF_MEMALLOC; | |
60 | ||
61 | pending = local_softirq_pending(); | |
62 | account_system_vtime(current); | |
63 | @@ -228,6 +230,7 @@ restart: | |
64 | ||
65 | account_system_vtime(current); | |
66 | _local_bh_enable(); | |
67 | + tsk_restore_flags(current, pflags, PF_MEMALLOC); | |
68 | } | |
69 | ||
70 | #ifndef __ARCH_HAS_DO_SOFTIRQ | |
71 | --- a/mm/page_alloc.c | |
72 | +++ b/mm/page_alloc.c | |
73 | @@ -1483,9 +1483,10 @@ int gfp_to_alloc_flags(gfp_t gfp_mask) | |
74 | alloc_flags |= ALLOC_HARDER; | |
75 | ||
76 | if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) { | |
77 | - if (!in_interrupt() && | |
78 | - ((p->flags & PF_MEMALLOC) || | |
79 | - unlikely(test_thread_flag(TIF_MEMDIE)))) | |
80 | + if (!in_irq() && (p->flags & PF_MEMALLOC)) | |
81 | + alloc_flags |= ALLOC_NO_WATERMARKS; | |
82 | + else if (!in_interrupt() && | |
83 | + unlikely(test_thread_flag(TIF_MEMDIE))) | |
84 | alloc_flags |= ALLOC_NO_WATERMARKS; | |
85 | } | |
86 |