]>
Commit | Line | Data |
---|---|---|
721818c9 GKH |
1 | From 1ee14e6c8cddeeb8a490d7b54cd9016e4bb900b4 Mon Sep 17 00:00:00 2001 |
2 | From: Ben Segall <bsegall@google.com> | |
3 | Date: Wed, 16 Oct 2013 11:16:12 -0700 | |
4 | Subject: sched: Fix race on toggling cfs_bandwidth_used | |
5 | ||
6 | From: Ben Segall <bsegall@google.com> | |
7 | ||
8 | commit 1ee14e6c8cddeeb8a490d7b54cd9016e4bb900b4 upstream. | |
9 | ||
10 | When we transition cfs_bandwidth_used to false, any currently | |
11 | throttled groups will incorrectly return false from cfs_rq_throttled. | |
12 | While tg_set_cfs_bandwidth will unthrottle them eventually, currently | |
13 | running code (including at least dequeue_task_fair and | |
14 | distribute_cfs_runtime) will cause errors. | |
15 | ||
16 | Fix this by turning off cfs_bandwidth_used only after unthrottling all | |
17 | cfs_rqs. | |
18 | ||
19 | Tested: toggle bandwidth back and forth on a loaded cgroup. Caused | |
20 | crashes in minutes without the patch, hasn't crashed with it. | |
21 | ||
22 | Signed-off-by: Ben Segall <bsegall@google.com> | |
23 | Signed-off-by: Peter Zijlstra <peterz@infradead.org> | |
24 | Cc: pjt@google.com | |
25 | Link: http://lkml.kernel.org/r/20131016181611.22647.80365.stgit@sword-of-the-dawn.mtv.corp.google.com | |
26 | Signed-off-by: Ingo Molnar <mingo@kernel.org> | |
27 | Cc: Chris J Arges <chris.j.arges@canonical.com> | |
28 | Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> | |
29 | ||
30 | --- | |
31 | kernel/sched/core.c | 9 ++++++++- | |
32 | kernel/sched/fair.c | 16 +++++++++------- | |
33 | kernel/sched/sched.h | 3 ++- | |
34 | 3 files changed, 19 insertions(+), 9 deletions(-) | |
35 | ||
36 | --- a/kernel/sched/core.c | |
37 | +++ b/kernel/sched/core.c | |
38 | @@ -7906,7 +7906,12 @@ static int tg_set_cfs_bandwidth(struct t | |
39 | ||
40 | runtime_enabled = quota != RUNTIME_INF; | |
41 | runtime_was_enabled = cfs_b->quota != RUNTIME_INF; | |
42 | - account_cfs_bandwidth_used(runtime_enabled, runtime_was_enabled); | |
43 | + /* | |
44 | + * If we need to toggle cfs_bandwidth_used, off->on must occur | |
45 | + * before making related changes, and on->off must occur afterwards | |
46 | + */ | |
47 | + if (runtime_enabled && !runtime_was_enabled) | |
48 | + cfs_bandwidth_usage_inc(); | |
49 | raw_spin_lock_irq(&cfs_b->lock); | |
50 | cfs_b->period = ns_to_ktime(period); | |
51 | cfs_b->quota = quota; | |
52 | @@ -7932,6 +7937,8 @@ static int tg_set_cfs_bandwidth(struct t | |
53 | unthrottle_cfs_rq(cfs_rq); | |
54 | raw_spin_unlock_irq(&rq->lock); | |
55 | } | |
56 | + if (runtime_was_enabled && !runtime_enabled) | |
57 | + cfs_bandwidth_usage_dec(); | |
58 | out_unlock: | |
59 | mutex_unlock(&cfs_constraints_mutex); | |
60 | ||
61 | --- a/kernel/sched/fair.c | |
62 | +++ b/kernel/sched/fair.c | |
63 | @@ -1393,13 +1393,14 @@ static inline bool cfs_bandwidth_used(vo | |
64 | return static_key_false(&__cfs_bandwidth_used); | |
65 | } | |
66 | ||
67 | -void account_cfs_bandwidth_used(int enabled, int was_enabled) | |
68 | +void cfs_bandwidth_usage_inc(void) | |
69 | { | |
70 | - /* only need to count groups transitioning between enabled/!enabled */ | |
71 | - if (enabled && !was_enabled) | |
72 | - static_key_slow_inc(&__cfs_bandwidth_used); | |
73 | - else if (!enabled && was_enabled) | |
74 | - static_key_slow_dec(&__cfs_bandwidth_used); | |
75 | + static_key_slow_inc(&__cfs_bandwidth_used); | |
76 | +} | |
77 | + | |
78 | +void cfs_bandwidth_usage_dec(void) | |
79 | +{ | |
80 | + static_key_slow_dec(&__cfs_bandwidth_used); | |
81 | } | |
82 | #else /* HAVE_JUMP_LABEL */ | |
83 | static bool cfs_bandwidth_used(void) | |
84 | @@ -1407,7 +1408,8 @@ static bool cfs_bandwidth_used(void) | |
85 | return true; | |
86 | } | |
87 | ||
88 | -void account_cfs_bandwidth_used(int enabled, int was_enabled) {} | |
89 | +void cfs_bandwidth_usage_inc(void) {} | |
90 | +void cfs_bandwidth_usage_dec(void) {} | |
91 | #endif /* HAVE_JUMP_LABEL */ | |
92 | ||
93 | /* | |
94 | --- a/kernel/sched/sched.h | |
95 | +++ b/kernel/sched/sched.h | |
96 | @@ -1140,7 +1140,8 @@ extern void init_cfs_rq(struct cfs_rq *c | |
97 | extern void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq); | |
98 | extern void unthrottle_offline_cfs_rqs(struct rq *rq); | |
99 | ||
100 | -extern void account_cfs_bandwidth_used(int enabled, int was_enabled); | |
101 | +extern void cfs_bandwidth_usage_inc(void); | |
102 | +extern void cfs_bandwidth_usage_dec(void); | |
103 | ||
104 | #ifdef CONFIG_NO_HZ | |
105 | enum rq_nohz_flag_bits { |