]> git.ipfire.org Git - thirdparty/kernel/stable-queue.git/blob
projects / thirdparty / kernel / stable-queue.git / blob
? search:


3ffa0ff7699fe3ec354c3705828380e725b41b42
[thirdparty/kernel/stable-queue.git] /
1 From c4279968c7fe0f89a96adbe9576cb77ac8d048a3 Mon Sep 17 00:00:00 2001
2 From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
3 Date: Mon, 30 Dec 2024 15:47:08 +0100
4 Subject: Revert "rcu-tasks: Fix access non-existent percpu rtpcp variable in rcu_tasks_need_gpcb()"
5
6 From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
7
8 This reverts commit 224fd631c41b81697aa622d38615bfbf446b91cf which is
9 commit fd70e9f1d85f5323096ad313ba73f5fe3d15ea41 upstream.
10
11 It is reported to cause problems in testing, so revert it for now.
12
13 Link: https://lore.kernel.org/r/20241216-comic-handling-3bcf108cc465@wendy
14 Reported-by: Conor Dooley <conor.dooley@microchip.com>
15 CC: Zhixu Liu <zhixu.liu@gmail.com>
16 Cc: Zqiang <qiang.zhang1211@gmail.com>
17 Cc: Neeraj Upadhyay <neeraj.upadhyay@kernel.org>
18 Cc: Sasha Levin <sashal@kernel.org>
19 Cc: Xiangyu Chen <xiangyu.chen@windriver.com>
20 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
21 ---
22 kernel/rcu/tasks.h | 82 ++++++++++++++++++-----------------------------------
23 1 file changed, 28 insertions(+), 54 deletions(-)
24
25 --- a/kernel/rcu/tasks.h
26 +++ b/kernel/rcu/tasks.h
27 @@ -31,7 +31,6 @@ typedef void (*postgp_func_t)(struct rcu
28 * @barrier_q_head: RCU callback for barrier operation.
29 * @rtp_blkd_tasks: List of tasks blocked as readers.
30 * @cpu: CPU number corresponding to this entry.
31 - * @index: Index of this CPU in rtpcp_array of the rcu_tasks structure.
32 * @rtpp: Pointer to the rcu_tasks structure.
33 */
34 struct rcu_tasks_percpu {
35 @@ -44,7 +43,6 @@ struct rcu_tasks_percpu {
36 struct rcu_head barrier_q_head;
37 struct list_head rtp_blkd_tasks;
38 int cpu;
39 - int index;
40 struct rcu_tasks *rtpp;
41 };
42
43 @@ -70,7 +68,6 @@ struct rcu_tasks_percpu {
44 * @postgp_func: This flavor's post-grace-period function (optional).
45 * @call_func: This flavor's call_rcu()-equivalent function.
46 * @rtpcpu: This flavor's rcu_tasks_percpu structure.
47 - * @rtpcp_array: Array of pointers to rcu_tasks_percpu structure of CPUs in cpu_possible_mask.
48 * @percpu_enqueue_shift: Shift down CPU ID this much when enqueuing callbacks.
49 * @percpu_enqueue_lim: Number of per-CPU callback queues in use for enqueuing.
50 * @percpu_dequeue_lim: Number of per-CPU callback queues in use for dequeuing.
51 @@ -103,7 +100,6 @@ struct rcu_tasks {
52 postgp_func_t postgp_func;
53 call_rcu_func_t call_func;
54 struct rcu_tasks_percpu __percpu *rtpcpu;
55 - struct rcu_tasks_percpu **rtpcp_array;
56 int percpu_enqueue_shift;
57 int percpu_enqueue_lim;
58 int percpu_dequeue_lim;
59 @@ -168,8 +164,6 @@ module_param(rcu_task_contend_lim, int,
60 static int rcu_task_collapse_lim __read_mostly = 10;
61 module_param(rcu_task_collapse_lim, int, 0444);
62
63 -static int rcu_task_cpu_ids;
64 -
65 /* RCU tasks grace-period state for debugging. */
66 #define RTGS_INIT 0
67 #define RTGS_WAIT_WAIT_CBS 1
68 @@ -234,8 +228,6 @@ static void cblist_init_generic(struct r
69 unsigned long flags;
70 int lim;
71 int shift;
72 - int maxcpu;
73 - int index = 0;
74
75 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
76 if (rcu_task_enqueue_lim < 0) {
77 @@ -246,9 +238,14 @@ static void cblist_init_generic(struct r
78 }
79 lim = rcu_task_enqueue_lim;
80
81 - rtp->rtpcp_array = kcalloc(num_possible_cpus(), sizeof(struct rcu_tasks_percpu *), GFP_KERNEL);
82 - BUG_ON(!rtp->rtpcp_array);
83 -
84 + if (lim > nr_cpu_ids)
85 + lim = nr_cpu_ids;
86 + shift = ilog2(nr_cpu_ids / lim);
87 + if (((nr_cpu_ids - 1) >> shift) >= lim)
88 + shift++;
89 + WRITE_ONCE(rtp->percpu_enqueue_shift, shift);
90 + WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
91 + smp_store_release(&rtp->percpu_enqueue_lim, lim);
92 for_each_possible_cpu(cpu) {
93 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
94
95 @@ -261,33 +258,16 @@ static void cblist_init_generic(struct r
96 INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq);
97 rtpcp->cpu = cpu;
98 rtpcp->rtpp = rtp;
99 - rtpcp->index = index;
100 - rtp->rtpcp_array[index] = rtpcp;
101 - index++;
102 if (!rtpcp->rtp_blkd_tasks.next)
103 INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);
104 raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled.
105 - maxcpu = cpu;
106 }
107 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
108
109 if (rcu_task_cb_adjust)
110 pr_info("%s: Setting adjustable number of callback queues.\n", __func__);
111
112 - rcu_task_cpu_ids = maxcpu + 1;
113 - if (lim > rcu_task_cpu_ids)
114 - lim = rcu_task_cpu_ids;
115 - shift = ilog2(rcu_task_cpu_ids / lim);
116 - if (((rcu_task_cpu_ids - 1) >> shift) >= lim)
117 - shift++;
118 - WRITE_ONCE(rtp->percpu_enqueue_shift, shift);
119 - WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
120 - smp_store_release(&rtp->percpu_enqueue_lim, lim);
121 -
122 - pr_info("%s: Setting shift to %d and lim to %d rcu_task_cb_adjust=%d rcu_task_cpu_ids=%d.\n",
123 - rtp->name, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim),
124 - rcu_task_cb_adjust, rcu_task_cpu_ids);
125 -
126 + pr_info("%s: Setting shift to %d and lim to %d.\n", __func__, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim));
127 }
128
129 // IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic().
130 @@ -327,7 +307,7 @@ static void call_rcu_tasks_generic(struc
131 rtpcp->rtp_n_lock_retries = 0;
132 }
133 if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim &&
134 - READ_ONCE(rtp->percpu_enqueue_lim) != rcu_task_cpu_ids)
135 + READ_ONCE(rtp->percpu_enqueue_lim) != nr_cpu_ids)
136 needadjust = true; // Defer adjustment to avoid deadlock.
137 }
138 if (!rcu_segcblist_is_enabled(&rtpcp->cblist)) {
139 @@ -340,10 +320,10 @@ static void call_rcu_tasks_generic(struc
140 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
141 if (unlikely(needadjust)) {
142 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
143 - if (rtp->percpu_enqueue_lim != rcu_task_cpu_ids) {
144 + if (rtp->percpu_enqueue_lim != nr_cpu_ids) {
145 WRITE_ONCE(rtp->percpu_enqueue_shift, 0);
146 - WRITE_ONCE(rtp->percpu_dequeue_lim, rcu_task_cpu_ids);
147 - smp_store_release(&rtp->percpu_enqueue_lim, rcu_task_cpu_ids);
148 + WRITE_ONCE(rtp->percpu_dequeue_lim, nr_cpu_ids);
149 + smp_store_release(&rtp->percpu_enqueue_lim, nr_cpu_ids);
150 pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
151 }
152 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
153 @@ -414,8 +394,6 @@ static int rcu_tasks_need_gpcb(struct rc
154 int needgpcb = 0;
155
156 for (cpu = 0; cpu < smp_load_acquire(&rtp->percpu_dequeue_lim); cpu++) {
157 - if (!cpu_possible(cpu))
158 - continue;
159 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
160
161 /* Advance and accelerate any new callbacks. */
162 @@ -448,7 +426,7 @@ static int rcu_tasks_need_gpcb(struct rc
163 if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {
164 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
165 if (rtp->percpu_enqueue_lim > 1) {
166 - WRITE_ONCE(rtp->percpu_enqueue_shift, order_base_2(rcu_task_cpu_ids));
167 + WRITE_ONCE(rtp->percpu_enqueue_shift, order_base_2(nr_cpu_ids));
168 smp_store_release(&rtp->percpu_enqueue_lim, 1);
169 rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();
170 gpdone = false;
171 @@ -463,9 +441,7 @@ static int rcu_tasks_need_gpcb(struct rc
172 pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name);
173 }
174 if (rtp->percpu_dequeue_lim == 1) {
175 - for (cpu = rtp->percpu_dequeue_lim; cpu < rcu_task_cpu_ids; cpu++) {
176 - if (!cpu_possible(cpu))
177 - continue;
178 + for (cpu = rtp->percpu_dequeue_lim; cpu < nr_cpu_ids; cpu++) {
179 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
180
181 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist));
182 @@ -480,32 +456,30 @@ static int rcu_tasks_need_gpcb(struct rc
183 // Advance callbacks and invoke any that are ready.
184 static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp)
185 {
186 + int cpu;
187 + int cpunext;
188 int cpuwq;
189 unsigned long flags;
190 int len;
191 - int index;
192 struct rcu_head *rhp;
193 struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
194 struct rcu_tasks_percpu *rtpcp_next;
195
196 - index = rtpcp->index * 2 + 1;
197 - if (index < num_possible_cpus()) {
198 - rtpcp_next = rtp->rtpcp_array[index];
199 - if (rtpcp_next->cpu < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
200 - cpuwq = rcu_cpu_beenfullyonline(rtpcp_next->cpu) ? rtpcp_next->cpu : WORK_CPU_UNBOUND;
201 + cpu = rtpcp->cpu;
202 + cpunext = cpu * 2 + 1;
203 + if (cpunext < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
204 + rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext);
205 + cpuwq = rcu_cpu_beenfullyonline(cpunext) ? cpunext : WORK_CPU_UNBOUND;
206 + queue_work_on(cpuwq, system_wq, &rtpcp_next->rtp_work);
207 + cpunext++;
208 + if (cpunext < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
209 + rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext);
210 + cpuwq = rcu_cpu_beenfullyonline(cpunext) ? cpunext : WORK_CPU_UNBOUND;
211 queue_work_on(cpuwq, system_wq, &rtpcp_next->rtp_work);
212 - index++;
213 - if (index < num_possible_cpus()) {
214 - rtpcp_next = rtp->rtpcp_array[index];
215 - if (rtpcp_next->cpu < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
216 - cpuwq = rcu_cpu_beenfullyonline(rtpcp_next->cpu) ? rtpcp_next->cpu : WORK_CPU_UNBOUND;
217 - queue_work_on(cpuwq, system_wq, &rtpcp_next->rtp_work);
218 - }
219 - }
220 }
221 }
222
223 - if (rcu_segcblist_empty(&rtpcp->cblist))
224 + if (rcu_segcblist_empty(&rtpcp->cblist) || !cpu_possible(cpu))
225 return;
226 raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
227 rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
Mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/stable-queue.git