]>
Commit | Line | Data |
---|---|---|
3dcf60bc | 1 | // SPDX-License-Identifier: GPL-2.0 |
75bb4625 | 2 | /* |
88459642 OS |
3 | * Tag allocation using scalable bitmaps. Uses active queue tracking to support |
4 | * fairer distribution of tags between multiple submitters when a shared tag map | |
5 | * is used. | |
75bb4625 JA |
6 | * |
7 | * Copyright (C) 2013-2014 Jens Axboe | |
8 | */ | |
320ae51f JA |
9 | #include <linux/kernel.h> |
10 | #include <linux/module.h> | |
320ae51f JA |
11 | |
12 | #include <linux/blk-mq.h> | |
f9934a80 | 13 | #include <linux/delay.h> |
320ae51f JA |
14 | #include "blk.h" |
15 | #include "blk-mq.h" | |
d97e594c | 16 | #include "blk-mq-sched.h" |
320ae51f JA |
17 | #include "blk-mq-tag.h" |
18 | ||
180dccb0 LQ |
19 | /* |
20 | * Recalculate wakeup batch when tag is shared by hctx. | |
21 | */ | |
22 | static void blk_mq_update_wake_batch(struct blk_mq_tags *tags, | |
23 | unsigned int users) | |
24 | { | |
25 | if (!users) | |
26 | return; | |
27 | ||
28 | sbitmap_queue_recalculate_wake_batch(&tags->bitmap_tags, | |
29 | users); | |
30 | sbitmap_queue_recalculate_wake_batch(&tags->breserved_tags, | |
31 | users); | |
32 | } | |
33 | ||
0d2602ca JA |
34 | /* |
35 | * If a previously inactive queue goes active, bump the active user count. | |
d263ed99 JW |
36 | * We need to do this before try to allocate driver tag, then even if fail |
37 | * to get tag when first time, the other shared-tag users could reserve | |
38 | * budget for it. | |
0d2602ca | 39 | */ |
ee78ec10 | 40 | void __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) |
0d2602ca | 41 | { |
180dccb0 LQ |
42 | unsigned int users; |
43 | ||
079a2e3e | 44 | if (blk_mq_is_shared_tags(hctx->flags)) { |
f1b49fdc | 45 | struct request_queue *q = hctx->queue; |
f1b49fdc | 46 | |
ee78ec10 LS |
47 | if (test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags)) |
48 | return; | |
49 | set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags); | |
f1b49fdc | 50 | } else { |
ee78ec10 LS |
51 | if (test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) |
52 | return; | |
53 | set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state); | |
f1b49fdc | 54 | } |
0d2602ca | 55 | |
180dccb0 LQ |
56 | users = atomic_inc_return(&hctx->tags->active_queues); |
57 | ||
58 | blk_mq_update_wake_batch(hctx->tags, users); | |
0d2602ca JA |
59 | } |
60 | ||
61 | /* | |
aed3ea94 | 62 | * Wakeup all potentially sleeping on tags |
0d2602ca | 63 | */ |
aed3ea94 | 64 | void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve) |
0d2602ca | 65 | { |
ae0f1a73 | 66 | sbitmap_queue_wake_all(&tags->bitmap_tags); |
88459642 | 67 | if (include_reserve) |
ae0f1a73 | 68 | sbitmap_queue_wake_all(&tags->breserved_tags); |
0d2602ca JA |
69 | } |
70 | ||
e3a2b3f9 JA |
71 | /* |
72 | * If a previously busy queue goes inactive, potential waiters could now | |
73 | * be allowed to queue. Wake them up and check. | |
74 | */ | |
75 | void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) | |
76 | { | |
77 | struct blk_mq_tags *tags = hctx->tags; | |
180dccb0 | 78 | unsigned int users; |
e3a2b3f9 | 79 | |
079a2e3e | 80 | if (blk_mq_is_shared_tags(hctx->flags)) { |
e155b0c2 JG |
81 | struct request_queue *q = hctx->queue; |
82 | ||
f1b49fdc JG |
83 | if (!test_and_clear_bit(QUEUE_FLAG_HCTX_ACTIVE, |
84 | &q->queue_flags)) | |
85 | return; | |
f1b49fdc JG |
86 | } else { |
87 | if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) | |
88 | return; | |
f1b49fdc | 89 | } |
e3a2b3f9 | 90 | |
180dccb0 LQ |
91 | users = atomic_dec_return(&tags->active_queues); |
92 | ||
93 | blk_mq_update_wake_batch(tags, users); | |
079a2e3e | 94 | |
aed3ea94 | 95 | blk_mq_tag_wakeup_all(tags, false); |
e3a2b3f9 JA |
96 | } |
97 | ||
200e86b3 JA |
98 | static int __blk_mq_get_tag(struct blk_mq_alloc_data *data, |
99 | struct sbitmap_queue *bt) | |
4bb659b1 | 100 | { |
28500850 ML |
101 | if (!data->q->elevator && !(data->flags & BLK_MQ_REQ_RESERVED) && |
102 | !hctx_may_queue(data->hctx, bt)) | |
76647368 | 103 | return BLK_MQ_NO_TAG; |
42fdc5e4 | 104 | |
229a9287 | 105 | if (data->shallow_depth) |
3f607293 | 106 | return sbitmap_queue_get_shallow(bt, data->shallow_depth); |
229a9287 OS |
107 | else |
108 | return __sbitmap_queue_get(bt); | |
4bb659b1 JA |
109 | } |
110 | ||
349302da JA |
111 | unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags, |
112 | unsigned int *offset) | |
113 | { | |
114 | struct blk_mq_tags *tags = blk_mq_tags_from_data(data); | |
115 | struct sbitmap_queue *bt = &tags->bitmap_tags; | |
116 | unsigned long ret; | |
117 | ||
118 | if (data->shallow_depth ||data->flags & BLK_MQ_REQ_RESERVED || | |
119 | data->hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) | |
120 | return 0; | |
121 | ret = __sbitmap_queue_get_batch(bt, nr_tags, offset); | |
122 | *offset += tags->nr_reserved_tags; | |
123 | return ret; | |
124 | } | |
125 | ||
4941115b | 126 | unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) |
320ae51f | 127 | { |
4941115b JA |
128 | struct blk_mq_tags *tags = blk_mq_tags_from_data(data); |
129 | struct sbitmap_queue *bt; | |
88459642 | 130 | struct sbq_wait_state *ws; |
5d2ee712 | 131 | DEFINE_SBQ_WAIT(wait); |
4941115b | 132 | unsigned int tag_offset; |
320ae51f JA |
133 | int tag; |
134 | ||
4941115b JA |
135 | if (data->flags & BLK_MQ_REQ_RESERVED) { |
136 | if (unlikely(!tags->nr_reserved_tags)) { | |
137 | WARN_ON_ONCE(1); | |
419c3d5e | 138 | return BLK_MQ_NO_TAG; |
4941115b | 139 | } |
ae0f1a73 | 140 | bt = &tags->breserved_tags; |
4941115b JA |
141 | tag_offset = 0; |
142 | } else { | |
ae0f1a73 | 143 | bt = &tags->bitmap_tags; |
4941115b JA |
144 | tag_offset = tags->nr_reserved_tags; |
145 | } | |
146 | ||
200e86b3 | 147 | tag = __blk_mq_get_tag(data, bt); |
76647368 | 148 | if (tag != BLK_MQ_NO_TAG) |
4941115b | 149 | goto found_tag; |
4bb659b1 | 150 | |
6f3b0e8b | 151 | if (data->flags & BLK_MQ_REQ_NOWAIT) |
419c3d5e | 152 | return BLK_MQ_NO_TAG; |
4bb659b1 | 153 | |
4941115b | 154 | ws = bt_wait_ptr(bt, data->hctx); |
4bb659b1 | 155 | do { |
e6fc4649 ML |
156 | struct sbitmap_queue *bt_prev; |
157 | ||
b3223207 BVA |
158 | /* |
159 | * We're out of tags on this hardware queue, kick any | |
160 | * pending IO submits before going to sleep waiting for | |
8cecb07d | 161 | * some to complete. |
b3223207 | 162 | */ |
8cecb07d | 163 | blk_mq_run_hw_queue(data->hctx, false); |
b3223207 | 164 | |
080ff351 JA |
165 | /* |
166 | * Retry tag allocation after running the hardware queue, | |
167 | * as running the queue may also have found completions. | |
168 | */ | |
200e86b3 | 169 | tag = __blk_mq_get_tag(data, bt); |
76647368 | 170 | if (tag != BLK_MQ_NO_TAG) |
080ff351 JA |
171 | break; |
172 | ||
5d2ee712 | 173 | sbitmap_prepare_to_wait(bt, ws, &wait, TASK_UNINTERRUPTIBLE); |
4e5dff41 JA |
174 | |
175 | tag = __blk_mq_get_tag(data, bt); | |
76647368 | 176 | if (tag != BLK_MQ_NO_TAG) |
4e5dff41 JA |
177 | break; |
178 | ||
e6fc4649 | 179 | bt_prev = bt; |
4bb659b1 | 180 | io_schedule(); |
cb96a42c | 181 | |
5d2ee712 JA |
182 | sbitmap_finish_wait(bt, ws, &wait); |
183 | ||
cb96a42c | 184 | data->ctx = blk_mq_get_ctx(data->q); |
f9afca4d | 185 | data->hctx = blk_mq_map_queue(data->q, data->cmd_flags, |
8ccdf4a3 | 186 | data->ctx); |
4941115b JA |
187 | tags = blk_mq_tags_from_data(data); |
188 | if (data->flags & BLK_MQ_REQ_RESERVED) | |
ae0f1a73 | 189 | bt = &tags->breserved_tags; |
4941115b | 190 | else |
ae0f1a73 | 191 | bt = &tags->bitmap_tags; |
4941115b | 192 | |
e6fc4649 ML |
193 | /* |
194 | * If destination hw queue is changed, fake wake up on | |
195 | * previous queue for compensating the wake up miss, so | |
196 | * other allocations on previous queue won't be starved. | |
197 | */ | |
198 | if (bt != bt_prev) | |
199 | sbitmap_queue_wake_up(bt_prev); | |
200 | ||
4941115b | 201 | ws = bt_wait_ptr(bt, data->hctx); |
4bb659b1 JA |
202 | } while (1); |
203 | ||
5d2ee712 | 204 | sbitmap_finish_wait(bt, ws, &wait); |
320ae51f | 205 | |
4941115b | 206 | found_tag: |
bf0beec0 ML |
207 | /* |
208 | * Give up this allocation if the hctx is inactive. The caller will | |
209 | * retry on an active hctx. | |
210 | */ | |
211 | if (unlikely(test_bit(BLK_MQ_S_INACTIVE, &data->hctx->state))) { | |
212 | blk_mq_put_tag(tags, data->ctx, tag + tag_offset); | |
213 | return BLK_MQ_NO_TAG; | |
214 | } | |
4941115b | 215 | return tag + tag_offset; |
320ae51f JA |
216 | } |
217 | ||
cae740a0 JG |
218 | void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, |
219 | unsigned int tag) | |
320ae51f | 220 | { |
415b806d | 221 | if (!blk_mq_tag_is_reserved(tags, tag)) { |
4bb659b1 JA |
222 | const int real_tag = tag - tags->nr_reserved_tags; |
223 | ||
70114c39 | 224 | BUG_ON(real_tag >= tags->nr_tags); |
ae0f1a73 | 225 | sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu); |
70114c39 | 226 | } else { |
ae0f1a73 | 227 | sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu); |
70114c39 | 228 | } |
320ae51f JA |
229 | } |
230 | ||
f794f335 JA |
231 | void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags) |
232 | { | |
233 | sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags, | |
234 | tag_array, nr_tags); | |
235 | } | |
236 | ||
88459642 OS |
237 | struct bt_iter_data { |
238 | struct blk_mq_hw_ctx *hctx; | |
fea9f92f | 239 | struct request_queue *q; |
fc39f8d2 | 240 | busy_tag_iter_fn *fn; |
88459642 OS |
241 | void *data; |
242 | bool reserved; | |
243 | }; | |
244 | ||
2e315dc0 ML |
245 | static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags, |
246 | unsigned int bitnr) | |
247 | { | |
bd63141d ML |
248 | struct request *rq; |
249 | unsigned long flags; | |
2e315dc0 | 250 | |
bd63141d ML |
251 | spin_lock_irqsave(&tags->lock, flags); |
252 | rq = tags->rqs[bitnr]; | |
0a467d0f | 253 | if (!rq || rq->tag != bitnr || !req_ref_inc_not_zero(rq)) |
bd63141d ML |
254 | rq = NULL; |
255 | spin_unlock_irqrestore(&tags->lock, flags); | |
2e315dc0 ML |
256 | return rq; |
257 | } | |
258 | ||
88459642 | 259 | static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) |
320ae51f | 260 | { |
88459642 OS |
261 | struct bt_iter_data *iter_data = data; |
262 | struct blk_mq_hw_ctx *hctx = iter_data->hctx; | |
fea9f92f JG |
263 | struct request_queue *q = iter_data->q; |
264 | struct blk_mq_tag_set *set = q->tag_set; | |
88459642 | 265 | bool reserved = iter_data->reserved; |
fea9f92f | 266 | struct blk_mq_tags *tags; |
81481eb4 | 267 | struct request *rq; |
2e315dc0 | 268 | bool ret = true; |
4bb659b1 | 269 | |
fea9f92f JG |
270 | if (blk_mq_is_shared_tags(set->flags)) |
271 | tags = set->shared_tags; | |
272 | else | |
273 | tags = hctx->tags; | |
274 | ||
88459642 OS |
275 | if (!reserved) |
276 | bitnr += tags->nr_reserved_tags; | |
7f5562d5 JA |
277 | /* |
278 | * We can hit rq == NULL here, because the tagging functions | |
c7b1bf5c | 279 | * test and set the bit before assigning ->rqs[]. |
7f5562d5 | 280 | */ |
2e315dc0 ML |
281 | rq = blk_mq_find_and_get_req(tags, bitnr); |
282 | if (!rq) | |
283 | return true; | |
284 | ||
fea9f92f | 285 | if (rq->q == q && (!hctx || rq->mq_hctx == hctx)) |
2dd6532e | 286 | ret = iter_data->fn(rq, iter_data->data); |
2e315dc0 ML |
287 | blk_mq_put_rq_ref(rq); |
288 | return ret; | |
88459642 | 289 | } |
4bb659b1 | 290 | |
c7b1bf5c BVA |
291 | /** |
292 | * bt_for_each - iterate over the requests associated with a hardware queue | |
293 | * @hctx: Hardware queue to examine. | |
fea9f92f | 294 | * @q: Request queue to examine. |
c7b1bf5c BVA |
295 | * @bt: sbitmap to examine. This is either the breserved_tags member |
296 | * or the bitmap_tags member of struct blk_mq_tags. | |
297 | * @fn: Pointer to the function that will be called for each request | |
298 | * associated with @hctx that has been assigned a driver tag. | |
299 | * @fn will be called as follows: @fn(@hctx, rq, @data, @reserved) | |
ab11fe5a JA |
300 | * where rq is a pointer to a request. Return true to continue |
301 | * iterating tags, false to stop. | |
c7b1bf5c BVA |
302 | * @data: Will be passed as third argument to @fn. |
303 | * @reserved: Indicates whether @bt is the breserved_tags member or the | |
304 | * bitmap_tags member of struct blk_mq_tags. | |
305 | */ | |
fea9f92f JG |
306 | static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct request_queue *q, |
307 | struct sbitmap_queue *bt, busy_tag_iter_fn *fn, | |
308 | void *data, bool reserved) | |
88459642 OS |
309 | { |
310 | struct bt_iter_data iter_data = { | |
311 | .hctx = hctx, | |
312 | .fn = fn, | |
313 | .data = data, | |
314 | .reserved = reserved, | |
fea9f92f | 315 | .q = q, |
88459642 OS |
316 | }; |
317 | ||
318 | sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data); | |
320ae51f JA |
319 | } |
320 | ||
88459642 OS |
321 | struct bt_tags_iter_data { |
322 | struct blk_mq_tags *tags; | |
323 | busy_tag_iter_fn *fn; | |
324 | void *data; | |
602380d2 | 325 | unsigned int flags; |
88459642 OS |
326 | }; |
327 | ||
602380d2 ML |
328 | #define BT_TAG_ITER_RESERVED (1 << 0) |
329 | #define BT_TAG_ITER_STARTED (1 << 1) | |
22f614bc | 330 | #define BT_TAG_ITER_STATIC_RQS (1 << 2) |
602380d2 | 331 | |
88459642 | 332 | static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) |
f26cdc85 | 333 | { |
88459642 OS |
334 | struct bt_tags_iter_data *iter_data = data; |
335 | struct blk_mq_tags *tags = iter_data->tags; | |
602380d2 | 336 | bool reserved = iter_data->flags & BT_TAG_ITER_RESERVED; |
f26cdc85 | 337 | struct request *rq; |
2e315dc0 ML |
338 | bool ret = true; |
339 | bool iter_static_rqs = !!(iter_data->flags & BT_TAG_ITER_STATIC_RQS); | |
f26cdc85 | 340 | |
88459642 OS |
341 | if (!reserved) |
342 | bitnr += tags->nr_reserved_tags; | |
7f5562d5 JA |
343 | |
344 | /* | |
345 | * We can hit rq == NULL here, because the tagging functions | |
22f614bc | 346 | * test and set the bit before assigning ->rqs[]. |
7f5562d5 | 347 | */ |
2e315dc0 | 348 | if (iter_static_rqs) |
22f614bc ML |
349 | rq = tags->static_rqs[bitnr]; |
350 | else | |
2e315dc0 | 351 | rq = blk_mq_find_and_get_req(tags, bitnr); |
602380d2 ML |
352 | if (!rq) |
353 | return true; | |
2e315dc0 ML |
354 | |
355 | if (!(iter_data->flags & BT_TAG_ITER_STARTED) || | |
356 | blk_mq_request_started(rq)) | |
2dd6532e | 357 | ret = iter_data->fn(rq, iter_data->data); |
2e315dc0 ML |
358 | if (!iter_static_rqs) |
359 | blk_mq_put_rq_ref(rq); | |
360 | return ret; | |
88459642 OS |
361 | } |
362 | ||
c7b1bf5c BVA |
363 | /** |
364 | * bt_tags_for_each - iterate over the requests in a tag map | |
365 | * @tags: Tag map to iterate over. | |
366 | * @bt: sbitmap to examine. This is either the breserved_tags member | |
367 | * or the bitmap_tags member of struct blk_mq_tags. | |
368 | * @fn: Pointer to the function that will be called for each started | |
369 | * request. @fn will be called as follows: @fn(rq, @data, | |
ab11fe5a JA |
370 | * @reserved) where rq is a pointer to a request. Return true |
371 | * to continue iterating tags, false to stop. | |
c7b1bf5c | 372 | * @data: Will be passed as second argument to @fn. |
602380d2 | 373 | * @flags: BT_TAG_ITER_* |
c7b1bf5c | 374 | */ |
88459642 | 375 | static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt, |
602380d2 | 376 | busy_tag_iter_fn *fn, void *data, unsigned int flags) |
88459642 OS |
377 | { |
378 | struct bt_tags_iter_data iter_data = { | |
379 | .tags = tags, | |
380 | .fn = fn, | |
381 | .data = data, | |
602380d2 | 382 | .flags = flags, |
88459642 OS |
383 | }; |
384 | ||
385 | if (tags->rqs) | |
386 | sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data); | |
f26cdc85 KB |
387 | } |
388 | ||
602380d2 ML |
389 | static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags, |
390 | busy_tag_iter_fn *fn, void *priv, unsigned int flags) | |
391 | { | |
392 | WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED); | |
393 | ||
394 | if (tags->nr_reserved_tags) | |
ae0f1a73 | 395 | bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, |
602380d2 | 396 | flags | BT_TAG_ITER_RESERVED); |
ae0f1a73 | 397 | bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, flags); |
602380d2 ML |
398 | } |
399 | ||
c7b1bf5c | 400 | /** |
602380d2 | 401 | * blk_mq_all_tag_iter - iterate over all requests in a tag map |
c7b1bf5c | 402 | * @tags: Tag map to iterate over. |
602380d2 | 403 | * @fn: Pointer to the function that will be called for each |
c7b1bf5c BVA |
404 | * request. @fn will be called as follows: @fn(rq, @priv, |
405 | * reserved) where rq is a pointer to a request. 'reserved' | |
ab11fe5a JA |
406 | * indicates whether or not @rq is a reserved request. Return |
407 | * true to continue iterating tags, false to stop. | |
c7b1bf5c | 408 | * @priv: Will be passed as second argument to @fn. |
22f614bc ML |
409 | * |
410 | * Caller has to pass the tag map from which requests are allocated. | |
c7b1bf5c | 411 | */ |
602380d2 ML |
412 | void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, |
413 | void *priv) | |
f26cdc85 | 414 | { |
a8a5e383 | 415 | __blk_mq_all_tag_iter(tags, fn, priv, BT_TAG_ITER_STATIC_RQS); |
f26cdc85 | 416 | } |
f26cdc85 | 417 | |
c7b1bf5c BVA |
418 | /** |
419 | * blk_mq_tagset_busy_iter - iterate over all started requests in a tag set | |
420 | * @tagset: Tag set to iterate over. | |
421 | * @fn: Pointer to the function that will be called for each started | |
422 | * request. @fn will be called as follows: @fn(rq, @priv, | |
423 | * reserved) where rq is a pointer to a request. 'reserved' | |
ab11fe5a JA |
424 | * indicates whether or not @rq is a reserved request. Return |
425 | * true to continue iterating tags, false to stop. | |
c7b1bf5c | 426 | * @priv: Will be passed as second argument to @fn. |
2e315dc0 ML |
427 | * |
428 | * We grab one request reference before calling @fn and release it after | |
429 | * @fn returns. | |
c7b1bf5c | 430 | */ |
e0489487 SG |
431 | void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, |
432 | busy_tag_iter_fn *fn, void *priv) | |
433 | { | |
0994c64e JG |
434 | unsigned int flags = tagset->flags; |
435 | int i, nr_tags; | |
e0489487 | 436 | |
0994c64e JG |
437 | nr_tags = blk_mq_is_shared_tags(flags) ? 1 : tagset->nr_hw_queues; |
438 | ||
439 | for (i = 0; i < nr_tags; i++) { | |
e0489487 | 440 | if (tagset->tags && tagset->tags[i]) |
602380d2 ML |
441 | __blk_mq_all_tag_iter(tagset->tags[i], fn, priv, |
442 | BT_TAG_ITER_STARTED); | |
e0489487 SG |
443 | } |
444 | } | |
445 | EXPORT_SYMBOL(blk_mq_tagset_busy_iter); | |
446 | ||
2dd6532e | 447 | static bool blk_mq_tagset_count_completed_rqs(struct request *rq, void *data) |
f9934a80 ML |
448 | { |
449 | unsigned *count = data; | |
450 | ||
451 | if (blk_mq_request_completed(rq)) | |
452 | (*count)++; | |
453 | return true; | |
454 | } | |
455 | ||
456 | /** | |
9cf1adc6 BC |
457 | * blk_mq_tagset_wait_completed_request - Wait until all scheduled request |
458 | * completions have finished. | |
f9934a80 ML |
459 | * @tagset: Tag set to drain completed request |
460 | * | |
461 | * Note: This function has to be run after all IO queues are shutdown | |
462 | */ | |
463 | void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset) | |
464 | { | |
465 | while (true) { | |
466 | unsigned count = 0; | |
467 | ||
468 | blk_mq_tagset_busy_iter(tagset, | |
469 | blk_mq_tagset_count_completed_rqs, &count); | |
470 | if (!count) | |
471 | break; | |
472 | msleep(5); | |
473 | } | |
474 | } | |
475 | EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request); | |
476 | ||
c7b1bf5c BVA |
477 | /** |
478 | * blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag | |
479 | * @q: Request queue to examine. | |
480 | * @fn: Pointer to the function that will be called for each request | |
481 | * on @q. @fn will be called as follows: @fn(hctx, rq, @priv, | |
482 | * reserved) where rq is a pointer to a request and hctx points | |
483 | * to the hardware queue associated with the request. 'reserved' | |
484 | * indicates whether or not @rq is a reserved request. | |
485 | * @priv: Will be passed as third argument to @fn. | |
486 | * | |
487 | * Note: if @q->tag_set is shared with other request queues then @fn will be | |
488 | * called for all requests on all queues that share that tag set and not only | |
489 | * for requests associated with @q. | |
490 | */ | |
fc39f8d2 | 491 | void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, |
81481eb4 | 492 | void *priv) |
320ae51f | 493 | { |
f5bbbbe4 | 494 | /* |
4e5cc99e | 495 | * __blk_mq_update_nr_hw_queues() updates nr_hw_queues and hctx_table |
c7b1bf5c | 496 | * while the queue is frozen. So we can use q_usage_counter to avoid |
76cffccd | 497 | * racing with it. |
f5bbbbe4 | 498 | */ |
530ca2c9 | 499 | if (!percpu_ref_tryget(&q->q_usage_counter)) |
f5bbbbe4 | 500 | return; |
0bf6cd5b | 501 | |
fea9f92f JG |
502 | if (blk_mq_is_shared_tags(q->tag_set->flags)) { |
503 | struct blk_mq_tags *tags = q->tag_set->shared_tags; | |
504 | struct sbitmap_queue *bresv = &tags->breserved_tags; | |
505 | struct sbitmap_queue *btags = &tags->bitmap_tags; | |
0bf6cd5b CH |
506 | |
507 | if (tags->nr_reserved_tags) | |
fea9f92f JG |
508 | bt_for_each(NULL, q, bresv, fn, priv, true); |
509 | bt_for_each(NULL, q, btags, fn, priv, false); | |
510 | } else { | |
511 | struct blk_mq_hw_ctx *hctx; | |
4f481208 | 512 | unsigned long i; |
fea9f92f JG |
513 | |
514 | queue_for_each_hw_ctx(q, hctx, i) { | |
515 | struct blk_mq_tags *tags = hctx->tags; | |
516 | struct sbitmap_queue *bresv = &tags->breserved_tags; | |
517 | struct sbitmap_queue *btags = &tags->bitmap_tags; | |
518 | ||
519 | /* | |
520 | * If no software queues are currently mapped to this | |
521 | * hardware queue, there's nothing to check | |
522 | */ | |
523 | if (!blk_mq_hw_queue_mapped(hctx)) | |
524 | continue; | |
525 | ||
526 | if (tags->nr_reserved_tags) | |
527 | bt_for_each(hctx, q, bresv, fn, priv, true); | |
528 | bt_for_each(hctx, q, btags, fn, priv, false); | |
529 | } | |
4bb659b1 | 530 | } |
530ca2c9 | 531 | blk_queue_exit(q); |
4bb659b1 JA |
532 | } |
533 | ||
f4a644db OS |
534 | static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth, |
535 | bool round_robin, int node) | |
4bb659b1 | 536 | { |
f4a644db OS |
537 | return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL, |
538 | node); | |
4bb659b1 JA |
539 | } |
540 | ||
56b68085 JG |
541 | int blk_mq_init_bitmaps(struct sbitmap_queue *bitmap_tags, |
542 | struct sbitmap_queue *breserved_tags, | |
543 | unsigned int queue_depth, unsigned int reserved, | |
544 | int node, int alloc_policy) | |
4bb659b1 | 545 | { |
56b68085 | 546 | unsigned int depth = queue_depth - reserved; |
f4a644db | 547 | bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR; |
4bb659b1 | 548 | |
56b68085 | 549 | if (bt_alloc(bitmap_tags, depth, round_robin, node)) |
4d063237 | 550 | return -ENOMEM; |
56b68085 | 551 | if (bt_alloc(breserved_tags, reserved, round_robin, node)) |
88459642 | 552 | goto free_bitmap_tags; |
4bb659b1 | 553 | |
56b68085 JG |
554 | return 0; |
555 | ||
556 | free_bitmap_tags: | |
557 | sbitmap_queue_free(bitmap_tags); | |
558 | return -ENOMEM; | |
559 | } | |
560 | ||
320ae51f | 561 | struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, |
24391c0d | 562 | unsigned int reserved_tags, |
e155b0c2 | 563 | int node, int alloc_policy) |
320ae51f | 564 | { |
320ae51f | 565 | struct blk_mq_tags *tags; |
320ae51f JA |
566 | |
567 | if (total_tags > BLK_MQ_TAG_MAX) { | |
568 | pr_err("blk-mq: tag depth too large\n"); | |
569 | return NULL; | |
570 | } | |
571 | ||
572 | tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node); | |
573 | if (!tags) | |
574 | return NULL; | |
575 | ||
320ae51f JA |
576 | tags->nr_tags = total_tags; |
577 | tags->nr_reserved_tags = reserved_tags; | |
bd63141d | 578 | spin_lock_init(&tags->lock); |
320ae51f | 579 | |
ae0f1a73 JG |
580 | if (blk_mq_init_bitmaps(&tags->bitmap_tags, &tags->breserved_tags, |
581 | total_tags, reserved_tags, node, | |
582 | alloc_policy) < 0) { | |
4d063237 HR |
583 | kfree(tags); |
584 | return NULL; | |
585 | } | |
586 | return tags; | |
320ae51f JA |
587 | } |
588 | ||
e155b0c2 | 589 | void blk_mq_free_tags(struct blk_mq_tags *tags) |
320ae51f | 590 | { |
ae0f1a73 JG |
591 | sbitmap_queue_free(&tags->bitmap_tags); |
592 | sbitmap_queue_free(&tags->breserved_tags); | |
320ae51f JA |
593 | kfree(tags); |
594 | } | |
595 | ||
70f36b60 JA |
596 | int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, |
597 | struct blk_mq_tags **tagsptr, unsigned int tdepth, | |
598 | bool can_grow) | |
e3a2b3f9 | 599 | { |
70f36b60 JA |
600 | struct blk_mq_tags *tags = *tagsptr; |
601 | ||
602 | if (tdepth <= tags->nr_reserved_tags) | |
e3a2b3f9 JA |
603 | return -EINVAL; |
604 | ||
605 | /* | |
70f36b60 JA |
606 | * If we are allowed to grow beyond the original size, allocate |
607 | * a new set of tags before freeing the old one. | |
e3a2b3f9 | 608 | */ |
70f36b60 JA |
609 | if (tdepth > tags->nr_tags) { |
610 | struct blk_mq_tag_set *set = hctx->queue->tag_set; | |
611 | struct blk_mq_tags *new; | |
70f36b60 JA |
612 | |
613 | if (!can_grow) | |
614 | return -EINVAL; | |
615 | ||
616 | /* | |
617 | * We need some sort of upper limit, set it high enough that | |
618 | * no valid use cases should require more. | |
619 | */ | |
d97e594c | 620 | if (tdepth > MAX_SCHED_RQ) |
70f36b60 JA |
621 | return -EINVAL; |
622 | ||
e155b0c2 JG |
623 | /* |
624 | * Only the sbitmap needs resizing since we allocated the max | |
625 | * initially. | |
626 | */ | |
079a2e3e | 627 | if (blk_mq_is_shared_tags(set->flags)) |
e155b0c2 JG |
628 | return 0; |
629 | ||
63064be1 | 630 | new = blk_mq_alloc_map_and_rqs(set, hctx->queue_num, tdepth); |
70f36b60 JA |
631 | if (!new) |
632 | return -ENOMEM; | |
70f36b60 | 633 | |
645db34e | 634 | blk_mq_free_map_and_rqs(set, *tagsptr, hctx->queue_num); |
70f36b60 JA |
635 | *tagsptr = new; |
636 | } else { | |
637 | /* | |
638 | * Don't need (or can't) update reserved tags here, they | |
639 | * remain static and should never need resizing. | |
640 | */ | |
ae0f1a73 | 641 | sbitmap_queue_resize(&tags->bitmap_tags, |
75d6e175 | 642 | tdepth - tags->nr_reserved_tags); |
70f36b60 | 643 | } |
88459642 | 644 | |
e3a2b3f9 JA |
645 | return 0; |
646 | } | |
647 | ||
079a2e3e | 648 | void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size) |
32bc15af | 649 | { |
079a2e3e | 650 | struct blk_mq_tags *tags = set->shared_tags; |
e155b0c2 | 651 | |
ae0f1a73 | 652 | sbitmap_queue_resize(&tags->bitmap_tags, size - set->reserved_tags); |
32bc15af JG |
653 | } |
654 | ||
079a2e3e | 655 | void blk_mq_tag_update_sched_shared_tags(struct request_queue *q) |
a7e7388d | 656 | { |
079a2e3e | 657 | sbitmap_queue_resize(&q->sched_shared_tags->bitmap_tags, |
a7e7388d JG |
658 | q->nr_requests - q->tag_set->reserved_tags); |
659 | } | |
660 | ||
205fb5f5 BVA |
661 | /** |
662 | * blk_mq_unique_tag() - return a tag that is unique queue-wide | |
663 | * @rq: request for which to compute a unique tag | |
664 | * | |
665 | * The tag field in struct request is unique per hardware queue but not over | |
666 | * all hardware queues. Hence this function that returns a tag with the | |
667 | * hardware context index in the upper bits and the per hardware queue tag in | |
668 | * the lower bits. | |
669 | * | |
670 | * Note: When called for a request that is queued on a non-multiqueue request | |
671 | * queue, the hardware context index is set to zero. | |
672 | */ | |
673 | u32 blk_mq_unique_tag(struct request *rq) | |
674 | { | |
ea4f995e | 675 | return (rq->mq_hctx->queue_num << BLK_MQ_UNIQUE_TAG_BITS) | |
205fb5f5 BVA |
676 | (rq->tag & BLK_MQ_UNIQUE_TAG_MASK); |
677 | } | |
678 | EXPORT_SYMBOL(blk_mq_unique_tag); |