1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2003 Russell King, All Rights Reserved.
4 * Copyright 2006-2007 Pierre Ossman
6 #include <linux/slab.h>
7 #include <linux/module.h>
8 #include <linux/blkdev.h>
9 #include <linux/freezer.h>
10 #include <linux/kthread.h>
11 #include <linux/scatterlist.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/backing-dev.h>
15 #include <linux/mmc/card.h>
16 #include <linux/mmc/host.h>
24 #define MMC_DMA_MAP_MERGE_SEGMENTS 512
26 static inline bool mmc_cqe_dcmd_busy(struct mmc_queue
*mq
)
28 /* Allow only 1 DCMD at a time */
29 return mq
->in_flight
[MMC_ISSUE_DCMD
];
32 void mmc_cqe_check_busy(struct mmc_queue
*mq
)
34 if ((mq
->cqe_busy
& MMC_CQE_DCMD_BUSY
) && !mmc_cqe_dcmd_busy(mq
))
35 mq
->cqe_busy
&= ~MMC_CQE_DCMD_BUSY
;
37 mq
->cqe_busy
&= ~MMC_CQE_QUEUE_FULL
;
40 static inline bool mmc_cqe_can_dcmd(struct mmc_host
*host
)
42 return host
->caps2
& MMC_CAP2_CQE_DCMD
;
45 static enum mmc_issue_type
mmc_cqe_issue_type(struct mmc_host
*host
,
48 switch (req_op(req
)) {
52 case REQ_OP_SECURE_ERASE
:
53 return MMC_ISSUE_SYNC
;
55 return mmc_cqe_can_dcmd(host
) ? MMC_ISSUE_DCMD
: MMC_ISSUE_SYNC
;
57 return MMC_ISSUE_ASYNC
;
61 enum mmc_issue_type
mmc_issue_type(struct mmc_queue
*mq
, struct request
*req
)
63 struct mmc_host
*host
= mq
->card
->host
;
65 if (mq
->use_cqe
&& !host
->hsq_enabled
)
66 return mmc_cqe_issue_type(host
, req
);
68 if (req_op(req
) == REQ_OP_READ
|| req_op(req
) == REQ_OP_WRITE
)
69 return MMC_ISSUE_ASYNC
;
71 return MMC_ISSUE_SYNC
;
74 static void __mmc_cqe_recovery_notifier(struct mmc_queue
*mq
)
76 if (!mq
->recovery_needed
) {
77 mq
->recovery_needed
= true;
78 schedule_work(&mq
->recovery_work
);
82 void mmc_cqe_recovery_notifier(struct mmc_request
*mrq
)
84 struct mmc_queue_req
*mqrq
= container_of(mrq
, struct mmc_queue_req
,
86 struct request
*req
= mmc_queue_req_to_req(mqrq
);
87 struct request_queue
*q
= req
->q
;
88 struct mmc_queue
*mq
= q
->queuedata
;
91 spin_lock_irqsave(&mq
->lock
, flags
);
92 __mmc_cqe_recovery_notifier(mq
);
93 spin_unlock_irqrestore(&mq
->lock
, flags
);
96 static enum blk_eh_timer_return
mmc_cqe_timed_out(struct request
*req
)
98 struct mmc_queue_req
*mqrq
= req_to_mmc_queue_req(req
);
99 struct mmc_request
*mrq
= &mqrq
->brq
.mrq
;
100 struct mmc_queue
*mq
= req
->q
->queuedata
;
101 struct mmc_host
*host
= mq
->card
->host
;
102 enum mmc_issue_type issue_type
= mmc_issue_type(mq
, req
);
103 bool recovery_needed
= false;
105 switch (issue_type
) {
106 case MMC_ISSUE_ASYNC
:
108 if (host
->cqe_ops
->cqe_timeout(host
, mrq
, &recovery_needed
)) {
110 __mmc_cqe_recovery_notifier(mq
);
111 return BLK_EH_RESET_TIMER
;
113 /* No timeout (XXX: huh? comment doesn't make much sense) */
114 blk_mq_complete_request(req
);
117 /* Timeout is handled by mmc core */
118 return BLK_EH_RESET_TIMER
;
122 static enum blk_eh_timer_return
mmc_mq_timed_out(struct request
*req
,
125 struct request_queue
*q
= req
->q
;
126 struct mmc_queue
*mq
= q
->queuedata
;
127 struct mmc_card
*card
= mq
->card
;
128 struct mmc_host
*host
= card
->host
;
132 spin_lock_irqsave(&mq
->lock
, flags
);
134 if (mq
->recovery_needed
|| !mq
->use_cqe
|| host
->hsq_enabled
)
135 ret
= BLK_EH_RESET_TIMER
;
137 ret
= mmc_cqe_timed_out(req
);
139 spin_unlock_irqrestore(&mq
->lock
, flags
);
144 static void mmc_mq_recovery_handler(struct work_struct
*work
)
146 struct mmc_queue
*mq
= container_of(work
, struct mmc_queue
,
148 struct request_queue
*q
= mq
->queue
;
149 struct mmc_host
*host
= mq
->card
->host
;
151 mmc_get_card(mq
->card
, &mq
->ctx
);
153 mq
->in_recovery
= true;
155 if (mq
->use_cqe
&& !host
->hsq_enabled
)
156 mmc_blk_cqe_recovery(mq
);
158 mmc_blk_mq_recovery(mq
);
160 mq
->in_recovery
= false;
162 spin_lock_irq(&mq
->lock
);
163 mq
->recovery_needed
= false;
164 spin_unlock_irq(&mq
->lock
);
166 if (host
->hsq_enabled
)
167 host
->cqe_ops
->cqe_recovery_finish(host
);
169 mmc_put_card(mq
->card
, &mq
->ctx
);
171 blk_mq_run_hw_queues(q
, true);
174 static struct scatterlist
*mmc_alloc_sg(int sg_len
, gfp_t gfp
)
176 struct scatterlist
*sg
;
178 sg
= kmalloc_array(sg_len
, sizeof(*sg
), gfp
);
180 sg_init_table(sg
, sg_len
);
185 static void mmc_queue_setup_discard(struct request_queue
*q
,
186 struct mmc_card
*card
)
188 unsigned max_discard
;
190 max_discard
= mmc_calc_max_discard(card
);
194 blk_queue_flag_set(QUEUE_FLAG_DISCARD
, q
);
195 blk_queue_max_discard_sectors(q
, max_discard
);
196 q
->limits
.discard_granularity
= card
->pref_erase
<< 9;
197 /* granularity must not be greater than max. discard */
198 if (card
->pref_erase
> max_discard
)
199 q
->limits
.discard_granularity
= 0;
200 if (mmc_can_secure_erase_trim(card
))
201 blk_queue_flag_set(QUEUE_FLAG_SECERASE
, q
);
204 static unsigned int mmc_get_max_segments(struct mmc_host
*host
)
206 return host
->can_dma_map_merge
? MMC_DMA_MAP_MERGE_SEGMENTS
:
211 * mmc_init_request() - initialize the MMC-specific per-request data
212 * @q: the request queue
214 * @gfp: memory allocation policy
216 static int __mmc_init_request(struct mmc_queue
*mq
, struct request
*req
,
219 struct mmc_queue_req
*mq_rq
= req_to_mmc_queue_req(req
);
220 struct mmc_card
*card
= mq
->card
;
221 struct mmc_host
*host
= card
->host
;
223 mq_rq
->sg
= mmc_alloc_sg(mmc_get_max_segments(host
), gfp
);
230 static void mmc_exit_request(struct request_queue
*q
, struct request
*req
)
232 struct mmc_queue_req
*mq_rq
= req_to_mmc_queue_req(req
);
238 static int mmc_mq_init_request(struct blk_mq_tag_set
*set
, struct request
*req
,
239 unsigned int hctx_idx
, unsigned int numa_node
)
241 return __mmc_init_request(set
->driver_data
, req
, GFP_KERNEL
);
244 static void mmc_mq_exit_request(struct blk_mq_tag_set
*set
, struct request
*req
,
245 unsigned int hctx_idx
)
247 struct mmc_queue
*mq
= set
->driver_data
;
249 mmc_exit_request(mq
->queue
, req
);
252 static blk_status_t
mmc_mq_queue_rq(struct blk_mq_hw_ctx
*hctx
,
253 const struct blk_mq_queue_data
*bd
)
255 struct request
*req
= bd
->rq
;
256 struct request_queue
*q
= req
->q
;
257 struct mmc_queue
*mq
= q
->queuedata
;
258 struct mmc_card
*card
= mq
->card
;
259 struct mmc_host
*host
= card
->host
;
260 enum mmc_issue_type issue_type
;
261 enum mmc_issued issued
;
262 bool get_card
, cqe_retune_ok
;
265 if (mmc_card_removed(mq
->card
)) {
266 req
->rq_flags
|= RQF_QUIET
;
267 return BLK_STS_IOERR
;
270 issue_type
= mmc_issue_type(mq
, req
);
272 spin_lock_irq(&mq
->lock
);
274 if (mq
->recovery_needed
|| mq
->busy
) {
275 spin_unlock_irq(&mq
->lock
);
276 return BLK_STS_RESOURCE
;
279 switch (issue_type
) {
281 if (mmc_cqe_dcmd_busy(mq
)) {
282 mq
->cqe_busy
|= MMC_CQE_DCMD_BUSY
;
283 spin_unlock_irq(&mq
->lock
);
284 return BLK_STS_RESOURCE
;
287 case MMC_ISSUE_ASYNC
:
289 * For MMC host software queue, we only allow 2 requests in
290 * flight to avoid a long latency.
292 if (host
->hsq_enabled
&& mq
->in_flight
[issue_type
] > 2) {
293 spin_unlock_irq(&mq
->lock
);
294 return BLK_STS_RESOURCE
;
299 * Timeouts are handled by mmc core, and we don't have a host
300 * API to abort requests, so we can't handle the timeout anyway.
301 * However, when the timeout happens, blk_mq_complete_request()
302 * no longer works (to stop the request disappearing under us).
303 * To avoid racing with that, set a large timeout.
305 req
->timeout
= 600 * HZ
;
309 /* Parallel dispatch of requests is not supported at the moment */
312 mq
->in_flight
[issue_type
] += 1;
313 get_card
= (mmc_tot_in_flight(mq
) == 1);
314 cqe_retune_ok
= (mmc_cqe_qcnt(mq
) == 1);
316 spin_unlock_irq(&mq
->lock
);
318 if (!(req
->rq_flags
& RQF_DONTPREP
)) {
319 req_to_mmc_queue_req(req
)->retries
= 0;
320 req
->rq_flags
|= RQF_DONTPREP
;
324 mmc_get_card(card
, &mq
->ctx
);
327 host
->retune_now
= host
->need_retune
&& cqe_retune_ok
&&
331 blk_mq_start_request(req
);
333 issued
= mmc_blk_mq_issue_rq(mq
, req
);
337 ret
= BLK_STS_RESOURCE
;
339 case MMC_REQ_FAILED_TO_START
:
347 if (issued
!= MMC_REQ_STARTED
) {
348 bool put_card
= false;
350 spin_lock_irq(&mq
->lock
);
351 mq
->in_flight
[issue_type
] -= 1;
352 if (mmc_tot_in_flight(mq
) == 0)
355 spin_unlock_irq(&mq
->lock
);
357 mmc_put_card(card
, &mq
->ctx
);
359 WRITE_ONCE(mq
->busy
, false);
365 static const struct blk_mq_ops mmc_mq_ops
= {
366 .queue_rq
= mmc_mq_queue_rq
,
367 .init_request
= mmc_mq_init_request
,
368 .exit_request
= mmc_mq_exit_request
,
369 .complete
= mmc_blk_mq_complete
,
370 .timeout
= mmc_mq_timed_out
,
373 static void mmc_setup_queue(struct mmc_queue
*mq
, struct mmc_card
*card
)
375 struct mmc_host
*host
= card
->host
;
376 unsigned block_size
= 512;
378 blk_queue_flag_set(QUEUE_FLAG_NONROT
, mq
->queue
);
379 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM
, mq
->queue
);
380 if (mmc_can_erase(card
))
381 mmc_queue_setup_discard(mq
->queue
, card
);
383 if (!mmc_dev(host
)->dma_mask
|| !*mmc_dev(host
)->dma_mask
)
384 blk_queue_bounce_limit(mq
->queue
, BLK_BOUNCE_HIGH
);
385 blk_queue_max_hw_sectors(mq
->queue
,
386 min(host
->max_blk_count
, host
->max_req_size
/ 512));
387 if (host
->can_dma_map_merge
)
388 WARN(!blk_queue_can_use_dma_map_merging(mq
->queue
,
390 "merging was advertised but not possible");
391 blk_queue_max_segments(mq
->queue
, mmc_get_max_segments(host
));
393 if (mmc_card_mmc(card
))
394 block_size
= card
->ext_csd
.data_sector_size
;
396 blk_queue_logical_block_size(mq
->queue
, block_size
);
398 * After blk_queue_can_use_dma_map_merging() was called with succeed,
399 * since it calls blk_queue_virt_boundary(), the mmc should not call
400 * both blk_queue_max_segment_size().
402 if (!host
->can_dma_map_merge
)
403 blk_queue_max_segment_size(mq
->queue
,
404 round_down(host
->max_seg_size
, block_size
));
406 dma_set_max_seg_size(mmc_dev(host
), queue_max_segment_size(mq
->queue
));
408 INIT_WORK(&mq
->recovery_work
, mmc_mq_recovery_handler
);
409 INIT_WORK(&mq
->complete_work
, mmc_blk_mq_complete_work
);
411 mutex_init(&mq
->complete_lock
);
413 init_waitqueue_head(&mq
->wait
);
416 static inline bool mmc_merge_capable(struct mmc_host
*host
)
418 return host
->caps2
& MMC_CAP2_MERGE_CAPABLE
;
421 /* Set queue depth to get a reasonable value for q->nr_requests */
422 #define MMC_QUEUE_DEPTH 64
425 * mmc_init_queue - initialise a queue structure.
427 * @card: mmc card to attach this queue
429 * Initialise a MMC card request queue.
431 int mmc_init_queue(struct mmc_queue
*mq
, struct mmc_card
*card
)
433 struct mmc_host
*host
= card
->host
;
437 mq
->use_cqe
= host
->cqe_enabled
;
439 spin_lock_init(&mq
->lock
);
441 memset(&mq
->tag_set
, 0, sizeof(mq
->tag_set
));
442 mq
->tag_set
.ops
= &mmc_mq_ops
;
444 * The queue depth for CQE must match the hardware because the request
445 * tag is used to index the hardware queue.
447 if (mq
->use_cqe
&& !host
->hsq_enabled
)
448 mq
->tag_set
.queue_depth
=
449 min_t(int, card
->ext_csd
.cmdq_depth
, host
->cqe_qdepth
);
451 mq
->tag_set
.queue_depth
= MMC_QUEUE_DEPTH
;
452 mq
->tag_set
.numa_node
= NUMA_NO_NODE
;
453 mq
->tag_set
.flags
= BLK_MQ_F_SHOULD_MERGE
| BLK_MQ_F_BLOCKING
;
454 mq
->tag_set
.nr_hw_queues
= 1;
455 mq
->tag_set
.cmd_size
= sizeof(struct mmc_queue_req
);
456 mq
->tag_set
.driver_data
= mq
;
459 * Since blk_mq_alloc_tag_set() calls .init_request() of mmc_mq_ops,
460 * the host->can_dma_map_merge should be set before to get max_segs
461 * from mmc_get_max_segments().
463 if (mmc_merge_capable(host
) &&
464 host
->max_segs
< MMC_DMA_MAP_MERGE_SEGMENTS
&&
465 dma_get_merge_boundary(mmc_dev(host
)))
466 host
->can_dma_map_merge
= 1;
468 host
->can_dma_map_merge
= 0;
470 ret
= blk_mq_alloc_tag_set(&mq
->tag_set
);
474 mq
->queue
= blk_mq_init_queue(&mq
->tag_set
);
475 if (IS_ERR(mq
->queue
)) {
476 ret
= PTR_ERR(mq
->queue
);
480 if (mmc_host_is_spi(host
) && host
->use_spi_crc
)
481 mq
->queue
->backing_dev_info
->capabilities
|=
482 BDI_CAP_STABLE_WRITES
;
484 mq
->queue
->queuedata
= mq
;
485 blk_queue_rq_timeout(mq
->queue
, 60 * HZ
);
487 mmc_setup_queue(mq
, card
);
491 blk_mq_free_tag_set(&mq
->tag_set
);
495 void mmc_queue_suspend(struct mmc_queue
*mq
)
497 blk_mq_quiesce_queue(mq
->queue
);
500 * The host remains claimed while there are outstanding requests, so
501 * simply claiming and releasing here ensures there are none.
503 mmc_claim_host(mq
->card
->host
);
504 mmc_release_host(mq
->card
->host
);
507 void mmc_queue_resume(struct mmc_queue
*mq
)
509 blk_mq_unquiesce_queue(mq
->queue
);
512 void mmc_cleanup_queue(struct mmc_queue
*mq
)
514 struct request_queue
*q
= mq
->queue
;
517 * The legacy code handled the possibility of being suspended,
518 * so do that here too.
520 if (blk_queue_quiesced(q
))
521 blk_mq_unquiesce_queue(q
);
523 blk_cleanup_queue(q
);
524 blk_mq_free_tag_set(&mq
->tag_set
);
527 * A request can be completed before the next request, potentially
528 * leaving a complete_work with nothing to do. Such a work item might
529 * still be queued at this point. Flush it.
531 flush_work(&mq
->complete_work
);
537 * Prepare the sg list(s) to be handed of to the host driver
539 unsigned int mmc_queue_map_sg(struct mmc_queue
*mq
, struct mmc_queue_req
*mqrq
)
541 struct request
*req
= mmc_queue_req_to_req(mqrq
);
543 return blk_rq_map_sg(mq
->queue
, req
, mqrq
->sg
);