block: Introduce zone write plugging

[thirdparty/kernel/stable.git] / block / blk-mq.c

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 88b541e8873fb5e27630935c70ab739344520311..48eb7dd049d1fd7a46ec4fdae7375a60cbef176a 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -828,6 +828,8 @@ static void blk_complete_request(struct request *req)
                bio = next;
        } while (bio);
 
+       blk_zone_complete_request(req);
+
        /*
         * Reset counters so that the request stacking driver
         * can find how many bytes remain in the request
@@ -939,6 +941,7 @@ bool blk_update_request(struct request *req, blk_status_t error,
         * completely done
         */
        if (!req->bio) {
+               blk_zone_complete_request(req);
                /*
                 * Reset counters so that the request stacking driver
                 * can find how many bytes remain in the request
@@ -2963,15 +2966,30 @@ void blk_mq_submit_bio(struct bio *bio)
        struct request *rq;
        blk_status_t ret;
 
+       /*
+        * If the plug has a cached request for this queue, try to use it.
+        */
+       rq = blk_mq_peek_cached_request(plug, q, bio->bi_opf);
+
+       /*
+        * A BIO that was released from a zone write plug has already been
+        * through the preparation in this function, already holds a reference
+        * on the queue usage counter, and is the only write BIO in-flight for
+        * the target zone. Go straight to preparing a request for it.
+        */
+       if (bio_zone_write_plugging(bio)) {
+               nr_segs = bio->__bi_nr_segments;
+               if (rq)
+                       blk_queue_exit(q);
+               goto new_request;
+       }
+
        bio = blk_queue_bounce(bio, q);
 
        /*
-        * If the plug has a cached request for this queue, try use it.
-        *
         * The cached request already holds a q_usage_counter reference and we
         * don't have to acquire a new one if we use it.
         */
-       rq = blk_mq_peek_cached_request(plug, q, bio->bi_opf);
        if (!rq) {
                if (unlikely(bio_queue_enter(bio)))
                        return;
@@ -2988,6 +3006,10 @@ void blk_mq_submit_bio(struct bio *bio)
        if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
                goto queue_exit;
 
+       if (blk_queue_is_zoned(q) && blk_zone_plug_bio(bio, nr_segs))
+               goto queue_exit;
+
+new_request:
        if (!rq) {
                rq = blk_mq_get_new_requests(q, plug, bio, nr_segs);
                if (unlikely(!rq))
@@ -3006,6 +3028,7 @@ void blk_mq_submit_bio(struct bio *bio)
        if (ret != BLK_STS_OK) {
                bio->bi_status = ret;
                bio_endio(bio);
+               blk_zone_complete_request(rq);
                blk_mq_free_request(rq);
                return;
        }
@@ -3013,6 +3036,9 @@ void blk_mq_submit_bio(struct bio *bio)
        if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
                return;
 
+       if (bio_zone_write_plugging(bio))
+               blk_zone_write_plug_attempt_merge(rq);
+
        if (plug) {
                blk_add_rq_to_plug(plug, rq);
                return;