block: don't merge across cgroup boundaries if blkcg is enabled

commit 6b2b04590b51aa4cf395fcd185ce439cab5961dc upstream.

blk-iocost and iolatency are cgroup aware rq-qos policies but they didn't
disable merges across different cgroups. This obviously can lead to
accounting and control errors but more importantly to priority inversions -
e.g. an IO which belongs to a higher priority cgroup or IO class may end up
getting throttled incorrectly because it gets merged to an IO issued from a
low priority cgroup.

Fix it by adding blk_cgroup_mergeable() which is called from merge paths and
rejects cross-cgroup and cross-issue_as_root merges.

Signed-off-by: Tejun Heo <tj@kernel.org>
Fixes: d70675121546 ("block: introduce blk-iolatency io controller")
Cc: stable@vger.kernel.org # v4.19+
Cc: Josef Bacik <jbacik@fb.com>
Link: https://lore.kernel.org/r/Yi/eE/6zFNyWJ+qd@slm.duckdns.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/block/blk-merge.c b/block/blk-merge.c
index 7efa8c3e2b727f6367af02e0afb201d938f74b65..8054410536fd985575355d0507ce0d4a2132d907 100644 (file)
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -7,6 +7,8 @@
 #include <linux/bio.h>
 #include <linux/blkdev.h>
 #include <linux/scatterlist.h>
+#include <linux/blkdev.h>
+#include <linux/blk-cgroup.h>
 
 #include <trace/events/block.h>
 
@@ -486,6 +488,9 @@ static inline int ll_new_hw_segment(struct request_queue *q,
        if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(q))
                goto no_merge;
 
+       if (!blk_cgroup_mergeable(req, bio))
+               goto no_merge;
+
        if (blk_integrity_merge_bio(q, req, bio) == false)
                goto no_merge;
 
@@ -609,6 +614,9 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
        if (total_phys_segments > queue_max_segments(q))
                return 0;
 
+       if (!blk_cgroup_mergeable(req, next->bio))
+               return 0;
+
        if (blk_integrity_merge_rq(q, req, next) == false)
                return 0;
 
@@ -843,6 +851,10 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
        if (rq->rq_disk != bio->bi_disk || req_no_special_merge(rq))
                return false;
 
+       /* don't merge across cgroup boundaries */
+       if (!blk_cgroup_mergeable(rq, bio))
+               return false;
+
        /* only merge integrity protected bio into ditto rq */
        if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
                return false;

diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h
index 6d766a19f2bbb2b62facc79ff3871aa81be68534..8f1be8b49391099539b57a6640bb5ea726ca955a 100644 (file)
--- a/include/linux/blk-cgroup.h
+++ b/include/linux/blk-cgroup.h
@@ -21,6 +21,7 @@
 #include <linux/blkdev.h>
 #include <linux/atomic.h>
 #include <linux/kthread.h>
+#include <linux/blkdev.h>
 
 /* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
 #define BLKG_STAT_CPU_BATCH    (INT_MAX / 2)
@@ -844,6 +845,21 @@ static inline void blkcg_use_delay(struct blkcg_gq *blkg)
                atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);
 }
 
+/**
+ * blk_cgroup_mergeable - Determine whether to allow or disallow merges
+ * @rq: request to merge into
+ * @bio: bio to merge
+ *
+ * @bio and @rq should belong to the same cgroup and their issue_as_root should
+ * match. The latter is necessary as we don't want to throttle e.g. a metadata
+ * update because it happens to be next to a regular IO.
+ */
+static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio)
+{
+       return rq->bio->bi_blkg == bio->bi_blkg &&
+               bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio);
+}
+
 static inline int blkcg_unuse_delay(struct blkcg_gq *blkg)
 {
        int old = atomic_read(&blkg->use_delay);
@@ -947,6 +963,7 @@ static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q
 
 static inline bool blkcg_bio_issue_check(struct request_queue *q,
                                         struct bio *bio) { return true; }
+static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) { return true; }
 
 #define blk_queue_for_each_rl(rl, q)   \
        for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)