From: Caleb Sander Mateos Date: Wed, 30 Apr 2025 22:52:34 +0000 (-0600) Subject: ublk: store request pointer in ublk_io X-Git-Tag: v6.16-rc1~216^2~114 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=64d1dc522b349a257256f636feb335441cca2571;p=thirdparty%2Fkernel%2Flinux.git ublk: store request pointer in ublk_io A ublk_io is converted to a request in several places in the I/O path by using blk_mq_tag_to_rq() to look up the (qid, tag) on the ublk device's tagset. This involves a bunch of dereferences and a tag bounds check. To make this conversion cheaper, store the request pointer in ublk_io. Overlap this storage with the io_uring_cmd pointer. This is safe because the io_uring_cmd pointer is only valid if UBLK_IO_FLAG_ACTIVE is set on the ublk_io, the request pointer is valid if UBLK_IO_FLAG_OWNED_BY_SRV, and these flags are mutually exclusive. Suggested-by: Ming Lei Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20250430225234.2676781-10-csander@purestorage.com Signed-off-by: Jens Axboe --- diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 97c61c0bf964c..02e52b0663182 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -142,7 +142,12 @@ struct ublk_io { unsigned int flags; int res; - struct io_uring_cmd *cmd; + union { + /* valid if UBLK_IO_FLAG_ACTIVE is set */ + struct io_uring_cmd *cmd; + /* valid if UBLK_IO_FLAG_OWNED_BY_SRV is set */ + struct request *req; + }; }; struct ublk_queue { @@ -1124,9 +1129,12 @@ static void ublk_complete_rq(struct kref *ref) __ublk_complete_rq(req); } -static void ublk_complete_io_cmd(struct ublk_io *io, int res, - unsigned issue_flags) +static void ublk_complete_io_cmd(struct ublk_io *io, struct request *req, + int res, unsigned issue_flags) { + /* read cmd first because req will overwrite it */ + struct io_uring_cmd *cmd = io->cmd; + /* mark this cmd owned by ublksrv */ io->flags |= UBLK_IO_FLAG_OWNED_BY_SRV; @@ -1136,8 +1144,10 @@ static void ublk_complete_io_cmd(struct ublk_io *io, int res, */ io->flags &= ~UBLK_IO_FLAG_ACTIVE; + io->req = req; + /* tell ublksrv one io request is coming */ - io_uring_cmd_done(io->cmd, res, 0, issue_flags); + io_uring_cmd_done(cmd, res, 0, issue_flags); } #define UBLK_REQUEUE_DELAY_MS 3 @@ -1215,7 +1225,7 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, io->flags |= UBLK_IO_FLAG_NEED_GET_DATA; pr_devel("%s: need get data. qid %d tag %d io_flags %x\n", __func__, ubq->q_id, req->tag, io->flags); - ublk_complete_io_cmd(io, UBLK_IO_RES_NEED_GET_DATA, + ublk_complete_io_cmd(io, req, UBLK_IO_RES_NEED_GET_DATA, issue_flags); return; } @@ -1223,7 +1233,7 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, if (!ublk_start_io(ubq, req, io)) return; - ublk_complete_io_cmd(io, UBLK_IO_RES_OK, issue_flags); + ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags); } static void ublk_cmd_tw_cb(struct io_uring_cmd *cmd, @@ -1611,12 +1621,8 @@ static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq) for (i = 0; i < ubq->q_depth; i++) { struct ublk_io *io = &ubq->ios[i]; - if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV) { - struct request *rq; - - rq = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], i); - __ublk_fail_req(ubq, io, rq); - } + if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV) + __ublk_fail_req(ubq, io, io->req); } } @@ -1990,12 +1996,12 @@ static int ublk_commit_and_fetch(const struct ublk_queue *ubq, struct ublk_io *io, struct io_uring_cmd *cmd, const struct ublksrv_io_cmd *ub_cmd) { - struct blk_mq_tags *tags = ubq->dev->tag_set.tags[ub_cmd->q_id]; - struct request *req = blk_mq_tag_to_rq(tags, ub_cmd->tag); + struct request *req; if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) return -EINVAL; + req = io->req; if (ublk_need_map_io(ubq)) { /* * COMMIT_AND_FETCH_REQ has to provide IO buffer if @@ -2027,9 +2033,10 @@ static int ublk_commit_and_fetch(const struct ublk_queue *ubq, return 0; } -static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io, - struct request *req) +static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io) { + struct request *req = io->req; + /* * We have handled UBLK_IO_NEED_GET_DATA command, * so clear UBLK_IO_FLAG_NEED_GET_DATA now and just @@ -2055,7 +2062,6 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, u32 cmd_op = cmd->cmd_op; unsigned tag = ub_cmd->tag; int ret = -EINVAL; - struct request *req; pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n", __func__, cmd->cmd_op, ub_cmd->q_id, tag, @@ -2111,8 +2117,7 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) goto out; io->addr = ub_cmd->addr; - req = blk_mq_tag_to_rq(ub->tag_set.tags[ub_cmd->q_id], tag); - if (!ublk_get_data(ubq, io, req)) + if (!ublk_get_data(ubq, io)) return -EIOCBQUEUED; return UBLK_IO_RES_OK;