selftests: ublk: handle UBLK_U_IO_PREP_IO_CMDS

Implement support for UBLK_U_IO_PREP_IO_CMDS in the batch I/O framework:

- Add batch command initialization and setup functions
- Implement prep command queueing with proper buffer management
- Add command completion handling for prep and commit commands
- Integrate batch I/O setup into thread initialization
- Update CQE handling to support batch commands

The implementation uses the previously established buffer management
infrastructure to queue UBLK_U_IO_PREP_IO_CMDS commands. Commands are
prepared in the first thread context and use commit buffers for
efficient command batching.

Key changes:
- ublk_batch_queue_prep_io_cmds() prepares I/O command batches
- ublk_batch_compl_cmd() handles batch command completions
- Modified thread setup to use batch operations when enabled
- Enhanced buffer index calculation for batch mode

Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/tools/testing/selftests/ublk/batch.c b/tools/testing/selftests/ublk/batch.c
index 609e6073c9c03a536b960b51cc8cd69c98fb65e5..079cae77add131f1d6b8f4af30a3017a7dae357d 100644 (file)
--- a/tools/testing/selftests/ublk/batch.c
+++ b/tools/testing/selftests/ublk/batch.c
@@ -150,3 +150,117 @@ void ublk_batch_free_buf(struct ublk_thread *t)
 {
        free_batch_commit_buf(t);
 }
+
+static void ublk_init_batch_cmd(struct ublk_thread *t, __u16 q_id,
+                               struct io_uring_sqe *sqe, unsigned op,
+                               unsigned short elem_bytes,
+                               unsigned short nr_elem,
+                               unsigned short buf_idx)
+{
+       struct ublk_batch_io *cmd;
+       __u64 user_data;
+
+       cmd = (struct ublk_batch_io *)ublk_get_sqe_cmd(sqe);
+
+       ublk_set_sqe_cmd_op(sqe, op);
+
+       sqe->fd = 0;    /* dev->fds[0] */
+       sqe->opcode     = IORING_OP_URING_CMD;
+       sqe->flags      = IOSQE_FIXED_FILE;
+
+       cmd->q_id       = q_id;
+       cmd->flags      = 0;
+       cmd->reserved   = 0;
+       cmd->elem_bytes = elem_bytes;
+       cmd->nr_elem    = nr_elem;
+
+       user_data = build_user_data(buf_idx, _IOC_NR(op), 0, q_id, 0);
+       io_uring_sqe_set_data64(sqe, user_data);
+
+       t->cmd_inflight += 1;
+
+       ublk_dbg(UBLK_DBG_IO_CMD, "%s: thread %u qid %d cmd_op %x data %lx "
+                       "nr_elem %u elem_bytes %u buf_size %u buf_idx %d "
+                       "cmd_inflight %u\n",
+                       __func__, t->idx, q_id, op, user_data,
+                       cmd->nr_elem, cmd->elem_bytes,
+                       nr_elem * elem_bytes, buf_idx, t->cmd_inflight);
+}
+
+static void ublk_setup_commit_sqe(struct ublk_thread *t,
+                                 struct io_uring_sqe *sqe,
+                                 unsigned short buf_idx)
+{
+       struct ublk_batch_io *cmd;
+
+       cmd = (struct ublk_batch_io *)ublk_get_sqe_cmd(sqe);
+
+       /* Use plain user buffer instead of fixed buffer */
+       cmd->flags |= t->cmd_flags;
+}
+
+int ublk_batch_queue_prep_io_cmds(struct ublk_thread *t, struct ublk_queue *q)
+{
+       unsigned short nr_elem = q->q_depth;
+       unsigned short buf_idx = ublk_alloc_commit_buf(t);
+       struct io_uring_sqe *sqe;
+       void *buf;
+       int i;
+
+       ublk_assert(buf_idx != UBLKS_T_COMMIT_BUF_INV_IDX);
+
+       ublk_io_alloc_sqes(t, &sqe, 1);
+
+       ublk_assert(nr_elem == q->q_depth);
+       buf = ublk_get_commit_buf(t, buf_idx);
+       for (i = 0; i < nr_elem; i++) {
+               struct ublk_batch_elem *elem = (struct ublk_batch_elem *)(
+                               buf + i * t->commit_buf_elem_size);
+               struct ublk_io *io = &q->ios[i];
+
+               elem->tag = i;
+               elem->result = 0;
+
+               if (ublk_queue_use_auto_zc(q))
+                       elem->buf_index = ublk_batch_io_buf_idx(t, q, i);
+               else if (!ublk_queue_no_buf(q))
+                       elem->buf_addr = (__u64)io->buf_addr;
+       }
+
+       sqe->addr = (__u64)buf;
+       sqe->len = t->commit_buf_elem_size * nr_elem;
+
+       ublk_init_batch_cmd(t, q->q_id, sqe, UBLK_U_IO_PREP_IO_CMDS,
+                       t->commit_buf_elem_size, nr_elem, buf_idx);
+       ublk_setup_commit_sqe(t, sqe, buf_idx);
+       return 0;
+}
+
+static void ublk_batch_compl_commit_cmd(struct ublk_thread *t,
+                                       const struct io_uring_cqe *cqe,
+                                       unsigned op)
+{
+       unsigned short buf_idx = user_data_to_tag(cqe->user_data);
+
+       if (op == _IOC_NR(UBLK_U_IO_PREP_IO_CMDS))
+               ublk_assert(cqe->res == 0);
+       else if (op == _IOC_NR(UBLK_U_IO_COMMIT_IO_CMDS))
+               ;//assert(cqe->res == t->commit_buf_size);
+       else
+               ublk_assert(0);
+
+       ublk_free_commit_buf(t, buf_idx);
+}
+
+void ublk_batch_compl_cmd(struct ublk_thread *t,
+                         const struct io_uring_cqe *cqe)
+{
+       unsigned op = user_data_to_op(cqe->user_data);
+
+       if (op == _IOC_NR(UBLK_U_IO_PREP_IO_CMDS) ||
+                       op == _IOC_NR(UBLK_U_IO_COMMIT_IO_CMDS)) {
+               t->cmd_inflight--;
+               ublk_batch_compl_commit_cmd(t, cqe, op);
+               return;
+       }
+}

diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c
index 3864f42e6c296457cfcb719bfb9c33ac0efbad90..dba912a44eb3aee670182a433c80d61603707b42 100644 (file)
--- a/tools/testing/selftests/ublk/kublk.c
+++ b/tools/testing/selftests/ublk/kublk.c
@@ -840,6 +840,8 @@ static void ublk_handle_uring_cmd(struct ublk_thread *t,
        unsigned tag = user_data_to_tag(cqe->user_data);
        struct ublk_io *io = &q->ios[tag];
 
+       t->cmd_inflight--;
+
        if (!fetch) {
                t->state |= UBLKS_T_STOPPING;
                io->flags &= ~UBLKS_IO_NEED_FETCH_RQ;
@@ -874,28 +876,30 @@ static void ublk_handle_cqe(struct ublk_thread *t,
 {
        struct ublk_dev *dev = t->dev;
        unsigned q_id = user_data_to_q_id(cqe->user_data);
-       struct ublk_queue *q = &dev->q[q_id];
        unsigned cmd_op = user_data_to_op(cqe->user_data);
 
        if (cqe->res < 0 && cqe->res != -ENODEV)
-               ublk_err("%s: res %d userdata %llx queue state %x\n", __func__,
-                               cqe->res, cqe->user_data, q->flags);
+               ublk_err("%s: res %d userdata %llx thread state %x\n", __func__,
+                               cqe->res, cqe->user_data, t->state);
 
-       ublk_dbg(UBLK_DBG_IO_CMD, "%s: res %d (qid %d tag %u cmd_op %u target %d/%d) stopping %d\n",
-                       __func__, cqe->res, q->q_id, user_data_to_tag(cqe->user_data),
-                       cmd_op, is_target_io(cqe->user_data),
+       ublk_dbg(UBLK_DBG_IO_CMD, "%s: res %d (thread %d qid %d tag %u cmd_op %x "
+                       "data %lx target %d/%d) stopping %d\n",
+                       __func__, cqe->res, t->idx, q_id,
+                       user_data_to_tag(cqe->user_data),
+                       cmd_op, cqe->user_data, is_target_io(cqe->user_data),
                        user_data_to_tgt_data(cqe->user_data),
                        (t->state & UBLKS_T_STOPPING));
 
        /* Don't retrieve io in case of target io */
        if (is_target_io(cqe->user_data)) {
-               ublksrv_handle_tgt_cqe(t, q, cqe);
+               ublksrv_handle_tgt_cqe(t, &dev->q[q_id], cqe);
                return;
        }
 
-       t->cmd_inflight--;
-
-       ublk_handle_uring_cmd(t, q, cqe);
+       if (ublk_thread_batch_io(t))
+               ublk_batch_compl_cmd(t, cqe);
+       else
+               ublk_handle_uring_cmd(t, &dev->q[q_id], cqe);
 }
 
 static int ublk_reap_events_uring(struct ublk_thread *t)
@@ -952,6 +956,22 @@ static void ublk_thread_set_sched_affinity(const struct ublk_thread_info *info)
                                info->dev->dev_info.dev_id, info->idx);
 }
 
+static void ublk_batch_setup_queues(struct ublk_thread *t)
+{
+       int i;
+
+       /* setup all queues in the 1st thread */
+       for (i = 0; i < t->dev->dev_info.nr_hw_queues; i++) {
+               struct ublk_queue *q = &t->dev->q[i];
+               int ret;
+
+               ret = ublk_batch_queue_prep_io_cmds(t, q);
+               ublk_assert(ret == 0);
+               ret = ublk_process_io(t);
+               ublk_assert(ret >= 0);
+       }
+}
+
 static __attribute__((noinline)) int __ublk_io_handler_fn(struct ublk_thread_info *info)
 {
        struct ublk_thread t = {
@@ -972,8 +992,14 @@ static __attribute__((noinline)) int __ublk_io_handler_fn(struct ublk_thread_inf
        ublk_dbg(UBLK_DBG_THREAD, "tid %d: ublk dev %d thread %u started\n",
                        gettid(), dev_id, t.idx);
 
-       /* submit all io commands to ublk driver */
-       ublk_submit_fetch_commands(&t);
+       if (!ublk_thread_batch_io(&t)) {
+               /* submit all io commands to ublk driver */
+               ublk_submit_fetch_commands(&t);
+       } else if (!t.idx) {
+               /* prepare all io commands in the 1st thread context */
+               ublk_batch_setup_queues(&t);
+       }
+
        do {
                if (ublk_process_io(&t) < 0)
                        break;

diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h
index 424c333596ac7e760ee75588c01465218dc7bf1c..08320d44c7c29b4dc775aa97e6c9c4dbc253862a 100644 (file)
--- a/tools/testing/selftests/ublk/kublk.h
+++ b/tools/testing/selftests/ublk/kublk.h
@@ -440,10 +440,16 @@ static inline void ublk_set_sqe_cmd_op(struct io_uring_sqe *sqe, __u32 cmd_op)
        addr[1] = 0;
 }
 
+static inline unsigned short ublk_batch_io_buf_idx(
+               const struct ublk_thread *t, const struct ublk_queue *q,
+               unsigned tag);
+
 static inline unsigned short ublk_io_buf_idx(const struct ublk_thread *t,
                                             const struct ublk_queue *q,
                                             unsigned tag)
 {
+       if (ublk_queue_batch_io(q))
+               return ublk_batch_io_buf_idx(t, q, tag);
        return q->ios[tag].buf_index;
 }
 
@@ -511,6 +517,22 @@ static inline int ublk_queue_no_buf(const struct ublk_queue *q)
        return ublk_queue_use_zc(q) || ublk_queue_use_auto_zc(q);
 }
 
+/*
+ * Each IO's buffer index has to be calculated by this helper for
+ * UBLKS_T_BATCH_IO
+ */
+static inline unsigned short ublk_batch_io_buf_idx(
+               const struct ublk_thread *t, const struct ublk_queue *q,
+               unsigned tag)
+{
+       return tag;
+}
+
+/* Queue UBLK_U_IO_PREP_IO_CMDS for a specific queue with batch elements */
+int ublk_batch_queue_prep_io_cmds(struct ublk_thread *t, struct ublk_queue *q);
+/* Handle completion of batch I/O commands (prep/commit) */
+void ublk_batch_compl_cmd(struct ublk_thread *t,
+                         const struct io_uring_cqe *cqe);
 /* Initialize batch I/O state and calculate buffer parameters */
 void ublk_batch_prepare(struct ublk_thread *t);
 /* Allocate and register commit buffers for batch operations */