scsi: target: Add support for completing commands from backend context

To complete a command several drivers just drop their reference and add
it to list to be processed by a driver specific thread. So there's no
need to go from backend context to the LIO thread then to the driver's
thread. When avoiding the LIO thread, IOPS can increase from 20-30% for
workloads like:

  fio --filename=/dev/sdb  --direct=1 --rw=randrw --bs=8K \
    --ioengine=libaio --iodepth=128  --numjobs=$jobs

where increasing jobs increases the performance improvement (this is
using NVMe drives with LIO's submit_type=1 to directly submit).

Add the infrastructure so drivers and userspace can control how to
complete a command like is done for the submission path. In this commit
there is no behavior change and we continue to defer to the LIO
workqueue thread. In the subsequent commits we will allow drivers to
report what they support and allow userspace to control the behavior.

Signed-off-by: Mike Christie <michael.christie@oracle.com>
Link: https://patch.msgid.link/20260222232946.7637-2-michael.christie@oracle.com
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index 74c6383f9eede7cdfaa2d3968bd36aa6d0cb4342..883a866e96ab242c95ca7c022da3f747f7c897df 100644 (file)
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -813,6 +813,7 @@ struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
                                DA_UNMAP_ZEROES_DATA_DEFAULT;
        dev->dev_attrib.max_write_same_len = DA_MAX_WRITE_SAME_LEN;
        dev->dev_attrib.submit_type = TARGET_FABRIC_DEFAULT_SUBMIT;
+       dev->dev_attrib.submit_type = TARGET_QUEUE_COMPL;
 
        /* Skip allocating lun_stats since we can't export them. */
        xcopy_lun = &dev->xcopy_lun;

diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index a7330c4fedde343e48bde4867aecc5ecbcbd082c..34249fb80c678cae540881eebeffa4f1e0f72f4b 100644 (file)
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -902,13 +902,59 @@ static bool target_cmd_interrupted(struct se_cmd *cmd)
        return false;
 }
 
+static void target_complete(struct se_cmd *cmd, int success)
+{
+       struct se_wwn *wwn = cmd->se_sess->se_tpg->se_tpg_wwn;
+       struct se_dev_attrib *da;
+       u8 compl_type;
+       int cpu;
+
+       if (!wwn) {
+               cpu = cmd->cpuid;
+               goto queue_work;
+       }
+
+       da = &cmd->se_dev->dev_attrib;
+       if (da->complete_type == TARGET_FABRIC_DEFAULT_COMPL)
+               compl_type = wwn->wwn_tf->tf_ops->default_compl_type;
+       else if (da->complete_type == TARGET_DIRECT_SUBMIT &&
+                wwn->wwn_tf->tf_ops->direct_compl_supp)
+               compl_type = TARGET_DIRECT_COMPL;
+       else
+               compl_type = TARGET_QUEUE_COMPL;
+
+       if (compl_type == TARGET_DIRECT_COMPL) {
+               /*
+                * Failure handling and processing secondary stages of
+                * complex commands can be too heavy to handle from the
+                * fabric driver so always defer.
+                */
+               if (success && !cmd->transport_complete_callback) {
+                       target_complete_ok_work(&cmd->work);
+                       return;
+               }
+
+               compl_type = TARGET_QUEUE_COMPL;
+       }
+
+queue_work:
+       INIT_WORK(&cmd->work, success ? target_complete_ok_work :
+                 target_complete_failure_work);
+
+       if (!wwn || wwn->cmd_compl_affinity == SE_COMPL_AFFINITY_CPUID)
+               cpu = cmd->cpuid;
+       else
+               cpu = wwn->cmd_compl_affinity;
+
+       queue_work_on(cpu, target_completion_wq, &cmd->work);
+}
+
 /* May be called from interrupt context so must not sleep. */
 void target_complete_cmd_with_sense(struct se_cmd *cmd, u8 scsi_status,
                                    sense_reason_t sense_reason)
 {
-       struct se_wwn *wwn = cmd->se_sess->se_tpg->se_tpg_wwn;
-       int success, cpu;
        unsigned long flags;
+       int success;
 
        if (target_cmd_interrupted(cmd))
                return;
@@ -933,15 +979,7 @@ void target_complete_cmd_with_sense(struct se_cmd *cmd, u8 scsi_status,
        cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
        spin_unlock_irqrestore(&cmd->t_state_lock, flags);
 
-       INIT_WORK(&cmd->work, success ? target_complete_ok_work :
-                 target_complete_failure_work);
-
-       if (!wwn || wwn->cmd_compl_affinity == SE_COMPL_AFFINITY_CPUID)
-               cpu = cmd->cpuid;
-       else
-               cpu = wwn->cmd_compl_affinity;
-
-       queue_work_on(cpu, target_completion_wq, &cmd->work);
+       target_complete(cmd, success);
 }
 EXPORT_SYMBOL(target_complete_cmd_with_sense);
 

diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index b62d5fcce950de2af67185d8472b49e663db0e3e..9a0e9f9e1ec4478d8d32c27318a0230ff2c1a9a5 100644 (file)
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -111,6 +111,15 @@
 /* Peripheral Device Text Identification Information */
 #define PD_TEXT_ID_INFO_LEN                    256
 
+enum target_compl_type {
+       /* Use the fabric driver's default completion type */
+       TARGET_FABRIC_DEFAULT_COMPL,
+       /* Complete from the backend calling context */
+       TARGET_DIRECT_COMPL,
+       /* Defer completion to the LIO workqueue */
+       TARGET_QUEUE_COMPL,
+};
+
 enum target_submit_type {
        /* Use the fabric driver's default submission type */
        TARGET_FABRIC_DEFAULT_SUBMIT,
@@ -741,6 +750,7 @@ struct se_dev_attrib {
        u32             atomic_granularity;
        u32             atomic_max_with_boundary;
        u32             atomic_max_boundary;
+       u8              complete_type;
        u8              submit_type;
        struct se_device *da_dev;
        struct config_group da_group;

diff --git a/include/target/target_core_fabric.h b/include/target/target_core_fabric.h
index 3378ff9ee271c942536f341678d5045b3c782116..e9039e73d058a7ebfed6016c1c979e5842dc2c8c 100644 (file)
--- a/include/target/target_core_fabric.h
+++ b/include/target/target_core_fabric.h
@@ -118,15 +118,21 @@ struct target_core_fabric_ops {
         * its entirety before a command is aborted.
         */
        unsigned int write_pending_must_be_called:1;
+       /*
+        * Set this if the driver does not require calling queue_data_in
+        * queue_status and check_stop_free from a worker thread when
+        * completing successful commands.
+        */
+       unsigned int direct_compl_supp:1;
        /*
         * Set this if the driver supports submitting commands to the backend
         * from target_submit/target_submit_cmd.
         */
        unsigned int direct_submit_supp:1;
-       /*
-        * Set this to a target_submit_type value.
-        */
+       /* Set this to a target_submit_type value. */
        u8 default_submit_type;
+       /* Set this to the target_compl_type value. */
+       u8 default_compl_type;
 };
 
 int target_register_template(const struct target_core_fabric_ops *fo);