scsi: ufs: core: Optimize the hot path

Set .cmd_size in the SCSI host template such that the SCSI core makes
struct scsi_cmnd and struct ufshcd_lrb adjacent. Convert the cmd->lrbp
and lrbp->cmd memory loads into pointer offset calculations. Remove the
data structure members that became superfluous, namely ufshcd_lrb.cmd
and ufs_hba.lrb. Since ufshcd_lrb.cmd is removed, this pointer cannot be
used anymore to test whether or not a command is a SCSI command.
Introduce a new function for this purpose, namely ufshcd_is_scsi_cmd().

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Link: https://patch.msgid.link/20251031204029.2883185-24-bvanassche@acm.org
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c
index 1de3360a7af1d2352e09f3b4fe96ee45c7d8ac5f..776ff0896a2a3ebd06a43965d6b2bf33a12e2c87 100644 (file)
--- a/drivers/ufs/core/ufs-mcq.c
+++ b/drivers/ufs/core/ufs-mcq.c
@@ -534,8 +534,8 @@ static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
  */
 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
 {
-       struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
-       struct scsi_cmnd *cmd = lrbp->cmd;
+       struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        struct ufs_hw_queue *hwq;
        void __iomem *reg, *opr_sqd_base;
        u32 nexus, id, val;
@@ -618,7 +618,8 @@ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
                                  struct ufs_hw_queue *hwq, int task_tag)
 {
-       struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+       struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        struct utp_transfer_req_desc *utrd;
        __le64  cmd_desc_base_addr;
        bool ret = false;
@@ -669,7 +670,7 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
        struct Scsi_Host *host = cmd->device->host;
        struct ufs_hba *hba = shost_priv(host);
        int tag = scsi_cmd_to_rq(cmd)->tag;
-       struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        struct ufs_hw_queue *hwq;
        int err;
 

diff --git a/drivers/ufs/core/ufshcd-crypto.h b/drivers/ufs/core/ufshcd-crypto.h
index 89bb97c14c15b6c14a5ff3a9bdeeb1ac2c81eeb7..c148a51943786f088d3bef9dee6a201122b6d4ed 100644 (file)
--- a/drivers/ufs/core/ufshcd-crypto.h
+++ b/drivers/ufs/core/ufshcd-crypto.h
@@ -38,10 +38,10 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
 }
 
 static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
-                                         struct ufshcd_lrb *lrbp)
+                                         struct scsi_cmnd *cmd)
 {
-       struct scsi_cmnd *cmd = lrbp->cmd;
        const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx;
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
 
        if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt)
                return hba->vops->fill_crypto_prdt(hba, crypt_ctx,
@@ -51,17 +51,19 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
 }
 
 static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
-                                           struct ufshcd_lrb *lrbp)
+                                           struct scsi_cmnd *cmd)
 {
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
        if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT))
                return;
 
-       if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx))
+       if (!(scsi_cmd_to_rq(cmd)->crypt_ctx))
                return;
 
        /* Zeroize the PRDT because it can contain cryptographic keys. */
        memzero_explicit(lrbp->ucd_prdt_ptr,
-                        ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd));
+                        ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd));
 }
 
 bool ufshcd_crypto_enable(struct ufs_hba *hba);
@@ -82,13 +84,15 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
                                   struct request_desc_header *h) { }
 
 static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
-                                         struct ufshcd_lrb *lrbp)
+                                         struct scsi_cmnd *cmd)
 {
        return 0;
 }
 
 static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
-                                           struct ufshcd_lrb *lrbp) { }
+                                           struct scsi_cmnd *cmd)
+{
+}
 
 static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)
 {

diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index 749c0ab2a4ca6b74242e3dfe812cce16091a3f5c..72d2766b19e30bfaf0ab5dc8d4b21f62cb9de59a 100644 (file)
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -77,8 +77,7 @@ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd);
 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);
 int ufshcd_mcq_abort(struct scsi_cmnd *cmd);
 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
-void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
-                            struct ufshcd_lrb *lrbp);
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd);
 
 #define SD_ASCII_STD true
 #define SD_RAW false
@@ -363,6 +362,44 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8
        return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported);
 }
 
+/*
+ * Convert a block layer tag into a SCSI command pointer. This function is
+ * called once per I/O completion path and is also called from error paths.
+ */
+static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag)
+{
+       struct blk_mq_tags *tags = hba->host->tag_set.shared_tags;
+       struct request *rq;
+
+       /*
+        * Handle reserved tags differently because the UFS driver does not
+        * call blk_mq_alloc_request() for allocating reserved requests.
+        * Allocating reserved tags with blk_mq_alloc_request() would require
+        * the following:
+        * - Allocate an additional request queue from &hba->host->tag_set for
+        *   allocating reserved requests from.
+        * - For that request queue, allocate a SCSI device.
+        * - Calling blk_mq_alloc_request(hba->dev_mgmt_queue, REQ_OP_DRV_OUT,
+        *   BLK_MQ_REQ_RESERVED) for allocating a reserved request and
+        *   blk_mq_free_request() for freeing reserved requests.
+        * - Set the .device pointer for these reserved requests.
+        * - Submit reserved requests with blk_execute_rq().
+        * - Modify ufshcd_queuecommand() such that it handles reserved requests
+        *   in another way than SCSI requests.
+        * - Modify ufshcd_compl_one_cqe() such that it calls scsi_done() for
+        *   device management commands.
+        * - Modify all callback functions called by blk_mq_tagset_busy_iter()
+        *   calls in the UFS driver and skip device management commands.
+        */
+       rq = tag < UFSHCD_NUM_RESERVED ? tags->static_rqs[tag] :
+                                        blk_mq_tag_to_rq(tags, tag);
+
+       if (WARN_ON_ONCE(!rq))
+               return NULL;
+
+       return blk_mq_rq_to_pdu(rq);
+}
+
 static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)
        __must_hold(&q->sq_lock)
 {

diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 1757aa0237da6bf3a2e9f552d9baca96b12bb793..ce657b2506fb0de89a72ef311136099a265fb2de 100644 (file)
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -28,6 +28,7 @@
 #include <scsi/scsi_dbg.h>
 #include <scsi/scsi_driver.h>
 #include <scsi/scsi_eh.h>
+#include <scsi/scsi_tcq.h>
 #include "ufshcd-priv.h"
 #include <ufs/ufs_quirks.h>
 #include <ufs/unipro.h>
@@ -483,7 +484,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd,
        u32 hwq_id = 0;
        struct request *rq = scsi_cmd_to_rq(cmd);
        unsigned int tag = rq->tag;
-       struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        int transfer_len = -1;
 
        /* trace UPIU also */
@@ -594,14 +595,13 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba)
        ufshcd_vops_dbg_register_dump(hba);
 }
 
-static
-void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
+static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+                           bool pr_prdt)
 {
-       const struct ufshcd_lrb *lrbp;
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+       const int tag = lrbp->task_tag;
        int prdt_length;
 
-       lrbp = &hba->lrb[tag];
-
        if (hba->monitor.enabled) {
                dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag,
                        div_u64(lrbp->issue_time_stamp_local_clock, 1000));
@@ -644,7 +644,7 @@ static bool ufshcd_print_tr_iter(struct request *req, void *priv)
        struct Scsi_Host *shost = sdev->host;
        struct ufs_hba *hba = shost_priv(shost);
 
-       ufshcd_print_tr(hba, req->tag, *(bool *)priv);
+       ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv);
 
        return true;
 }
@@ -2298,8 +2298,7 @@ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
                                                struct scsi_cmnd *cmd)
 {
        const struct ufs_hba_monitor *m = &hba->monitor;
-       struct request *rq = scsi_cmd_to_rq(cmd);
-       struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag];
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
 
        return m->enabled &&
               (!m->chunk_size || m->chunk_size == cmd->sdb.length) &&
@@ -2320,7 +2319,7 @@ static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 {
        struct request *req = scsi_cmd_to_rq(cmd);
-       struct ufshcd_lrb *lrbp = &hba->lrb[req->tag];
+       const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]);
        unsigned long flags;
 
@@ -2350,17 +2349,26 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
        spin_unlock_irqrestore(hba->host->host_lock, flags);
 }
 
+/*
+ * Returns %true for SCSI commands and %false for device management commands.
+ * Must not be called for SCSI commands that have not yet been started.
+ */
+static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd)
+{
+       return blk_mq_request_started(scsi_cmd_to_rq(cmd));
+}
+
 /**
  * ufshcd_send_command - Send SCSI or device management commands
  * @hba: per adapter instance
- * @lrbp: Local reference block of SCSI command
+ * @cmd: SCSI command or device management command pointer
  * @hwq: pointer to hardware queue instance
  */
 static inline void ufshcd_send_command(struct ufs_hba *hba,
-                                      struct ufshcd_lrb *lrbp,
+                                      struct scsi_cmnd *cmd,
                                       struct ufs_hw_queue *hwq)
 {
-       struct scsi_cmnd *cmd = lrbp->cmd;
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        unsigned long flags;
 
        if (hba->monitor.enabled) {
@@ -2369,7 +2377,7 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
                lrbp->compl_time_stamp = ktime_set(0, 0);
                lrbp->compl_time_stamp_local_clock = 0;
        }
-       if (cmd) {
+       if (ufshcd_is_scsi_cmd(cmd)) {
                ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND);
                ufshcd_clk_scaling_start_busy(hba);
                if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
@@ -2389,7 +2397,8 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
        } else {
                spin_lock_irqsave(&hba->outstanding_lock, flags);
                if (hba->vops && hba->vops->setup_xfer_req)
-                       hba->vops->setup_xfer_req(hba, lrbp->task_tag, !!cmd);
+                       hba->vops->setup_xfer_req(hba, lrbp->task_tag,
+                                                 ufshcd_is_scsi_cmd(cmd));
                __set_bit(lrbp->task_tag, &hba->outstanding_reqs);
                ufshcd_writel(hba, 1 << lrbp->task_tag,
                              REG_UTP_TRANSFER_REQ_DOOR_BELL);
@@ -2399,11 +2408,12 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
 
 /**
  * ufshcd_copy_sense_data - Copy sense data in case of check condition
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command
  */
-static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
+static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd)
 {
-       u8 *const sense_buffer = lrbp->cmd->sense_buffer;
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+       u8 *const sense_buffer = cmd->sense_buffer;
        u16 resp_len;
        int len;
 
@@ -2708,13 +2718,13 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int
 /**
  * ufshcd_map_sg - Map scatter-gather list to prdt
  * @hba: per adapter instance
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command
  *
  * Return: 0 in case of success, non-zero value in case of failure.
  */
-static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 {
-       struct scsi_cmnd *cmd = lrbp->cmd;
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        int sg_segments = scsi_dma_map(cmd);
 
        if (sg_segments < 0)
@@ -2722,7 +2732,7 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
 
        ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
 
-       return ufshcd_crypto_fill_prdt(hba, lrbp);
+       return ufshcd_crypto_fill_prdt(hba, cmd);
 }
 
 /**
@@ -2781,13 +2791,13 @@ ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
 /**
  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
  * for scsi commands
- * @lrbp: local reference block pointer
+ * @cmd: SCSI command
  * @upiu_flags: flags
  */
-static
-void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
+static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd,
+                                            u8 upiu_flags)
 {
-       struct scsi_cmnd *cmd = lrbp->cmd;
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
        unsigned short cdb_len;
 
@@ -2890,22 +2900,25 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
  *                        for SCSI Purposes
  * @hba: per adapter instance
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command
  */
-static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 {
-       struct request *rq = scsi_cmd_to_rq(lrbp->cmd);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+       struct request *rq = scsi_cmd_to_rq(cmd);
        unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
        u8 upiu_flags;
 
-       ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0);
+       ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags,
+                                   cmd->sc_data_direction, 0);
        if (ioprio_class == IOPRIO_CLASS_RT)
                upiu_flags |= UPIU_CMD_FLAGS_CP;
-       ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
+       ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags);
 }
 
-static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
+static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 {
+       const int i = scsi_cmd_to_rq(cmd)->tag;
        struct utp_transfer_cmd_desc *cmd_descp =
                (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba);
        struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
@@ -2913,6 +2926,7 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
                hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba);
        u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
        u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
+       struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd);
 
        lrb->utr_descriptor_ptr = utrdlp + i;
        lrb->utrd_dma_addr =
@@ -2925,27 +2939,31 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
        lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
 }
 
-static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
-                              struct scsi_cmnd *cmd, u8 lun, int tag)
+static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+                              u8 lun, int tag)
 {
-       ufshcd_init_lrb(hba, lrbp, tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+       ufshcd_init_lrb(hba, cmd);
 
        memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr));
 
-       lrbp->cmd = cmd;
        lrbp->task_tag = tag;
        lrbp->lun = lun;
-       ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp);
+       ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ?
+                                  scsi_cmd_to_rq(cmd) : NULL, lrbp);
 }
 
-static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
-                                 struct scsi_cmnd *cmd, u8 lun, int tag)
+static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+                                 u8 lun, int tag)
 {
-       __ufshcd_setup_cmd(hba, lrbp, cmd, lun, tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+       __ufshcd_setup_cmd(hba, cmd, lun, tag);
        lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
        lrbp->req_abort_skip = false;
 
-       ufshcd_comp_scsi_upiu(hba, lrbp);
+       ufshcd_comp_scsi_upiu(hba, cmd);
 }
 
 /**
@@ -3016,7 +3034,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 {
        struct ufs_hba *hba = shost_priv(host);
        int tag = scsi_cmd_to_rq(cmd)->tag;
-       struct ufshcd_lrb *lrbp;
        int err = 0;
        struct ufs_hw_queue *hwq = NULL;
 
@@ -3067,11 +3084,10 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 
        ufshcd_hold(hba);
 
-       lrbp = &hba->lrb[tag];
-
-       ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
+       ufshcd_setup_scsi_cmd(hba, cmd,
+                             ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
 
-       err = ufshcd_map_sg(hba, lrbp);
+       err = ufshcd_map_sg(hba, cmd);
        if (err) {
                ufshcd_release(hba);
                goto out;
@@ -3080,7 +3096,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
        if (hba->mcq_enabled)
                hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
 
-       ufshcd_send_command(hba, lrbp, hwq);
+       ufshcd_send_command(hba, cmd, hwq);
 
 out:
        if (ufs_trigger_eh(hba)) {
@@ -3094,10 +3110,12 @@ out:
        return err;
 }
 
-static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
-                            enum dev_cmd_type cmd_type, u8 lun, int tag)
+static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+                                enum dev_cmd_type cmd_type, u8 lun, int tag)
 {
-       __ufshcd_setup_cmd(hba, lrbp, NULL, lun, tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+       __ufshcd_setup_cmd(hba, cmd, lun, tag);
        lrbp->intr_cmd = true; /* No interrupt aggregation */
        hba->dev_cmd.type = cmd_type;
 }
@@ -3105,10 +3123,12 @@ static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
 /*
  * Return: 0 upon success; < 0 upon failure.
  */
-static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
-               struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
+static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+                                 enum dev_cmd_type cmd_type, int tag)
 {
-       ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+       ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
 
        return ufshcd_compose_devman_upiu(hba, lrbp);
 }
@@ -3320,13 +3340,14 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba)
  * Return: 0 upon success; > 0 in case the UFS device reported an OCS error;
  * < 0 if another error occurred.
  */
-static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
-                         const u32 tag, int timeout)
+static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+                               const u32 tag, int timeout)
 {
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        int err;
 
        ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
-       ufshcd_send_command(hba, lrbp, hba->dev_cmd_queue);
+       ufshcd_send_command(hba, cmd, hba->dev_cmd_queue);
        err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
 
        ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
@@ -3351,17 +3372,17 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
                enum dev_cmd_type cmd_type, int timeout)
 {
        const u32 tag = hba->reserved_slot;
-       struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+       struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
        int err;
 
        /* Protects use of hba->reserved_slot. */
        lockdep_assert_held(&hba->dev_cmd.lock);
 
-       err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
+       err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag);
        if (unlikely(err))
                return err;
 
-       return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout);
+       return ufshcd_issue_dev_cmd(hba, cmd, tag, timeout);
 }
 
 /**
@@ -3991,14 +4012,6 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
        }
 
 skip_utmrdl:
-       /* Allocate memory for local reference block */
-       hba->lrb = devm_kcalloc(hba->dev,
-                               hba->nutrs, sizeof(struct ufshcd_lrb),
-                               GFP_KERNEL);
-       if (!hba->lrb) {
-               dev_err(hba->dev, "LRB Memory allocation failed\n");
-               goto out;
-       }
        return 0;
 out:
        return -ENOMEM;
@@ -5411,19 +5424,18 @@ static void ufshcd_sdev_destroy(struct scsi_device *sdev)
 
 /**
  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
- * @lrbp: pointer to local reference block of completed command
+ * @cmd: SCSI command
  * @scsi_status: SCSI command status
  *
  * Return: value base on SCSI command status.
  */
-static inline int
-ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
+static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status)
 {
        int result = 0;
 
        switch (scsi_status) {
        case SAM_STAT_CHECK_CONDITION:
-               ufshcd_copy_sense_data(lrbp);
+               ufshcd_copy_sense_data(cmd);
                fallthrough;
        case SAM_STAT_GOOD:
                result |= DID_OK << 16 | scsi_status;
@@ -5431,7 +5443,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
        case SAM_STAT_TASK_SET_FULL:
        case SAM_STAT_BUSY:
        case SAM_STAT_TASK_ABORTED:
-               ufshcd_copy_sense_data(lrbp);
+               ufshcd_copy_sense_data(cmd);
                result |= scsi_status;
                break;
        default:
@@ -5445,15 +5457,16 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
 /**
  * ufshcd_transfer_rsp_status - Get overall status of the response
  * @hba: per adapter instance
- * @lrbp: pointer to local reference block of completed command
+ * @cmd: SCSI command
  * @cqe: pointer to the completion queue entry
  *
  * Return: result of the command to notify SCSI midlayer.
  */
-static inline int
-ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
-                          struct cq_entry *cqe)
+static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba,
+                                            struct scsi_cmnd *cmd,
+                                            struct cq_entry *cqe)
 {
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        int result = 0;
        int scsi_status;
        enum utp_ocs ocs;
@@ -5467,7 +5480,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
         * not set either flag.
         */
        if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
-               scsi_set_resid(lrbp->cmd, resid);
+               scsi_set_resid(cmd, resid);
 
        /* overall command status of utrd */
        ocs = ufshcd_get_tr_ocs(lrbp, cqe);
@@ -5488,7 +5501,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
                         * to notify the SCSI midlayer of the command status
                         */
                        scsi_status = lrbp->ucd_rsp_ptr->header.status;
-                       result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
+                       result = ufshcd_scsi_cmd_status(cmd, scsi_status);
 
                        /*
                         * Currently we are only supporting BKOPs exception
@@ -5553,7 +5566,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
            (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) {
                if (cqe)
                        ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry));
-               ufshcd_print_tr(hba, lrbp->task_tag, true);
+               ufshcd_print_tr(hba, cmd, true);
        }
        return result;
 }
@@ -5620,13 +5633,10 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
 }
 
 /* Release the resources allocated for processing a SCSI command. */
-void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
-                            struct ufshcd_lrb *lrbp)
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 {
-       struct scsi_cmnd *cmd = lrbp->cmd;
-
        scsi_dma_unmap(cmd);
-       ufshcd_crypto_clear_prdt(hba, lrbp);
+       ufshcd_crypto_clear_prdt(hba, cmd);
        ufshcd_release(hba);
        ufshcd_clk_scaling_update_busy(hba);
 }
@@ -5640,20 +5650,20 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
                          struct cq_entry *cqe)
 {
-       struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
-       struct scsi_cmnd *cmd = lrbp->cmd;
+       struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        enum utp_ocs ocs;
 
        if (hba->monitor.enabled) {
                lrbp->compl_time_stamp = ktime_get();
                lrbp->compl_time_stamp_local_clock = local_clock();
        }
-       if (cmd) {
+       if (ufshcd_is_scsi_cmd(cmd)) {
                if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
                        ufshcd_update_monitor(hba, cmd);
                ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP);
-               cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
-               ufshcd_release_scsi_cmd(hba, lrbp);
+               cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe);
+               ufshcd_release_scsi_cmd(hba, cmd);
                /* Do not touch lrbp after scsi done */
                scsi_done(cmd);
        } else {
@@ -5690,7 +5700,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba,
        int tag;
 
        for_each_set_bit(tag, completed_reqs, hba->nutrs) {
-               struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
+               struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag);
 
                if (!cmd)
                        continue;
@@ -5741,7 +5751,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
        struct scsi_device *sdev = rq->q->queuedata;
        struct Scsi_Host *shost = sdev->host;
        struct ufs_hba *hba = shost_priv(shost);
-       struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag];
        struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
 
        if (!hwq)
@@ -5756,7 +5765,7 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
        scoped_guard(spinlock_irqsave, &hwq->cq_lock) {
                if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
                        set_host_byte(cmd, DID_REQUEUE);
-                       ufshcd_release_scsi_cmd(hba, lrbp);
+                       ufshcd_release_scsi_cmd(hba, cmd);
                        scsi_done(cmd);
                }
        }
@@ -6641,7 +6650,7 @@ static bool ufshcd_abort_one(struct request *rq, void *priv)
 
        *ret = ufshcd_try_to_abort_task(hba, tag);
        dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
-               hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
+               ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1,
                *ret ? "failed" : "succeeded");
 
        return *ret == 0;
@@ -7371,14 +7380,15 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
                                        enum query_opcode desc_op)
 {
        const u32 tag = hba->reserved_slot;
-       struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+       struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        int err = 0;
        u8 upiu_flags;
 
        /* Protects use of hba->reserved_slot. */
        lockdep_assert_held(&hba->dev_cmd.lock);
 
-       ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
+       ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
 
        ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
 
@@ -7403,7 +7413,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
         * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
         * read the response directly ignoring all errors.
         */
-       ufshcd_issue_dev_cmd(hba, lrbp, tag, dev_cmd_timeout);
+       ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout);
 
        /* just copy the upiu response as it is */
        memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
@@ -7518,7 +7528,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
                         enum dma_data_direction dir)
 {
        const u32 tag = hba->reserved_slot;
-       struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+       struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        int err = 0;
        int result;
        u8 upiu_flags;
@@ -7529,7 +7540,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
        /* Protects use of hba->reserved_slot. */
        ufshcd_dev_man_lock(hba);
 
-       ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag);
+       ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN,
+                            tag);
 
        ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs);
 
@@ -7546,7 +7558,7 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
 
        memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
 
-       err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT);
+       err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT);
 
        if (!err) {
                /* Just copy the upiu response as it is */
@@ -7647,11 +7659,12 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
 
 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
 {
-       struct ufshcd_lrb *lrbp;
        int tag;
 
        for_each_set_bit(tag, &bitmap, hba->nutrs) {
-               lrbp = &hba->lrb[tag];
+               struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+               struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
                lrbp->req_abort_skip = true;
        }
 }
@@ -7659,7 +7672,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
 /**
  * ufshcd_try_to_abort_task - abort a specific task
  * @hba: Pointer to adapter instance
- * @tag: Task tag/index to be aborted
+ * @tag: Tag of the task to be aborted
  *
  * Abort the pending command in device by sending UFS_ABORT_TASK task management
  * command, and in host controller by clearing the door-bell register. There can
@@ -7671,7 +7684,8 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
  */
 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
 {
-       struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+       struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        int err;
        int poll_cnt;
        u8 resp = 0xF;
@@ -7693,7 +7707,7 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
                                hba->dev,
                                "%s: cmd with tag %d not pending in the device.\n",
                                __func__, tag);
-                       if (!ufshcd_cmd_inflight(lrbp->cmd)) {
+                       if (!ufshcd_cmd_inflight(cmd)) {
                                dev_info(hba->dev,
                                         "%s: cmd with tag=%d completed.\n",
                                         __func__, tag);
@@ -7741,7 +7755,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
        struct Scsi_Host *host = cmd->device->host;
        struct ufs_hba *hba = shost_priv(host);
        int tag = scsi_cmd_to_rq(cmd)->tag;
-       struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+       struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
        unsigned long flags;
        int err = FAILED;
        bool outstanding;
@@ -7776,9 +7790,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
                ufshcd_print_evt_hist(hba);
                ufshcd_print_host_state(hba);
                ufshcd_print_pwr_info(hba);
-               ufshcd_print_tr(hba, tag, true);
+               ufshcd_print_tr(hba, cmd, true);
        } else {
-               ufshcd_print_tr(hba, tag, false);
+               ufshcd_print_tr(hba, cmd, false);
        }
        hba->req_abort_count++;
 
@@ -7822,7 +7836,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
                goto release;
        }
 
-       err = ufshcd_try_to_abort_task(hba, tag);
+       err = ufshcd_try_to_abort_task(hba, lrbp->task_tag);
        if (err) {
                dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
                ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
@@ -7839,7 +7853,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
        spin_unlock_irqrestore(&hba->outstanding_lock, flags);
 
        if (outstanding)
-               ufshcd_release_scsi_cmd(hba, lrbp);
+               ufshcd_release_scsi_cmd(hba, cmd);
 
        err = SUCCESS;
 
@@ -8919,8 +8933,6 @@ static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs)
        utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs;
        dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
                           hba->utrdl_dma_addr);
-
-       devm_kfree(hba->dev, hba->lrb);
 }
 
 static int ufshcd_alloc_mcq(struct ufs_hba *hba)
@@ -9191,6 +9203,7 @@ static const struct scsi_host_template ufshcd_driver_template = {
        .name                   = UFSHCD,
        .proc_name              = UFSHCD,
        .map_queues             = ufshcd_map_queues,
+       .cmd_size               = sizeof(struct ufshcd_lrb),
        .init_cmd_priv          = ufshcd_init_cmd_priv,
        .queuecommand           = ufshcd_queuecommand,
        .nr_reserved_cmds       = UFSHCD_NUM_RESERVED,

diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h
index 00152e135fc9539b36a49cbcc94bd313422b28d9..fbed47b6c61f33cf276d9229bd05f610bbfce831 100644 (file)
--- a/include/ufs/ufshcd.h
+++ b/include/ufs/ufshcd.h
@@ -161,7 +161,6 @@ struct ufs_pm_lvl_states {
  * @ucd_prdt_dma_addr: PRDT dma address for debug
  * @ucd_rsp_dma_addr: UPIU response dma address for debug
  * @ucd_req_dma_addr: UPIU request dma address for debug
- * @cmd: pointer to SCSI command
  * @scsi_status: SCSI status of the command
  * @command_type: SCSI, UFS, Query.
  * @task_tag: Task tag of the command
@@ -186,7 +185,6 @@ struct ufshcd_lrb {
        dma_addr_t ucd_rsp_dma_addr;
        dma_addr_t ucd_prdt_dma_addr;
 
-       struct scsi_cmnd *cmd;
        int scsi_status;
 
        int command_type;
@@ -833,7 +831,6 @@ enum ufshcd_mcq_opr {
  * @spm_lvl: desired UFS power management level during system PM.
  * @pm_op_in_progress: whether or not a PM operation is in progress.
  * @ahit: value of Auto-Hibernate Idle Timer register.
- * @lrb: local reference block
  * @outstanding_tasks: Bits representing outstanding task requests
  * @outstanding_lock: Protects @outstanding_reqs.
  * @outstanding_reqs: Bits representing outstanding transfer requests
@@ -976,8 +973,6 @@ struct ufs_hba {
        /* Auto-Hibernate Idle Timer register value */
        u32 ahit;
 
-       struct ufshcd_lrb *lrb;
-
        unsigned long outstanding_tasks;
        spinlock_t outstanding_lock;
        unsigned long outstanding_reqs;