iommu/arm-smmu-v3: Directly encode CMDQ_OP_SYNC

Change the flow so the caller controls the CS field and remove the
weird u64p_replace_bits() thing to override it.

Reviewed-by: Pranjal Shrivastava <praan@google.com>
Reviewed-by: Mostafa Saleh <smostafa@google.com>
Tested-by: Pranjal Shrivastava <praan@google.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Nicolin Chen <nicolinc@nvidia.com>
Tested-by: Samiullah Khawaja <skhawaja@google.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index d30bac229d68f15560f73d0e4a8f5941ad48078e..5fbde147f77147d80d39b630541b9dd48937b460 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -308,16 +308,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out,
        case CMDQ_OP_TLBI_EL2_ASID:
                cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
                break;
-       case CMDQ_OP_CMD_SYNC:
-               if (ent->sync.msiaddr) {
-                       cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
-                       cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
-               } else {
-                       cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
-               }
-               cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
-               cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
-               break;
        default:
                return -ENOENT;
        }
@@ -350,23 +340,24 @@ static void arm_smmu_cmdq_build_sync_cmd(struct arm_smmu_cmd *cmd,
                                         struct arm_smmu_cmdq *cmdq, u32 prod)
 {
        struct arm_smmu_queue *q = &cmdq->q;
-       struct arm_smmu_cmdq_ent ent = {
-               .opcode = CMDQ_OP_CMD_SYNC,
-       };
+       u64 msiaddr = 0;
+       unsigned int cs;
 
        /*
         * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
         * payload, so the write will zero the entire command on that platform.
         */
-       if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
-               ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
-                                  q->ent_dwords * 8;
+       if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) {
+               cs = CMDQ_SYNC_0_CS_NONE;
+       } else if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
+               cs = CMDQ_SYNC_0_CS_IRQ;
+               msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
+                         q->ent_dwords * 8;
+       } else {
+               cs = CMDQ_SYNC_0_CS_SEV;
        }
 
-       arm_smmu_cmdq_build_cmd(cmd, &ent);
-       if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
-               u64p_replace_bits(&cmd->data[0], CMDQ_SYNC_0_CS_NONE,
-                                 CMDQ_SYNC_0_CS);
+       *cmd = arm_smmu_make_cmd_sync(cs, msiaddr);
 }
 
 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
@@ -383,9 +374,6 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
        struct arm_smmu_cmd cmd;
        u32 cons = readl_relaxed(q->cons_reg);
        u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
-       struct arm_smmu_cmdq_ent cmd_sync = {
-               .opcode = CMDQ_OP_CMD_SYNC,
-       };
 
        dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
                idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
@@ -419,10 +407,10 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
                dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd.data[i]);
 
        /* Convert the erroneous command into a CMD_SYNC */
-       arm_smmu_cmdq_build_cmd(&cmd, &cmd_sync);
-       if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
-               u64p_replace_bits(&cmd.data[0], CMDQ_SYNC_0_CS_NONE,
-                                 CMDQ_SYNC_0_CS);
+       cmd = arm_smmu_make_cmd_sync(
+               arm_smmu_cmdq_needs_busy_polling(smmu, cmdq) ?
+                       CMDQ_SYNC_0_CS_NONE : CMDQ_SYNC_0_CS_SEV,
+               0);
 
        queue_write(Q_ENT(q, cons), cmd.data, q->ent_dwords);
 }

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 194f73cabef5c9165bde5848e945f9e07595479b..538380de7d48a07daed6ae2f95885e4f05e1904b 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -571,6 +571,18 @@ static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv_all(u32 sid,
        return arm_smmu_make_cmd_atc_inv(sid, ssid, 0, ATC_INV_SIZE_ALL);
 }
 
+static inline struct arm_smmu_cmd arm_smmu_make_cmd_sync(unsigned int cs,
+                                                        u64 msiaddr)
+{
+       struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CMD_SYNC);
+
+       cmd.data[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, cs) |
+                      FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH) |
+                      FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
+       cmd.data[1] |= msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
+       return cmd;
+}
+
 /* Event queue */
 #define EVTQ_ENT_SZ_SHIFT              5
 #define EVTQ_ENT_DWORDS                        ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
@@ -648,10 +660,6 @@ struct arm_smmu_cmdq_ent {
                        u8                      tg;
                        u64                     addr;
                } tlbi;
-
-               struct {
-                       u64                     msiaddr;
-               } sync;
        };
 };