iommu/arm-smmu-v3: Convert arm_smmu_cmdq_batch cmds to struct arm_smmu_cmd

Convert the batch's type to also get the remaining helper functions to
use the new type and complete replacing naked u64s with the new struct.

The low-level queue_write()/queue_read()/queue_remove_raw() functions
remain u64-based since they are shared by event and PRI queues which
have different entry sizes.

Reviewed-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Pranjal Shrivastava <praan@google.com>
Tested-by: Pranjal Shrivastava <praan@google.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Nicolin Chen <nicolinc@nvidia.com>
Tested-by: Samiullah Khawaja <skhawaja@google.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
index ddae0b07c76b505fa153ffbd995e027c0f908abc..1e9f7d2de34414e152d043ea015c2680d3bf81ea 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
@@ -300,7 +300,7 @@ unlock:
 /* This is basically iommu_viommu_arm_smmuv3_invalidate in u64 for conversion */
 struct arm_vsmmu_invalidation_cmd {
        union {
-               u64 cmd[2];
+               struct arm_smmu_cmd cmd;
                struct iommu_viommu_arm_smmuv3_invalidate ucmd;
        };
 };
@@ -316,32 +316,32 @@ static int arm_vsmmu_convert_user_cmd(struct arm_vsmmu *vsmmu,
                                      struct arm_vsmmu_invalidation_cmd *cmd)
 {
        /* Commands are le64 stored in u64 */
-       cmd->cmd[0] = le64_to_cpu(cmd->ucmd.cmd[0]);
-       cmd->cmd[1] = le64_to_cpu(cmd->ucmd.cmd[1]);
+       cmd->cmd.data[0] = le64_to_cpu(cmd->ucmd.cmd[0]);
+       cmd->cmd.data[1] = le64_to_cpu(cmd->ucmd.cmd[1]);
 
-       switch (cmd->cmd[0] & CMDQ_0_OP) {
+       switch (cmd->cmd.data[0] & CMDQ_0_OP) {
        case CMDQ_OP_TLBI_NSNH_ALL:
                /* Convert to NH_ALL */
-               cmd->cmd[0] = CMDQ_OP_TLBI_NH_ALL |
+               cmd->cmd.data[0] = CMDQ_OP_TLBI_NH_ALL |
                              FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
-               cmd->cmd[1] = 0;
+               cmd->cmd.data[1] = 0;
                break;
        case CMDQ_OP_TLBI_NH_VA:
        case CMDQ_OP_TLBI_NH_VAA:
        case CMDQ_OP_TLBI_NH_ALL:
        case CMDQ_OP_TLBI_NH_ASID:
-               cmd->cmd[0] &= ~CMDQ_TLBI_0_VMID;
-               cmd->cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
+               cmd->cmd.data[0] &= ~CMDQ_TLBI_0_VMID;
+               cmd->cmd.data[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
                break;
        case CMDQ_OP_ATC_INV:
        case CMDQ_OP_CFGI_CD:
        case CMDQ_OP_CFGI_CD_ALL: {
-               u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, cmd->cmd[0]);
+               u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, cmd->cmd.data[0]);
 
                if (arm_vsmmu_vsid_to_sid(vsmmu, vsid, &sid))
                        return -EIO;
-               cmd->cmd[0] &= ~CMDQ_CFGI_0_SID;
-               cmd->cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid);
+               cmd->cmd.data[0] &= ~CMDQ_CFGI_0_SID;
+               cmd->cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid);
                break;
        }
        default:
@@ -386,7 +386,7 @@ int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu,
                        continue;
 
                /* FIXME always uses the main cmdq rather than trying to group by type */
-               ret = arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, last->cmd,
+               ret = arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &last->cmd,
                                                  cur - last, true);
                if (ret) {
                        cur--;

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index fbf56c64da7f176822a8b87385a28de8fcbcce7d..a3ce3c11c4d1194bca5834119a08e135ca6bc9db 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -268,9 +268,12 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
 }
 
 /* High-level queue accessors */
-static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
+static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out,
+                                  struct arm_smmu_cmdq_ent *ent)
 {
-       memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
+       u64 *cmd = cmd_out->data;
+
+       memset(cmd_out, 0, sizeof(*cmd_out));
        cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
 
        switch (ent->opcode) {
@@ -390,7 +393,8 @@ static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu,
        return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV;
 }
 
-static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
+static void arm_smmu_cmdq_build_sync_cmd(struct arm_smmu_cmd *cmd,
+                                        struct arm_smmu_device *smmu,
                                         struct arm_smmu_cmdq *cmdq, u32 prod)
 {
        struct arm_smmu_queue *q = &cmdq->q;
@@ -409,7 +413,8 @@ static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
 
        arm_smmu_cmdq_build_cmd(cmd, &ent);
        if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
-               u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
+               u64p_replace_bits(&cmd->data[0], CMDQ_SYNC_0_CS_NONE,
+                                 CMDQ_SYNC_0_CS);
 }
 
 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
@@ -422,9 +427,8 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
                [CMDQ_ERR_CERROR_ATC_INV_IDX]   = "ATC invalidate timeout",
        };
        struct arm_smmu_queue *q = &cmdq->q;
-
        int i;
-       u64 cmd[CMDQ_ENT_DWORDS];
+       struct arm_smmu_cmd cmd;
        u32 cons = readl_relaxed(q->cons_reg);
        u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
        struct arm_smmu_cmdq_ent cmd_sync = {
@@ -457,17 +461,18 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
         * We may have concurrent producers, so we need to be careful
         * not to touch any of the shadow cmdq state.
         */
-       queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
+       queue_read(cmd.data, Q_ENT(q, cons), q->ent_dwords);
        dev_err(smmu->dev, "skipping command in error state:\n");
-       for (i = 0; i < ARRAY_SIZE(cmd); ++i)
-               dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
+       for (i = 0; i < ARRAY_SIZE(cmd.data); ++i)
+               dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd.data[i]);
 
        /* Convert the erroneous command into a CMD_SYNC */
-       arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
+       arm_smmu_cmdq_build_cmd(&cmd, &cmd_sync);
        if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
-               u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
+               u64p_replace_bits(&cmd.data[0], CMDQ_SYNC_0_CS_NONE,
+                                 CMDQ_SYNC_0_CS);
 
-       queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
+       queue_write(Q_ENT(q, cons), cmd.data, q->ent_dwords);
 }
 
 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
@@ -767,7 +772,8 @@ static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
        return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
 }
 
-static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
+static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq,
+                                       struct arm_smmu_cmd *cmds,
                                        u32 prod, int n)
 {
        int i;
@@ -777,10 +783,9 @@ static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
        };
 
        for (i = 0; i < n; ++i) {
-               u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
-
                prod = queue_inc_prod_n(&llq, i);
-               queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
+               queue_write(Q_ENT(&cmdq->q, prod), cmds[i].data,
+                           ARRAY_SIZE(cmds[i].data));
        }
 }
 
@@ -801,10 +806,11 @@ static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
  *   CPU will appear before any of the commands from the other CPU.
  */
 int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
-                               struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
+                               struct arm_smmu_cmdq *cmdq,
+                               struct arm_smmu_cmd *cmds, int n,
                                bool sync)
 {
-       u64 cmd_sync[CMDQ_ENT_DWORDS];
+       struct arm_smmu_cmd cmd_sync;
        u32 prod;
        unsigned long flags;
        bool owner;
@@ -847,8 +853,9 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
        arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
        if (sync) {
                prod = queue_inc_prod_n(&llq, n);
-               arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod);
-               queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
+               arm_smmu_cmdq_build_sync_cmd(&cmd_sync, smmu, cmdq, prod);
+               queue_write(Q_ENT(&cmdq->q, prod), cmd_sync.data,
+                           ARRAY_SIZE(cmd_sync.data));
 
                /*
                 * In order to determine completion of our CMD_SYNC, we must
@@ -925,7 +932,7 @@ static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
                                     bool sync)
 {
        return arm_smmu_cmdq_issue_cmdlist(
-               smmu, arm_smmu_get_cmdq(smmu, cmd), cmd->data, 1, sync);
+               smmu, arm_smmu_get_cmdq(smmu, cmd), cmd, 1, sync);
 }
 
 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
@@ -954,7 +961,7 @@ static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
 {
        struct arm_smmu_cmd cmd;
 
-       arm_smmu_cmdq_build_cmd(cmd.data, ent);
+       arm_smmu_cmdq_build_cmd(&cmd, ent);
        arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd);
 }
 
@@ -966,9 +973,8 @@ static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
                          (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
        struct arm_smmu_cmd cmd;
        bool unsupported_cmd;
-       int index;
 
-       if (unlikely(arm_smmu_cmdq_build_cmd(cmd.data, ent))) {
+       if (unlikely(arm_smmu_cmdq_build_cmd(&cmd, ent))) {
                dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
                         ent->opcode);
                return;
@@ -987,9 +993,7 @@ static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
                arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd);
        }
 
-       index = cmds->num * CMDQ_ENT_DWORDS;
-       memcpy(&cmds->cmds[index], cmd.data, sizeof(cmd.data));
-       cmds->num++;
+       cmds->cmds[cmds->num++] = cmd;
 }
 
 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
@@ -1025,7 +1029,7 @@ static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused
                break;
        }
 
-       arm_smmu_cmdq_build_cmd(hw_cmd.data, &cmd);
+       arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd);
        arm_smmu_cmdq_issue_cmd(master->smmu, &hw_cmd);
 
        /*
@@ -1865,7 +1869,7 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
        };
        struct arm_smmu_cmd cmd;
 
-       arm_smmu_cmdq_build_cmd(cmd.data, &ent);
+       arm_smmu_cmdq_build_cmd(&cmd, &ent);
        arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
 }
 
@@ -1899,7 +1903,7 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
                                         } };
                struct arm_smmu_cmd prefetch_cmd;
 
-               arm_smmu_cmdq_build_cmd(prefetch_cmd.data, &prefetch_ent);
+               arm_smmu_cmdq_build_cmd(&prefetch_cmd, &prefetch_ent);
                arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
        }
 }
@@ -2339,7 +2343,7 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
                };
                struct arm_smmu_cmd cmd;
 
-               arm_smmu_cmdq_build_cmd(cmd.data, &ent);
+               arm_smmu_cmdq_build_cmd(&cmd, &ent);
                arm_smmu_cmdq_issue_cmd(smmu, &cmd);
        }
 }
@@ -3462,7 +3466,7 @@ static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv)
        }
 
        cmd.opcode = inv->nsize_opcode;
-       arm_smmu_cmdq_build_cmd(hw_cmd.data, &cmd);
+       arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd);
        arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &hw_cmd);
 }
 
@@ -4875,18 +4879,18 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
 
        /* Invalidate any cached configuration */
        ent.opcode = CMDQ_OP_CFGI_ALL;
-       arm_smmu_cmdq_build_cmd(cmd.data, &ent);
+       arm_smmu_cmdq_build_cmd(&cmd, &ent);
        arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 
        /* Invalidate any stale TLB entries */
        if (smmu->features & ARM_SMMU_FEAT_HYP) {
                ent.opcode = CMDQ_OP_TLBI_EL2_ALL;
-               arm_smmu_cmdq_build_cmd(cmd.data, &ent);
+               arm_smmu_cmdq_build_cmd(&cmd, &ent);
                arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
        }
 
        ent.opcode = CMDQ_OP_TLBI_NSNH_ALL;
-       arm_smmu_cmdq_build_cmd(cmd.data, &ent);
+       arm_smmu_cmdq_build_cmd(&cmd, &ent);
        arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 
        /* Event queue */

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 6d73f6b63e64a9bd9d3cc0894a34f2f51ac70aaf..1fe6917448b7746323ca9ae8fb89084859a34fe3 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -651,7 +651,7 @@ static inline bool arm_smmu_cmdq_supports_cmd(struct arm_smmu_cmdq *cmdq,
 }
 
 struct arm_smmu_cmdq_batch {
-       u64                             cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
+       struct arm_smmu_cmd             cmds[CMDQ_BATCH_ENTRIES];
        struct arm_smmu_cmdq            *cmdq;
        int                             num;
 };
@@ -1148,7 +1148,8 @@ void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
                                  const struct arm_smmu_ste *target);
 
 int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
-                               struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
+                               struct arm_smmu_cmdq *cmdq,
+                               struct arm_smmu_cmd *cmds, int n,
                                bool sync);
 
 #ifdef CONFIG_ARM_SMMU_V3_SVA

diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
index b4d8c1f2fd38784d91b6a2ba890709ae8b32b052..67be62a6e7640a1bfad35cc3668b90ca94ea9d35 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
+++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
@@ -427,16 +427,16 @@ tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu,
 static void tegra241_vcmdq_hw_flush_timeout(struct tegra241_vcmdq *vcmdq)
 {
        struct arm_smmu_device *smmu = &vcmdq->cmdqv->smmu;
-       u64 cmd_sync[CMDQ_ENT_DWORDS] = {};
+       struct arm_smmu_cmd cmd_sync = {};
 
-       cmd_sync[0] = FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CMD_SYNC) |
-                     FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE);
+       cmd_sync.data[0] = FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CMD_SYNC) |
+                          FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE);
 
        /*
         * It does not hurt to insert another CMD_SYNC, taking advantage of the
         * arm_smmu_cmdq_issue_cmdlist() that waits for the CMD_SYNC completion.
         */
-       arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, cmd_sync, 1, true);
+       arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &cmd_sync, 1, true);
 }
 
 /* This function is for LVCMDQ, so @vcmdq must not be unmapped yet */