iommu/arm-smmu-v3: Directly encode CMDQ_OP_ATC_INV

Add a new command make function and convert all the places using
ATC_INV.

Split out full invalidation to directly make the cmd instead of
overloading size=0 to mean full invalidation.

In section "3.9.1 ATS Interface" of F.b the specification says:

  When the SMMU returns an ATS Translation Completion for a request that
  had a PASID, the Global bit of the Translation Completion Data Entry
  must be zero.

Even though it faithfully forwards the G bit through to the ATS
invalidation command there is no way to create G mappings so there is
never any need to send a G invalidation. Thus don't expose global in the
new helpers and leave CMDQ_ATC_0_GLOBAL unused.

Reviewed-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Pranjal Shrivastava <praan@google.com>
Tested-by: Pranjal Shrivastava <praan@google.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Nicolin Chen <nicolinc@nvidia.com>
Tested-by: Samiullah Khawaja <skhawaja@google.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 340b60d1d10795bdfc3c88e7c9782dc0f2928cdb..d30bac229d68f15560f73d0e4a8f5941ad48078e 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -308,14 +308,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out,
        case CMDQ_OP_TLBI_EL2_ASID:
                cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
                break;
-       case CMDQ_OP_ATC_INV:
-               cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
-               cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
-               cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
-               cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
-               cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
-               cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
-               break;
        case CMDQ_OP_CMD_SYNC:
                if (ent->sync.msiaddr) {
                        cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
@@ -2371,9 +2363,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
        return IRQ_WAKE_THREAD;
 }
 
-static void
-arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
-                       struct arm_smmu_cmdq_ent *cmd)
+static struct arm_smmu_cmd
+arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size)
 {
        size_t log2_span;
        size_t span_mask;
@@ -2395,17 +2386,6 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
         * This has the unpleasant side-effect of invalidating all PASID-tagged
         * ATC entries within the address range.
         */
-       *cmd = (struct arm_smmu_cmdq_ent) {
-               .opcode                 = CMDQ_OP_ATC_INV,
-               .substream_valid        = (ssid != IOMMU_NO_PASID),
-               .atc.ssid               = ssid,
-       };
-
-       if (!size) {
-               cmd->atc.size = ATC_INV_SIZE_ALL;
-               return;
-       }
-
        page_start      = iova >> inval_grain_shift;
        page_end        = (iova + size - 1) >> inval_grain_shift;
 
@@ -2434,24 +2414,25 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
 
        page_start      &= ~span_mask;
 
-       cmd->atc.addr   = page_start << inval_grain_shift;
-       cmd->atc.size   = log2_span;
+       return arm_smmu_make_cmd_atc_inv(sid, ssid,
+                                        page_start << inval_grain_shift,
+                                        log2_span);
 }
 
 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
                                   ioasid_t ssid)
 {
        int i;
-       struct arm_smmu_cmdq_ent cmd;
+       struct arm_smmu_cmd cmd;
        struct arm_smmu_cmdq_batch cmds;
 
-       arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
-
-       arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
-       for (i = 0; i < master->num_streams; i++) {
-               cmd.atc.sid = master->streams[i].id;
-               arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
-       }
+       cmd = arm_smmu_make_cmd_atc_inv_all(0, IOMMU_NO_PASID);
+       arm_smmu_cmdq_batch_init_cmd(master->smmu, &cmds, &cmd);
+       for (i = 0; i < master->num_streams; i++)
+               arm_smmu_cmdq_batch_add_cmd(
+                       master->smmu, &cmds,
+                       arm_smmu_make_cmd_atc_inv_all(master->streams[i].id,
+                                                     ssid));
 
        return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
 }
@@ -2650,14 +2631,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
                        arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
                        break;
                case INV_TYPE_ATS:
-                       arm_smmu_atc_inv_to_cmd(cur->ssid, iova, size, &cmd);
-                       cmd.atc.sid = cur->id;
-                       arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
+                       arm_smmu_cmdq_batch_add_cmd(
+                               smmu, &cmds,
+                               arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid,
+                                                       iova, size));
                        break;
                case INV_TYPE_ATS_FULL:
-                       arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
-                       cmd.atc.sid = cur->id;
-                       arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
+                       arm_smmu_cmdq_batch_add_cmd(
+                               smmu, &cmds,
+                               arm_smmu_make_cmd_atc_inv_all(cur->id,
+                                                             IOMMU_NO_PASID));
                        break;
                default:
                        WARN_ON_ONCE(1);

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 10b3d95d9ee660e47cb46822e6a0ff816383d7ec..194f73cabef5c9165bde5848e945f9e07595479b 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -552,6 +552,25 @@ static inline struct arm_smmu_cmd arm_smmu_make_cmd_pri_resp(u32 sid, u32 ssid,
        return cmd;
 }
 
+static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv(u32 sid, u32 ssid,
+                                                           u64 addr, u8 size)
+{
+       struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_ATC_INV);
+
+       cmd.data[0] |= FIELD_PREP(CMDQ_0_SSV, ssid != IOMMU_NO_PASID) |
+                      FIELD_PREP(CMDQ_ATC_0_SSID, ssid) |
+                      FIELD_PREP(CMDQ_ATC_0_SID, sid);
+       cmd.data[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, size) |
+                      (addr & CMDQ_ATC_1_ADDR_MASK);
+       return cmd;
+}
+
+static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv_all(u32 sid,
+                                                               u32 ssid)
+{
+       return arm_smmu_make_cmd_atc_inv(sid, ssid, 0, ATC_INV_SIZE_ALL);
+}
+
 /* Event queue */
 #define EVTQ_ENT_SZ_SHIFT              5
 #define EVTQ_ENT_DWORDS                        ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
@@ -630,14 +649,6 @@ struct arm_smmu_cmdq_ent {
                        u64                     addr;
                } tlbi;
 
-               struct {
-                       u32                     sid;
-                       u32                     ssid;
-                       u64                     addr;
-                       u8                      size;
-                       bool                    global;
-               } atc;
-
                struct {
                        u64                     msiaddr;
                } sync;