iommu/arm-smmu-v3: Avoid back-to-back CMD_SYNC operations

Putting adjacent CMD_SYNCs into the command queue is nonsensical, but
can happen when multiple CPUs are inserting commands. Rather than leave
the poor old hardware to chew through these operations, we can instead
drop the subsequent SYNCs and poll for completion of the first. This
has been shown to improve IO performance under pressure, where the
number of SYNC operations reduces by about a third:

	CMD_SYNCs reduced:	19542181
	CMD_SYNCs total:	58098548	(include reduced)
	CMDs total:		116197099	(TLBI:SYNC about 1:1)

Signed-off-by: Zhen Lei <thunder.leizhen@huawei.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 48baffd07aef3cdee09a8e2688af384064f82d45..e395f1ff3f8123826961acb1116d5260080d595e 100644 (file)
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -568,6 +568,7 @@ struct arm_smmu_device {
        int                             gerr_irq;
        int                             combined_irq;
        u32                             sync_nr;
+       u8                              prev_cmd_opcode;
 
        unsigned long                   ias; /* IPA */
        unsigned long                   oas; /* PA */
@@ -901,6 +902,8 @@ static void arm_smmu_cmdq_insert_cmd(struct arm_smmu_device *smmu, u64 *cmd)
        struct arm_smmu_queue *q = &smmu->cmdq.q;
        bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
 
+       smmu->prev_cmd_opcode = FIELD_GET(CMDQ_0_OP, cmd[0]);
+
        while (queue_insert_raw(q, cmd) == -ENOSPC) {
                if (queue_poll_cons(q, false, wfe))
                        dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
@@ -953,9 +956,16 @@ static int __arm_smmu_cmdq_issue_sync_msi(struct arm_smmu_device *smmu)
        };
 
        spin_lock_irqsave(&smmu->cmdq.lock, flags);
-       ent.sync.msidata = ++smmu->sync_nr;
-       arm_smmu_cmdq_build_cmd(cmd, &ent);
-       arm_smmu_cmdq_insert_cmd(smmu, cmd);
+
+       /* Piggy-back on the previous command if it's a SYNC */
+       if (smmu->prev_cmd_opcode == CMDQ_OP_CMD_SYNC) {
+               ent.sync.msidata = smmu->sync_nr;
+       } else {
+               ent.sync.msidata = ++smmu->sync_nr;
+               arm_smmu_cmdq_build_cmd(cmd, &ent);
+               arm_smmu_cmdq_insert_cmd(smmu, cmd);
+       }
+
        spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
 
        return __arm_smmu_sync_poll_msi(smmu, ent.sync.msidata);