From: Jason Gunthorpe Date: Wed, 13 May 2026 23:57:46 +0000 (-0300) Subject: iommu/arm-smmu-v3: Directly encode CMDQ_OP_ATC_INV X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2eedb906f9c605aa789758f9aa172a930c88944f;p=thirdparty%2Fkernel%2Flinux.git iommu/arm-smmu-v3: Directly encode CMDQ_OP_ATC_INV Add a new command make function and convert all the places using ATC_INV. Split out full invalidation to directly make the cmd instead of overloading size=0 to mean full invalidation. In section "3.9.1 ATS Interface" of F.b the specification says: When the SMMU returns an ATS Translation Completion for a request that had a PASID, the Global bit of the Translation Completion Data Entry must be zero. Even though it faithfully forwards the G bit through to the ATS invalidation command there is no way to create G mappings so there is never any need to send a G invalidation. Thus don't expose global in the new helpers and leave CMDQ_ATC_0_GLOBAL unused. Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 340b60d1d1079..d30bac229d68f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -308,14 +308,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out, case CMDQ_OP_TLBI_EL2_ASID: cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); break; - case CMDQ_OP_ATC_INV: - cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); - cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global); - cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid); - cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid); - cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size); - cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK; - break; case CMDQ_OP_CMD_SYNC: if (ent->sync.msiaddr) { cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ); @@ -2371,9 +2363,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) return IRQ_WAKE_THREAD; } -static void -arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, - struct arm_smmu_cmdq_ent *cmd) +static struct arm_smmu_cmd +arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size) { size_t log2_span; size_t span_mask; @@ -2395,17 +2386,6 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, * This has the unpleasant side-effect of invalidating all PASID-tagged * ATC entries within the address range. */ - *cmd = (struct arm_smmu_cmdq_ent) { - .opcode = CMDQ_OP_ATC_INV, - .substream_valid = (ssid != IOMMU_NO_PASID), - .atc.ssid = ssid, - }; - - if (!size) { - cmd->atc.size = ATC_INV_SIZE_ALL; - return; - } - page_start = iova >> inval_grain_shift; page_end = (iova + size - 1) >> inval_grain_shift; @@ -2434,24 +2414,25 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, page_start &= ~span_mask; - cmd->atc.addr = page_start << inval_grain_shift; - cmd->atc.size = log2_span; + return arm_smmu_make_cmd_atc_inv(sid, ssid, + page_start << inval_grain_shift, + log2_span); } static int arm_smmu_atc_inv_master(struct arm_smmu_master *master, ioasid_t ssid) { int i; - struct arm_smmu_cmdq_ent cmd; + struct arm_smmu_cmd cmd; struct arm_smmu_cmdq_batch cmds; - arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd); - - arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd); - for (i = 0; i < master->num_streams; i++) { - cmd.atc.sid = master->streams[i].id; - arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd); - } + cmd = arm_smmu_make_cmd_atc_inv_all(0, IOMMU_NO_PASID); + arm_smmu_cmdq_batch_init_cmd(master->smmu, &cmds, &cmd); + for (i = 0; i < master->num_streams; i++) + arm_smmu_cmdq_batch_add_cmd( + master->smmu, &cmds, + arm_smmu_make_cmd_atc_inv_all(master->streams[i].id, + ssid)); return arm_smmu_cmdq_batch_submit(master->smmu, &cmds); } @@ -2650,14 +2631,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); break; case INV_TYPE_ATS: - arm_smmu_atc_inv_to_cmd(cur->ssid, iova, size, &cmd); - cmd.atc.sid = cur->id; - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + arm_smmu_cmdq_batch_add_cmd( + smmu, &cmds, + arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid, + iova, size)); break; case INV_TYPE_ATS_FULL: - arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd); - cmd.atc.sid = cur->id; - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + arm_smmu_cmdq_batch_add_cmd( + smmu, &cmds, + arm_smmu_make_cmd_atc_inv_all(cur->id, + IOMMU_NO_PASID)); break; default: WARN_ON_ONCE(1); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 10b3d95d9ee66..194f73cabef5c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -552,6 +552,25 @@ static inline struct arm_smmu_cmd arm_smmu_make_cmd_pri_resp(u32 sid, u32 ssid, return cmd; } +static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv(u32 sid, u32 ssid, + u64 addr, u8 size) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_ATC_INV); + + cmd.data[0] |= FIELD_PREP(CMDQ_0_SSV, ssid != IOMMU_NO_PASID) | + FIELD_PREP(CMDQ_ATC_0_SSID, ssid) | + FIELD_PREP(CMDQ_ATC_0_SID, sid); + cmd.data[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, size) | + (addr & CMDQ_ATC_1_ADDR_MASK); + return cmd; +} + +static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv_all(u32 sid, + u32 ssid) +{ + return arm_smmu_make_cmd_atc_inv(sid, ssid, 0, ATC_INV_SIZE_ALL); +} + /* Event queue */ #define EVTQ_ENT_SZ_SHIFT 5 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) @@ -630,14 +649,6 @@ struct arm_smmu_cmdq_ent { u64 addr; } tlbi; - struct { - u32 sid; - u32 ssid; - u64 addr; - u8 size; - bool global; - } atc; - struct { u64 msiaddr; } sync;