* Builds an invalidation address which is suitable for one page or multiple
* pages. Sets the size bit (S) as needed is more than one page is flushed.
*/
-------- -static inline u64 build_inv_address(u64 address, size_t size)
++++++++ +static inline u64 build_inv_address(u64 address, u64 last)
{
-------- - u64 pages, end, msb_diff;
------- -
------- - pages = iommu_num_pages(address, size, PAGE_SIZE);
++++++++ + unsigned int sz_lg2;
- pages = iommu_num_pages(address, size, PAGE_SIZE);
-
-------- - if (pages == 1)
-------- - return address & PAGE_MASK;
-------- -
-------- - end = address + size - 1;
++++++++ + address &= GENMASK_U64(63, 12);
++++++++ + sz_lg2 = fls64(address ^ last);
++++++++ + if (sz_lg2 <= 12)
++++++++ + return address;
/*
-------- - * msb_diff would hold the index of the most significant bit that
-------- - * flipped between the start and end.
++++++++ + * Encode sz_lg2 according to Table 14: Example Page Size Encodings
++++++++ + *
++++++++ + * See "Note *":
++++++++ + * Address bits 51:32 can be used to encode page sizes greater
++++++++ + * that 4 Gbytes.
++++++++ + * Which we take to mean that the highest page size has bit
++++++++ + * [51]=0, [50:12]=1
++++++++ + * and that coding happens when sz_lg2 is 52. Fall back to full
++++++++ + * invalidation if the size is too big.
++++++++ + *
*/
-------- - msb_diff = fls64(end ^ address) - 1;
++++++++ + if (unlikely(sz_lg2 > 52))
++++++++ + return CMD_INV_IOMMU_ALL_PAGES_ADDRESS |
++++++++ + CMD_INV_IOMMU_PAGES_SIZE_MASK;
/*
-------- - * Bits 63:52 are sign extended. If for some reason bit 51 is different
-------- - * between the start and the end, invalidate everything.
++++++++ + * The sz_lg2 calculation with fls() ensures that:
++++++++ + * address & BIT(sz_lg2 - 1) == 0
++++++++ + * Therefore only the 1's need to be added. 8KB requires no 1's
*/
-------- - if (unlikely(msb_diff > 51)) {
-------- - address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
-------- - } else {
-------- - /*
-------- - * The msb-bit must be clear on the address. Just set all the
-------- - * lower bits.
-------- - */
-------- - address |= (1ull << msb_diff) - 1;
-------- - }
-------- -
-------- - /* Clear bits 11:0 */
-------- - address &= PAGE_MASK;
-------- -
-------- - /* Set the size bit - we flush more than one 4kb page */
++++++++ + if (sz_lg2 > 13)
++++++++ + address |= GENMASK_U64(sz_lg2 - 2, 12);
return address | CMD_INV_IOMMU_PAGES_SIZE_MASK;
}
if (cmds->num == CMDQ_BATCH_ENTRIES) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
cmds->num, false);
-- ------- arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
- }
-
- index = cmds->num * CMDQ_ENT_DWORDS;
- if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
- dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
- cmd->opcode);
- return;
++ +++++++ arm_smmu_cmdq_batch_init_cmd(smmu, cmds, cmd);
}
-- ------ index = cmds->num * CMDQ_ENT_DWORDS;
-- ------ if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
-- ------ dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
-- ------ cmd->opcode);
-- ------ return;
-- ------ }
-- ------
-- ------- cmds->num++;
++ +++++++ cmds->cmds[cmds->num++] = *cmd;
}
++ +++++++#define arm_smmu_cmdq_batch_add_cmd(smmu, cmds, cmd) \
++ +++++++ ({ \
++ +++++++ struct arm_smmu_cmd __cmd = cmd; \
++ +++++++ arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &__cmd); \
++ +++++++ })
++ +++++++
static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds)
{
PT_ITEM_WORD_SIZE);
}
------ ---static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather,
------ --- struct pt_iommu *iommu_table, pt_vaddr_t iova,
------ --- pt_vaddr_t len,
------ --- struct iommu_pages_list *free_list)
++++++ +++struct iommupt_pending_gather {
++++++ +++ struct iommu_iotlb_gather *iotlb_gather;
++++++ +++ struct iommu_pages_list free_list;
++++++ +++ u8 leaf_levels_bitmap;
++++++ +++ u8 table_levels_bitmap;
++++++ +++};
++++++ +++
++++++ +++static void gather_add_table(struct iommupt_pending_gather *pending,
++++++ +++ const struct pt_state *pts,
++++++ +++ struct pt_table_p *table)
++++++ +++{
++++++ +++ iommu_pages_list_add(&pending->free_list, table);
++++++ +++ if (pts_feature(pts, PT_FEAT_DETAILED_GATHER))
++++++ +++ pending->table_levels_bitmap |= BIT(pts->level);
++++++ +++}
++++++ +++
++++++ +++static void gather_add_leaf(struct iommupt_pending_gather *pending,
++++++ +++ const struct pt_state *pts)
++++++ ++ {
++++++ +++ if (!pts_feature(pts, PT_FEAT_DETAILED_GATHER))
++++++ +++ return;
++++++ +++
++++++ +++ pending->leaf_levels_bitmap |= BIT(pts->level);
++++++ +++}
++++++ +++
++++++ +++static void gather_range_pending(struct iommupt_pending_gather *pending,
++++++ +++ struct pt_iommu *iommu_table, pt_vaddr_t iova,
++++++ +++ pt_vaddr_t len)
+{
++++++ +++ struct iommu_iotlb_gather *iotlb_gather = pending->iotlb_gather;
struct pt_common *common = common_from_iommu(iommu_table);
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT))
projects / thirdparty / kernel / linux.git / commitdiff
